2023-10-09T08:11:18.760 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T08:11:18.761 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --name teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --verbose --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587 -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587/orig.config.yaml 2023-10-09T08:11:18.820 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T08:11:18.913 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7417587' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 kernel: hwe: true workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 3687 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi016.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi094.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T08:11:18.913 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T08:11:18.914 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T08:11:18.914 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T08:11:18.916 INFO:teuthology.task.internal:Checking packages... 2023-10-09T08:11:18.939 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T08:11:18.939 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:11:18.939 INFO:teuthology.packaging:ref: None 2023-10-09T08:11:18.940 INFO:teuthology.packaging:tag: None 2023-10-09T08:11:18.940 INFO:teuthology.packaging:branch: reef 2023-10-09T08:11:18.940 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:11:18.940 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-10-09T08:11:19.243 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610-g397de1b0-1focal 2023-10-09T08:11:19.243 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T08:11:19.249 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T08:11:19.249 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T08:11:19.276 INFO:teuthology.task.internal:Saving configuration 2023-10-09T08:11:19.298 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T08:11:19.304 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T08:11:19.330 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi016.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:05:17.397456', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T08:11:19.354 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi094.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:05:17.395501', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T08:11:19.354 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T08:11:19.360 INFO:teuthology.task.internal:roles: ubuntu@smithi016.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T08:11:19.361 INFO:teuthology.task.internal:roles: ubuntu@smithi094.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T08:11:19.361 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T08:11:19.459 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f816b7f4160>, signals=[15]) 2023-10-09T08:11:19.460 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T08:11:19.466 INFO:teuthology.task.internal:Opening connections... 2023-10-09T08:11:19.466 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi016.front.sepia.ceph.com 2023-10-09T08:11:19.468 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:11:19.518 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi094.front.sepia.ceph.com 2023-10-09T08:11:19.518 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:11:19.567 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T08:11:19.574 DEBUG:teuthology.orchestra.run.smithi016:> uname -m 2023-10-09T08:11:19.603 INFO:teuthology.orchestra.run.smithi016.stdout:x86_64 2023-10-09T08:11:19.603 DEBUG:teuthology.orchestra.run.smithi016:> cat /etc/os-release 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:NAME="Ubuntu" 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:ID=ubuntu 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:ID_LIKE=debian 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:VERSION_ID="20.04" 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T08:11:19.653 INFO:teuthology.orchestra.run.smithi016.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T08:11:19.654 INFO:teuthology.orchestra.run.smithi016.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T08:11:19.654 INFO:teuthology.orchestra.run.smithi016.stdout:VERSION_CODENAME=focal 2023-10-09T08:11:19.654 INFO:teuthology.orchestra.run.smithi016.stdout:UBUNTU_CODENAME=focal 2023-10-09T08:11:19.654 INFO:teuthology.lock.ops:Updating smithi016.front.sepia.ceph.com on lock server 2023-10-09T08:11:19.673 DEBUG:teuthology.orchestra.run.smithi094:> uname -m 2023-10-09T08:11:19.680 INFO:teuthology.orchestra.run.smithi094.stdout:x86_64 2023-10-09T08:11:19.680 DEBUG:teuthology.orchestra.run.smithi094:> cat /etc/os-release 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:NAME="Ubuntu" 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:ID=ubuntu 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:ID_LIKE=debian 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:VERSION_ID="20.04" 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T08:11:19.733 INFO:teuthology.orchestra.run.smithi094.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T08:11:19.734 INFO:teuthology.orchestra.run.smithi094.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T08:11:19.734 INFO:teuthology.orchestra.run.smithi094.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T08:11:19.734 INFO:teuthology.orchestra.run.smithi094.stdout:VERSION_CODENAME=focal 2023-10-09T08:11:19.734 INFO:teuthology.orchestra.run.smithi094.stdout:UBUNTU_CODENAME=focal 2023-10-09T08:11:19.734 INFO:teuthology.lock.ops:Updating smithi094.front.sepia.ceph.com on lock server 2023-10-09T08:11:19.765 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T08:11:19.774 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T08:11:19.801 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T08:11:19.801 DEBUG:teuthology.orchestra.run.smithi016:> test '!' -e /home/ubuntu/cephtest 2023-10-09T08:11:19.803 DEBUG:teuthology.orchestra.run.smithi094:> test '!' -e /home/ubuntu/cephtest 2023-10-09T08:11:19.809 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T08:11:19.859 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T08:11:19.859 DEBUG:teuthology.orchestra.run.smithi016:> test -z $(ls -A /var/lib/ceph) 2023-10-09T08:11:19.862 DEBUG:teuthology.orchestra.run.smithi094:> test -z $(ls -A /var/lib/ceph) 2023-10-09T08:11:19.879 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T08:11:19.933 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T08:11:19.949 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T08:11:19.949 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-10-09T08:11:19.950 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-10-09T08:11:19.950 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-10-09T08:11:19.951 DEBUG:teuthology.orchestra.run.smithi016:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T08:11:19.951 DEBUG:teuthology.orchestra.run.smithi094:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T08:11:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:11:19.957 DEBUG:teuthology.orchestra.run.smithi016:> uname -r 2023-10-09T08:11:19.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:11:19.958 DEBUG:teuthology.orchestra.run.smithi094:> uname -r 2023-10-09T08:11:19.965 INFO:teuthology.orchestra.run.smithi016.stdout:5.4.0-124-generic 2023-10-09T08:11:19.965 INFO:teuthology.task.kernel:Running kernel on smithi016: 5.4.0-124-generic 2023-10-09T08:11:19.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get clean 2023-10-09T08:11:19.966 INFO:teuthology.orchestra.run.smithi094.stdout:5.4.0-124-generic 2023-10-09T08:11:19.967 INFO:teuthology.task.kernel:Running kernel on smithi094: 5.4.0-124-generic 2023-10-09T08:11:19.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo apt-get clean 2023-10-09T08:11:20.072 DEBUG:teuthology.orchestra.run.smithi094:> sudo apt-get update 2023-10-09T08:11:20.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get update 2023-10-09T08:11:20.240 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:11:20.243 INFO:teuthology.orchestra.run.smithi016.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T08:11:20.252 INFO:teuthology.orchestra.run.smithi094.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T08:11:20.313 INFO:teuthology.orchestra.run.smithi094.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T08:11:20.355 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:11:20.356 INFO:teuthology.orchestra.run.smithi094.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T08:11:20.438 INFO:teuthology.orchestra.run.smithi016.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T08:11:20.523 INFO:teuthology.orchestra.run.smithi016.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T08:11:20.621 INFO:teuthology.orchestra.run.smithi094.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T08:11:20.646 INFO:teuthology.orchestra.run.smithi016.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T08:11:20.659 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T08:11:20.661 INFO:teuthology.orchestra.run.smithi016.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T08:11:20.661 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T08:11:20.662 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T08:11:20.677 INFO:teuthology.orchestra.run.smithi016.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T08:11:20.679 INFO:teuthology.orchestra.run.smithi016.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:11:20.679 INFO:teuthology.orchestra.run.smithi016.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T08:11:20.684 INFO:teuthology.orchestra.run.smithi016.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T08:11:20.690 INFO:teuthology.orchestra.run.smithi016.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T08:11:20.691 INFO:teuthology.orchestra.run.smithi016.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T08:11:20.696 INFO:teuthology.orchestra.run.smithi016.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T08:11:20.696 INFO:teuthology.orchestra.run.smithi016.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T08:11:20.697 INFO:teuthology.orchestra.run.smithi016.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T08:11:20.697 INFO:teuthology.orchestra.run.smithi016.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T08:11:20.703 INFO:teuthology.orchestra.run.smithi094.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T08:11:20.743 INFO:teuthology.orchestra.run.smithi094.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T08:11:20.744 INFO:teuthology.orchestra.run.smithi094.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T08:11:20.744 INFO:teuthology.orchestra.run.smithi094.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T08:11:20.759 INFO:teuthology.orchestra.run.smithi094.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T08:11:20.759 INFO:teuthology.orchestra.run.smithi094.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T08:11:20.763 INFO:teuthology.orchestra.run.smithi094.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:11:20.763 INFO:teuthology.orchestra.run.smithi094.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T08:11:20.769 INFO:teuthology.orchestra.run.smithi094.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T08:11:20.778 INFO:teuthology.orchestra.run.smithi094.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T08:11:20.780 INFO:teuthology.orchestra.run.smithi094.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T08:11:20.780 INFO:teuthology.orchestra.run.smithi094.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T08:11:20.780 INFO:teuthology.orchestra.run.smithi094.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T08:11:20.781 INFO:teuthology.orchestra.run.smithi016.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T08:11:20.781 INFO:teuthology.orchestra.run.smithi094.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T08:11:20.781 INFO:teuthology.orchestra.run.smithi094.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T08:11:20.794 INFO:teuthology.orchestra.run.smithi094.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T08:11:20.794 INFO:teuthology.orchestra.run.smithi094.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T08:11:20.794 INFO:teuthology.orchestra.run.smithi094.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T08:11:20.810 INFO:teuthology.orchestra.run.smithi094.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T08:11:20.811 INFO:teuthology.orchestra.run.smithi094.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T08:11:20.811 INFO:teuthology.orchestra.run.smithi094.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T08:11:20.922 INFO:teuthology.orchestra.run.smithi016.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T08:11:20.937 INFO:teuthology.orchestra.run.smithi094.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T08:11:21.169 INFO:teuthology.orchestra.run.smithi016.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T08:11:21.333 INFO:teuthology.orchestra.run.smithi016.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T08:11:21.345 INFO:teuthology.orchestra.run.smithi016.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T08:11:21.346 INFO:teuthology.orchestra.run.smithi016.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T08:11:21.346 INFO:teuthology.orchestra.run.smithi016.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T08:11:21.408 INFO:teuthology.orchestra.run.smithi016.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T08:11:21.412 INFO:teuthology.orchestra.run.smithi094.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T08:11:21.426 INFO:teuthology.orchestra.run.smithi016.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:11:21.427 INFO:teuthology.orchestra.run.smithi094.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T08:11:21.436 INFO:teuthology.orchestra.run.smithi094.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T08:11:21.436 INFO:teuthology.orchestra.run.smithi094.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T08:11:21.511 INFO:teuthology.orchestra.run.smithi094.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T08:11:21.511 INFO:teuthology.orchestra.run.smithi094.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T08:11:21.518 INFO:teuthology.orchestra.run.smithi094.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:11:21.518 INFO:teuthology.orchestra.run.smithi094.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T08:11:21.541 INFO:teuthology.orchestra.run.smithi094.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T08:11:21.590 INFO:teuthology.orchestra.run.smithi016.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T08:11:21.717 INFO:teuthology.orchestra.run.smithi094.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T08:11:22.132 INFO:teuthology.orchestra.run.smithi094.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T08:11:22.138 INFO:teuthology.orchestra.run.smithi094.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T08:11:22.140 INFO:teuthology.orchestra.run.smithi094.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T08:11:22.148 INFO:teuthology.orchestra.run.smithi094.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T08:11:22.150 INFO:teuthology.orchestra.run.smithi094.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T08:11:22.224 INFO:teuthology.orchestra.run.smithi016.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T08:11:22.262 INFO:teuthology.orchestra.run.smithi016.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T08:11:22.275 INFO:teuthology.orchestra.run.smithi016.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T08:11:22.277 INFO:teuthology.orchestra.run.smithi016.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T08:11:22.277 INFO:teuthology.orchestra.run.smithi016.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T08:11:22.278 INFO:teuthology.orchestra.run.smithi016.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T08:11:22.279 INFO:teuthology.orchestra.run.smithi016.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T08:11:22.279 INFO:teuthology.orchestra.run.smithi016.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T08:11:22.305 INFO:teuthology.orchestra.run.smithi016.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T08:11:22.307 INFO:teuthology.orchestra.run.smithi016.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T08:11:22.472 INFO:teuthology.orchestra.run.smithi016.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T08:11:22.635 INFO:teuthology.orchestra.run.smithi016.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T08:11:22.687 INFO:teuthology.orchestra.run.smithi016.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T08:11:24.574 INFO:teuthology.orchestra.run.smithi094.stdout:Fetched 17.6 MB in 3s (6,896 kB/s) 2023-10-09T08:11:24.717 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 17.6 MB in 3s (6,934 kB/s) 2023-10-09T08:11:26.041 INFO:teuthology.orchestra.run.smithi094.stdout:Reading package lists... 2023-10-09T08:11:26.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:11:26.134 INFO:teuthology.orchestra.run.smithi094.stdout:Reading package lists... 2023-10-09T08:11:26.208 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-10-09T08:11:26.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:11:26.304 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-10-09T08:11:26.336 INFO:teuthology.orchestra.run.smithi094.stdout:Building dependency tree... 2023-10-09T08:11:26.336 INFO:teuthology.orchestra.run.smithi094.stdout:Reading state information... 2023-10-09T08:11:26.506 INFO:teuthology.orchestra.run.smithi094.stdout:The following additional packages will be installed: 2023-10-09T08:11:26.506 INFO:teuthology.orchestra.run.smithi094.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:11:26.506 INFO:teuthology.orchestra.run.smithi094.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T08:11:26.506 INFO:teuthology.orchestra.run.smithi094.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T08:11:26.507 INFO:teuthology.orchestra.run.smithi094.stdout:Suggested packages: 2023-10-09T08:11:26.507 INFO:teuthology.orchestra.run.smithi094.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T08:11:26.507 INFO:teuthology.orchestra.run.smithi094.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T08:11:26.513 INFO:teuthology.orchestra.run.smithi016.stdout:Building dependency tree... 2023-10-09T08:11:26.513 INFO:teuthology.orchestra.run.smithi016.stdout:Reading state information... 2023-10-09T08:11:26.564 INFO:teuthology.orchestra.run.smithi094.stdout:The following NEW packages will be installed: 2023-10-09T08:11:26.565 INFO:teuthology.orchestra.run.smithi094.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:11:26.565 INFO:teuthology.orchestra.run.smithi094.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T08:11:26.565 INFO:teuthology.orchestra.run.smithi094.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T08:11:26.565 INFO:teuthology.orchestra.run.smithi094.stdout: thermald upower usbmuxd 2023-10-09T08:11:26.694 INFO:teuthology.orchestra.run.smithi016.stdout:The following additional packages will be installed: 2023-10-09T08:11:26.695 INFO:teuthology.orchestra.run.smithi016.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:11:26.695 INFO:teuthology.orchestra.run.smithi016.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T08:11:26.695 INFO:teuthology.orchestra.run.smithi016.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T08:11:26.696 INFO:teuthology.orchestra.run.smithi016.stdout:Suggested packages: 2023-10-09T08:11:26.696 INFO:teuthology.orchestra.run.smithi016.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T08:11:26.696 INFO:teuthology.orchestra.run.smithi016.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T08:11:26.748 INFO:teuthology.orchestra.run.smithi094.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:11:26.748 INFO:teuthology.orchestra.run.smithi094.stdout:Need to get 90.9 MB of archives. 2023-10-09T08:11:26.749 INFO:teuthology.orchestra.run.smithi094.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T08:11:26.749 INFO:teuthology.orchestra.run.smithi094.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T08:11:26.753 INFO:teuthology.orchestra.run.smithi016.stdout:The following NEW packages will be installed: 2023-10-09T08:11:26.754 INFO:teuthology.orchestra.run.smithi016.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:11:26.754 INFO:teuthology.orchestra.run.smithi016.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T08:11:26.754 INFO:teuthology.orchestra.run.smithi016.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T08:11:26.755 INFO:teuthology.orchestra.run.smithi016.stdout: thermald upower usbmuxd 2023-10-09T08:11:26.812 INFO:teuthology.orchestra.run.smithi016.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:11:26.812 INFO:teuthology.orchestra.run.smithi016.stdout:Need to get 90.9 MB of archives. 2023-10-09T08:11:26.812 INFO:teuthology.orchestra.run.smithi016.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T08:11:26.812 INFO:teuthology.orchestra.run.smithi016.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T08:11:26.881 INFO:teuthology.orchestra.run.smithi016.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T08:11:26.887 INFO:teuthology.orchestra.run.smithi016.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T08:11:26.892 INFO:teuthology.orchestra.run.smithi016.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T08:11:26.910 INFO:teuthology.orchestra.run.smithi016.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T08:11:26.918 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T08:11:27.022 INFO:teuthology.orchestra.run.smithi094.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T08:11:27.051 INFO:teuthology.orchestra.run.smithi094.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T08:11:27.069 INFO:teuthology.orchestra.run.smithi094.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T08:11:27.141 INFO:teuthology.orchestra.run.smithi094.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T08:11:27.177 INFO:teuthology.orchestra.run.smithi094.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T08:11:27.232 INFO:teuthology.orchestra.run.smithi016.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T08:11:27.335 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T08:11:28.102 INFO:teuthology.orchestra.run.smithi094.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T08:11:28.132 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T08:11:28.133 INFO:teuthology.orchestra.run.smithi016.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T08:11:28.133 INFO:teuthology.orchestra.run.smithi016.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T08:11:28.138 INFO:teuthology.orchestra.run.smithi016.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T08:11:28.140 INFO:teuthology.orchestra.run.smithi016.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T08:11:28.410 INFO:teuthology.orchestra.run.smithi094.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T08:11:28.798 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 90.9 MB in 1s (65.9 MB/s) 2023-10-09T08:11:28.984 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T08:11:30.307 INFO:teuthology.orchestra.run.smithi094.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T08:11:30.399 INFO:teuthology.orchestra.run.smithi094.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T08:11:30.400 INFO:teuthology.orchestra.run.smithi094.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T08:11:30.570 INFO:teuthology.orchestra.run.smithi094.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T08:11:30.718 INFO:teuthology.orchestra.run.smithi016.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T08:11:30.723 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T08:11:30.917 INFO:teuthology.orchestra.run.smithi094.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T08:11:30.931 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:11:31.310 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T08:11:31.324 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T08:11:31.384 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:11:31.569 INFO:teuthology.orchestra.run.smithi094.stdout:Fetched 90.9 MB in 4s (20.8 MB/s) 2023-10-09T08:11:31.742 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T08:11:31.754 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T08:11:31.768 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T08:11:31.813 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:11:32.182 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T08:11:32.196 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T08:11:32.240 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:11:32.560 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T08:11:32.575 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T08:11:32.610 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:11:32.980 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T08:11:32.993 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:11:33.030 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:33.513 INFO:teuthology.orchestra.run.smithi094.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T08:11:33.518 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T08:11:33.649 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:11:33.985 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T08:11:33.999 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T08:11:34.043 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:11:34.363 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T08:11:34.378 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T08:11:34.421 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:11:34.758 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T08:11:34.771 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T08:11:34.816 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:11:35.103 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T08:11:35.118 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T08:11:35.161 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:11:35.489 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T08:11:35.503 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:11:35.572 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:35.689 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T08:11:35.704 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:11:35.801 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:36.325 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T08:11:36.341 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:11:36.375 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:38.025 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T08:11:38.038 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:11:38.147 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:38.686 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T08:11:38.699 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:11:38.744 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:43.602 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T08:11:43.618 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T08:11:43.661 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:11:44.129 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T08:11:44.144 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T08:11:44.187 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:11:44.615 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package thermald. 2023-10-09T08:11:44.629 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T08:11:44.674 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:11:45.262 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package upower. 2023-10-09T08:11:45.278 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T08:11:45.331 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T08:11:45.800 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T08:11:45.816 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T08:11:45.858 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:11:45.975 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T08:11:45.991 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T08:11:46.033 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:11:46.181 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:11:46.330 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:11:46.352 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T08:11:46.366 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T08:11:46.410 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:11:46.439 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:11:46.598 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:11:46.724 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:11:46.772 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package thermald. 2023-10-09T08:11:46.789 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T08:11:46.830 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:11:46.875 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:11:47.001 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T08:11:47.377 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package upower. 2023-10-09T08:11:47.391 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T08:11:47.477 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T08:11:47.874 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T08:11:47.888 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T08:11:47.905 INFO:teuthology.orchestra.run.smithi016.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T08:11:47.932 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:11:47.966 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:11:48.161 INFO:teuthology.orchestra.run.smithi016.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T08:11:48.169 INFO:teuthology.orchestra.run.smithi016.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T08:11:48.169 INFO:teuthology.orchestra.run.smithi016.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T08:11:48.254 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:11:48.378 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:11:48.513 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:11:48.639 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:11:48.722 INFO:teuthology.orchestra.run.smithi016.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T08:11:48.765 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:11:48.891 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:11:48.938 INFO:teuthology.orchestra.run.smithi016.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T08:11:49.036 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T08:11:49.069 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:11:49.406 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:11:49.406 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:11:49.572 INFO:teuthology.orchestra.run.smithi094.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T08:11:49.638 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:11:49.849 INFO:teuthology.orchestra.run.smithi094.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T08:11:49.858 INFO:teuthology.orchestra.run.smithi094.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T08:11:49.858 INFO:teuthology.orchestra.run.smithi094.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T08:11:49.889 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:50.339 INFO:teuthology.orchestra.run.smithi094.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T08:11:50.551 INFO:teuthology.orchestra.run.smithi094.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T08:11:50.594 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:11:50.901 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:11:50.902 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:11:51.257 INFO:teuthology.orchestra.run.smithi016.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:11:51.258 INFO:teuthology.orchestra.run.smithi016.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T08:11:51.358 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:51.459 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:52.855 INFO:teuthology.orchestra.run.smithi094.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:11:52.855 INFO:teuthology.orchestra.run.smithi094.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T08:11:53.452 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:53.755 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:54.870 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:11:55.005 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:11:55.155 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:55.293 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:11:56.130 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T08:11:56.255 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:56.334 INFO:teuthology.orchestra.run.smithi016.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T08:11:56.334 INFO:teuthology.orchestra.run.smithi016.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:11:56.516 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:11:56.625 INFO:teuthology.orchestra.run.smithi094.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:11:56.881 INFO:teuthology.orchestra.run.smithi094.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:11:57.676 INFO:teuthology.orchestra.run.smithi094.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T08:11:57.759 INFO:teuthology.orchestra.run.smithi094.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:11:57.826 INFO:teuthology.orchestra.run.smithi094.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T08:11:57.827 INFO:teuthology.orchestra.run.smithi094.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:12:22.414 INFO:teuthology.orchestra.run.smithi016.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T08:12:22.414 INFO:teuthology.orchestra.run.smithi016.stdout:Sourcing file `/etc/default/grub' 2023-10-09T08:12:22.423 INFO:teuthology.orchestra.run.smithi016.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:12:22.452 INFO:teuthology.orchestra.run.smithi016.stdout:Generating grub configuration file ... 2023-10-09T08:12:22.907 INFO:teuthology.orchestra.run.smithi016.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:12:22.946 INFO:teuthology.orchestra.run.smithi016.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:12:23.276 INFO:teuthology.orchestra.run.smithi094.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T08:12:23.276 INFO:teuthology.orchestra.run.smithi094.stdout:Sourcing file `/etc/default/grub' 2023-10-09T08:12:23.290 INFO:teuthology.orchestra.run.smithi094.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:12:23.315 INFO:teuthology.orchestra.run.smithi094.stdout:Generating grub configuration file ... 2023-10-09T08:12:23.338 INFO:teuthology.orchestra.run.smithi016.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:12:23.350 INFO:teuthology.orchestra.run.smithi016.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:12:23.610 INFO:teuthology.orchestra.run.smithi094.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:12:23.642 INFO:teuthology.orchestra.run.smithi094.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:12:24.043 INFO:teuthology.orchestra.run.smithi094.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:12:24.055 INFO:teuthology.orchestra.run.smithi094.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:12:25.368 INFO:teuthology.orchestra.run.smithi016.stdout:done 2023-10-09T08:12:25.844 DEBUG:teuthology.orchestra.run.smithi016:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:12:25.895 INFO:teuthology.orchestra.run.smithi016.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:12:25.895 INFO:teuthology.orchestra.run.smithi016.stdout:Status: install ok installed 2023-10-09T08:12:25.895 INFO:teuthology.orchestra.run.smithi016.stdout:Priority: optional 2023-10-09T08:12:25.895 INFO:teuthology.orchestra.run.smithi016.stdout:Section: kernel 2023-10-09T08:12:25.895 INFO:teuthology.orchestra.run.smithi016.stdout:Installed-Size: 20 2023-10-09T08:12:25.895 INFO:teuthology.orchestra.run.smithi016.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:12:25.896 INFO:teuthology.orchestra.run.smithi016.stdout:Architecture: amd64 2023-10-09T08:12:25.896 INFO:teuthology.orchestra.run.smithi016.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:12:25.896 INFO:teuthology.orchestra.run.smithi016.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:12:25.896 INFO:teuthology.orchestra.run.smithi016.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:12:25.896 INFO:teuthology.orchestra.run.smithi016.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:12:25.896 INFO:teuthology.orchestra.run.smithi016.stdout:Recommends: thermald 2023-10-09T08:12:25.896 INFO:teuthology.orchestra.run.smithi016.stdout:Description: Generic Linux kernel image 2023-10-09T08:12:25.897 INFO:teuthology.orchestra.run.smithi016.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:12:25.897 INFO:teuthology.orchestra.run.smithi016.stdout: available. 2023-10-09T08:12:25.897 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T08:12:25.897 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T08:12:25.897 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T08:12:25.898 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-10-09T08:12:25.898 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi016.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T08:12:25.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get clean 2023-10-09T08:12:25.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get update 2023-10-09T08:12:26.009 INFO:teuthology.orchestra.run.smithi094.stdout:done 2023-10-09T08:12:26.156 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T08:12:26.274 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:12:26.355 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T08:12:26.431 DEBUG:teuthology.orchestra.run.smithi094:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:12:26.437 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T08:12:26.473 INFO:teuthology.orchestra.run.smithi094.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:12:26.473 INFO:teuthology.orchestra.run.smithi094.stdout:Status: install ok installed 2023-10-09T08:12:26.473 INFO:teuthology.orchestra.run.smithi094.stdout:Priority: optional 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Section: kernel 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Installed-Size: 20 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Architecture: amd64 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Recommends: thermald 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout:Description: Generic Linux kernel image 2023-10-09T08:12:26.474 INFO:teuthology.orchestra.run.smithi094.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:12:26.475 INFO:teuthology.orchestra.run.smithi094.stdout: available. 2023-10-09T08:12:26.475 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T08:12:26.475 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T08:12:26.475 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T08:12:26.475 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-10-09T08:12:26.475 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi094.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T08:12:26.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo apt-get clean 2023-10-09T08:12:26.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo apt-get update 2023-10-09T08:12:26.755 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T08:12:26.869 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:12:26.951 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T08:12:27.033 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T08:12:27.959 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-10-09T08:12:27.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:12:28.052 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-10-09T08:12:28.236 INFO:teuthology.orchestra.run.smithi016.stdout:Building dependency tree... 2023-10-09T08:12:28.237 INFO:teuthology.orchestra.run.smithi016.stdout:Reading state information... 2023-10-09T08:12:28.454 INFO:teuthology.orchestra.run.smithi016.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T08:12:28.454 INFO:teuthology.orchestra.run.smithi016.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:12:28.456 DEBUG:teuthology.orchestra.run.smithi016:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Status: install ok installed 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Priority: optional 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Section: kernel 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Installed-Size: 20 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Architecture: amd64 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:12:28.474 INFO:teuthology.orchestra.run.smithi016.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:12:28.475 INFO:teuthology.orchestra.run.smithi016.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:12:28.475 INFO:teuthology.orchestra.run.smithi016.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:12:28.475 INFO:teuthology.orchestra.run.smithi016.stdout:Recommends: thermald 2023-10-09T08:12:28.475 INFO:teuthology.orchestra.run.smithi016.stdout:Description: Generic Linux kernel image 2023-10-09T08:12:28.475 INFO:teuthology.orchestra.run.smithi016.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:12:28.475 INFO:teuthology.orchestra.run.smithi016.stdout: available. 2023-10-09T08:12:28.475 DEBUG:teuthology.orchestra.run.smithi016:> mktemp 2023-10-09T08:12:28.481 INFO:teuthology.orchestra.run.smithi016.stdout:/tmp/tmp.l5N0JBeyeJ 2023-10-09T08:12:28.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo cp /boot/grub/grub.cfg /tmp/tmp.l5N0JBeyeJ 2023-10-09T08:12:28.496 INFO:teuthology.orchestra.run.smithi094.stdout:Reading package lists... 2023-10-09T08:12:28.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:12:28.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 0666 /tmp/tmp.l5N0JBeyeJ 2023-10-09T08:12:28.576 DEBUG:teuthology.orchestra.remote:smithi016:/tmp/tmp.l5N0JBeyeJ is 9KB 2023-10-09T08:12:28.589 DEBUG:teuthology.orchestra.run.smithi016:> rm -fr /tmp/tmp.l5N0JBeyeJ 2023-10-09T08:12:28.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T08:12:28.597 INFO:teuthology.orchestra.run.smithi094.stdout:Reading package lists... 2023-10-09T08:12:28.654 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:12:28.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T08:12:28.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T08:12:28.686 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T08:12:28.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo update-grub 2023-10-09T08:12:28.802 INFO:teuthology.orchestra.run.smithi094.stdout:Building dependency tree... 2023-10-09T08:12:28.802 INFO:teuthology.orchestra.run.smithi094.stdout:Reading state information... 2023-10-09T08:12:29.016 INFO:teuthology.orchestra.run.smithi094.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T08:12:29.017 INFO:teuthology.orchestra.run.smithi094.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:12:29.019 DEBUG:teuthology.orchestra.run.smithi094:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:12:29.039 INFO:teuthology.orchestra.run.smithi094.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:12:29.039 INFO:teuthology.orchestra.run.smithi094.stdout:Status: install ok installed 2023-10-09T08:12:29.039 INFO:teuthology.orchestra.run.smithi094.stdout:Priority: optional 2023-10-09T08:12:29.039 INFO:teuthology.orchestra.run.smithi094.stdout:Section: kernel 2023-10-09T08:12:29.039 INFO:teuthology.orchestra.run.smithi094.stdout:Installed-Size: 20 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Architecture: amd64 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Recommends: thermald 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout:Description: Generic Linux kernel image 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:12:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: available. 2023-10-09T08:12:29.041 DEBUG:teuthology.orchestra.run.smithi094:> mktemp 2023-10-09T08:12:29.047 INFO:teuthology.orchestra.run.smithi094.stdout:/tmp/tmp.sXHDn7jI3P 2023-10-09T08:12:29.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo cp /boot/grub/grub.cfg /tmp/tmp.sXHDn7jI3P 2023-10-09T08:12:29.060 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod 0666 /tmp/tmp.sXHDn7jI3P 2023-10-09T08:12:29.154 DEBUG:teuthology.orchestra.remote:smithi094:/tmp/tmp.sXHDn7jI3P is 9KB 2023-10-09T08:12:29.173 DEBUG:teuthology.orchestra.run.smithi094:> rm -fr /tmp/tmp.sXHDn7jI3P 2023-10-09T08:12:29.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T08:12:29.267 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:12:29.268 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T08:12:29.268 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T08:12:29.303 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T08:12:29.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo update-grub 2023-10-09T08:12:30.167 INFO:teuthology.orchestra.run.smithi016.stderr:Sourcing file `/etc/default/grub' 2023-10-09T08:12:30.173 INFO:teuthology.orchestra.run.smithi016.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:12:30.184 INFO:teuthology.orchestra.run.smithi016.stderr:Generating grub configuration file ... 2023-10-09T08:12:30.493 INFO:teuthology.orchestra.run.smithi016.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:12:30.505 INFO:teuthology.orchestra.run.smithi016.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:12:30.619 INFO:teuthology.orchestra.run.smithi094.stderr:Sourcing file `/etc/default/grub' 2023-10-09T08:12:30.630 INFO:teuthology.orchestra.run.smithi094.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:12:30.656 INFO:teuthology.orchestra.run.smithi094.stderr:Generating grub configuration file ... 2023-10-09T08:12:30.890 INFO:teuthology.orchestra.run.smithi016.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:12:30.899 INFO:teuthology.orchestra.run.smithi094.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:12:30.902 INFO:teuthology.orchestra.run.smithi016.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:12:30.911 INFO:teuthology.orchestra.run.smithi094.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:12:31.260 INFO:teuthology.orchestra.run.smithi094.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:12:31.271 INFO:teuthology.orchestra.run.smithi094.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:12:31.895 INFO:teuthology.orchestra.run.smithi016.stderr:done 2023-10-09T08:12:31.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo shutdown -r now 2023-10-09T08:12:32.226 INFO:teuthology.orchestra.run.smithi094.stderr:done 2023-10-09T08:12:32.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo shutdown -r now 2023-10-09T08:13:01.930 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-10-09T08:13:01.931 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-10-09T08:13:01.931 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:13:02.230 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-10-09T08:13:02.230 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi094.front.sepia.ceph.com' 2023-10-09T08:13:02.231 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:14:01.993 DEBUG:teuthology.orchestra.remote:timed out 2023-10-09T08:14:02.233 DEBUG:teuthology.orchestra.remote:timed out 2023-10-09T08:14:11.002 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-10-09T08:14:11.003 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:14:11.234 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi094.front.sepia.ceph.com' 2023-10-09T08:14:11.235 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:14:12.254 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.94 2023-10-09T08:14:18.874 DEBUG:teuthology.orchestra.run.smithi016:> true 2023-10-09T08:14:19.825 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-10-09T08:14:19.826 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-86-generic"... 2023-10-09T08:14:19.826 DEBUG:teuthology.orchestra.run.smithi016:> uname -r 2023-10-09T08:14:19.831 INFO:teuthology.orchestra.run.smithi016.stdout:5.15.0-86-generic 2023-10-09T08:14:19.831 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T08:14:19.832 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T08:14:19.832 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T08:14:20.834 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T08:14:20.834 DEBUG:teuthology.orchestra.run.smithi016:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T08:14:20.968 INFO:teuthology.orchestra.run.smithi016.stdout:ttyS1 2023-10-09T08:14:20.992 DEBUG:teuthology.parallel:result is None 2023-10-09T08:14:24.255 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi094.front.sepia.ceph.com' 2023-10-09T08:14:24.256 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:14:24.559 DEBUG:teuthology.orchestra.run.smithi094:> true 2023-10-09T08:14:25.094 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi094.front.sepia.ceph.com' 2023-10-09T08:14:25.095 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-86-generic"... 2023-10-09T08:14:25.095 DEBUG:teuthology.orchestra.run.smithi094:> uname -r 2023-10-09T08:14:25.143 INFO:teuthology.orchestra.run.smithi094.stdout:5.15.0-86-generic 2023-10-09T08:14:25.144 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T08:14:25.144 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T08:14:25.144 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T08:14:26.145 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T08:14:26.145 DEBUG:teuthology.orchestra.run.smithi094:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T08:14:26.233 INFO:teuthology.orchestra.run.smithi094.stdout:ttyS1 2023-10-09T08:14:26.256 DEBUG:teuthology.parallel:result is None 2023-10-09T08:14:26.256 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T08:14:26.263 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T08:14:26.264 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T08:14:26.266 DEBUG:teuthology.orchestra.run.smithi094:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T08:14:26.273 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T08:14:26.328 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T08:14:26.378 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T08:14:26.379 DEBUG:teuthology.orchestra.run.smithi016:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T08:14:26.381 DEBUG:teuthology.orchestra.run.smithi094:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T08:14:26.400 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T08:14:26.497 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T08:14:26.497 DEBUG:teuthology.orchestra.run.smithi016:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T08:14:26.500 DEBUG:teuthology.orchestra.run.smithi094:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T08:14:26.517 INFO:teuthology.orchestra.run.smithi094.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:14:26.523 INFO:teuthology.orchestra.run.smithi094.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:14:26.526 INFO:teuthology.orchestra.run.smithi016.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:14:26.536 INFO:teuthology.orchestra.run.smithi016.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:14:26.538 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T08:14:26.547 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T08:14:26.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T08:14:26.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T08:14:26.601 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T08:14:26.610 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T08:14:26.610 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T08:14:26.644 DEBUG:teuthology.orchestra.run.smithi094:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T08:14:26.652 DEBUG:teuthology.orchestra.run.smithi016:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T08:14:26.699 DEBUG:teuthology.orchestra.run.smithi016:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T08:14:26.749 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:14:26.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T08:14:26.810 DEBUG:teuthology.orchestra.run.smithi094:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T08:14:26.817 DEBUG:teuthology.orchestra.run.smithi094:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T08:14:26.868 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:14:26.868 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T08:14:26.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo service rsyslog restart 2023-10-09T08:14:26.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo service rsyslog restart 2023-10-09T08:14:27.025 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T08:14:27.033 INFO:teuthology.task.internal:Starting timer... 2023-10-09T08:14:27.033 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T08:14:27.073 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T08:14:27.122 DEBUG:teuthology.task.selinux:Excluding smithi016: OS 'ubuntu' does not support SELinux 2023-10-09T08:14:27.122 DEBUG:teuthology.task.selinux:Excluding smithi094: OS 'ubuntu' does not support SELinux 2023-10-09T08:14:27.123 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T08:14:27.123 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-10-09T08:14:27.123 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T08:14:27.123 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T08:14:27.173 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-10-09T08:14:27.173 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T08:14:27.191 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T08:14:27.192 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi016.front.sepia.ceph.com,smithi094.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T08:20:18.174 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi016.front.sepia.ceph.com'), Remote(name='ubuntu@smithi094.front.sepia.ceph.com')] 2023-10-09T08:20:18.176 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-10-09T08:20:18.177 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:20:18.269 DEBUG:teuthology.orchestra.run.smithi016:> true 2023-10-09T08:20:18.392 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-10-09T08:20:18.392 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi094.front.sepia.ceph.com' 2023-10-09T08:20:18.393 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:20:18.472 DEBUG:teuthology.orchestra.run.smithi094:> true 2023-10-09T08:20:18.592 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi094.front.sepia.ceph.com' 2023-10-09T08:20:18.592 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T08:20:18.603 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T08:20:18.603 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T08:20:18.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T08:20:18.607 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T08:20:18.607 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T08:20:18.640 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T08:20:18.640 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Command line: ntpd -gq 2023-10-09T08:20:18.641 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: proto: precision = 0.095 usec (-23) 2023-10-09T08:20:18.642 INFO:teuthology.orchestra.run.smithi016.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:20:18.642 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:20:18.642 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:20:18.643 INFO:teuthology.orchestra.run.smithi016.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:20:18.644 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T08:20:18.644 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T08:20:18.644 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T08:20:18.644 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Listen normally on 3 ens1f0 172.21.15.16:123 2023-10-09T08:20:18.644 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Listen normally on 4 lo [::1]:123 2023-10-09T08:20:18.644 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Listen normally on 5 ens1f0 [fe80::ec4:7aff:febd:1564%4]:123 2023-10-09T08:20:18.644 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:18 ntpd[22628]: Listening on routing socket on fd #22 for interface updates 2023-10-09T08:20:18.665 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T08:20:18.666 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Command line: ntpd -gq 2023-10-09T08:20:18.666 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: proto: precision = 0.047 usec (-24) 2023-10-09T08:20:18.666 INFO:teuthology.orchestra.run.smithi094.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:20:18.667 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:20:18.667 INFO:teuthology.orchestra.run.smithi094.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:20:18.668 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:20:18.668 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T08:20:18.668 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T08:20:18.668 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T08:20:18.668 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Listen normally on 3 enp3s0f1 172.21.15.94:123 2023-10-09T08:20:18.668 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Listen normally on 4 lo [::1]:123 2023-10-09T08:20:18.669 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d763%5]:123 2023-10-09T08:20:18.669 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:18 ntpd[22701]: Listening on routing socket on fd #22 for interface updates 2023-10-09T08:20:19.641 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:19 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:19.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:19 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:19.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:19 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:19.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:19 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:19.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:19 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:19.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:19 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:20.641 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:20 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:20.641 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:20 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:20.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:20 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:20.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:20 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:20.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:20 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:20.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:20 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:20.665 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:20 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:20.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:20 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:21.641 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:21 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:21.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:21 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:21.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:21 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:21.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:21 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:21.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:21 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:21.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:21 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:21.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:21 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:21.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:21 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:21.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:21 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:21.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:21 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:21.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:21 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:21.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:21 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:22.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:22.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:22.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:22 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:22.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:22.668 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:22 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:23.641 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:23.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:23.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:23.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:23.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:23.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:23.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:23.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:23 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:23.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:23 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:23.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:23 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:23.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:23 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:23.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:23 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:23.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:23 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:23.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:23 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:24.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:24.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:24.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:24.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:24 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:24.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:24.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:24 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:25.641 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:25.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:25.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:25.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:25.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:25.642 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:25.642 INFO:teuthology.orchestra.run.smithi016.stdout: 9 Oct 08:20:25 ntpd[22628]: ntpd: time slew +0.001287 s 2023-10-09T08:20:25.642 INFO:teuthology.orchestra.run.smithi016.stdout:ntpd: time slew +0.001287s 2023-10-09T08:20:25.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T08:20:25.643 INFO:teuthology.orchestra.run.smithi016.stderr: 9 Oct 08:20:25 ntpd[22628]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T08:20:25.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:25 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:25.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:25 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:25.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:25 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:25.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:25 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:25.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:25 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:25.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:25 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:25.700 INFO:teuthology.orchestra.run.smithi016.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T08:20:25.700 INFO:teuthology.orchestra.run.smithi016.stdout:============================================================================== 2023-10-09T08:20:25.700 INFO:teuthology.orchestra.run.smithi016.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:25.700 INFO:teuthology.orchestra.run.smithi016.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:25.700 INFO:teuthology.orchestra.run.smithi016.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:25.700 INFO:teuthology.orchestra.run.smithi016.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:26.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:26.668 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:26.668 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:26.668 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:26.668 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:26 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:27.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:27.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:27.666 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:20:27.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:20:27.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:20:27.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:20:27.667 INFO:teuthology.orchestra.run.smithi094.stdout: 9 Oct 08:20:27 ntpd[22701]: ntpd: time slew -0.000010 s 2023-10-09T08:20:27.667 INFO:teuthology.orchestra.run.smithi094.stdout:ntpd: time slew -0.000010s 2023-10-09T08:20:27.667 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T08:20:27.668 INFO:teuthology.orchestra.run.smithi094.stderr: 9 Oct 08:20:27 ntpd[22701]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T08:20:27.728 INFO:teuthology.orchestra.run.smithi094.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T08:20:27.728 INFO:teuthology.orchestra.run.smithi094.stdout:============================================================================== 2023-10-09T08:20:27.728 INFO:teuthology.orchestra.run.smithi094.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:27.728 INFO:teuthology.orchestra.run.smithi094.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:27.728 INFO:teuthology.orchestra.run.smithi094.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:27.728 INFO:teuthology.orchestra.run.smithi094.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:20:27.729 INFO:teuthology.run_tasks:Running task install... 2023-10-09T08:20:27.738 DEBUG:teuthology.task.install:project ceph 2023-10-09T08:20:27.739 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T08:20:27.739 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T08:20:27.739 INFO:teuthology.task.install:Using flavor: default 2023-10-09T08:20:27.744 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T08:20:27.745 INFO:teuthology.task.install:extra packages: [] 2023-10-09T08:20:27.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-key list | grep Ceph 2023-10-09T08:20:27.746 DEBUG:teuthology.orchestra.run.smithi094:> sudo apt-key list | grep Ceph 2023-10-09T08:20:27.846 INFO:teuthology.orchestra.run.smithi016.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T08:20:27.869 INFO:teuthology.orchestra.run.smithi094.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T08:20:27.907 INFO:teuthology.orchestra.run.smithi016.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T08:20:27.907 INFO:teuthology.orchestra.run.smithi016.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T08:20:27.908 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T08:20:27.908 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:20:27.908 INFO:teuthology.packaging:ref: None 2023-10-09T08:20:27.908 INFO:teuthology.packaging:tag: None 2023-10-09T08:20:27.908 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:20:27.908 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:20:27.909 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:20:27.926 INFO:teuthology.orchestra.run.smithi094.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T08:20:27.927 INFO:teuthology.orchestra.run.smithi094.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T08:20:27.927 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T08:20:27.928 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:20:27.928 INFO:teuthology.packaging:ref: None 2023-10-09T08:20:27.928 INFO:teuthology.packaging:tag: None 2023-10-09T08:20:27.928 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:20:27.928 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:20:27.928 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:20:28.149 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T08:20:28.150 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:20:28.155 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T08:20:28.156 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:20:28.320 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:20:28.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T08:20:28.325 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:20:28.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T08:20:28.336 DEBUG:teuthology.orchestra.run.smithi094:> sudo apt-get update 2023-10-09T08:20:28.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get update 2023-10-09T08:20:28.529 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:20:28.543 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T08:20:28.563 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T08:20:28.590 INFO:teuthology.orchestra.run.smithi016.stdout:Ign:1 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T08:20:28.612 INFO:teuthology.orchestra.run.smithi016.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T08:20:28.635 INFO:teuthology.orchestra.run.smithi016.stdout:Ign:3 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T08:20:28.639 INFO:teuthology.orchestra.run.smithi094.stdout:Ign:4 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T08:20:28.664 INFO:teuthology.orchestra.run.smithi094.stdout:Hit:5 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T08:20:28.665 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:20:28.665 INFO:teuthology.orchestra.run.smithi016.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T08:20:28.665 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T08:20:28.675 INFO:teuthology.orchestra.run.smithi094.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T08:20:28.698 INFO:teuthology.orchestra.run.smithi016.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T08:20:28.714 INFO:teuthology.orchestra.run.smithi094.stdout:Ign:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T08:20:28.734 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T08:20:28.750 INFO:teuthology.orchestra.run.smithi094.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T08:20:28.802 INFO:teuthology.orchestra.run.smithi094.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T08:20:29.083 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T08:20:29.157 INFO:teuthology.orchestra.run.smithi094.stdout:Fetched 28.0 kB in 1s (46.0 kB/s) 2023-10-09T08:20:29.295 INFO:teuthology.orchestra.run.smithi016.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T08:20:29.629 INFO:teuthology.orchestra.run.smithi016.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T08:20:29.803 INFO:teuthology.orchestra.run.smithi016.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T08:20:29.834 INFO:teuthology.orchestra.run.smithi016.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T08:20:30.210 INFO:teuthology.orchestra.run.smithi094.stdout:Reading package lists... 2023-10-09T08:20:30.237 DEBUG:teuthology.orchestra.run.smithi094:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T08:20:30.312 INFO:teuthology.orchestra.run.smithi094.stdout:Reading package lists... 2023-10-09T08:20:30.421 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 6,017 kB in 2s (3,222 kB/s) 2023-10-09T08:20:30.514 INFO:teuthology.orchestra.run.smithi094.stdout:Building dependency tree... 2023-10-09T08:20:30.515 INFO:teuthology.orchestra.run.smithi094.stdout:Reading state information... 2023-10-09T08:20:30.687 INFO:teuthology.orchestra.run.smithi094.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T08:20:30.689 INFO:teuthology.orchestra.run.smithi094.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T08:20:30.689 INFO:teuthology.orchestra.run.smithi094.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T08:20:30.690 INFO:teuthology.orchestra.run.smithi094.stdout:The following additional packages will be installed: 2023-10-09T08:20:30.690 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T08:20:30.690 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T08:20:30.690 INFO:teuthology.orchestra.run.smithi094.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T08:20:30.690 INFO:teuthology.orchestra.run.smithi094.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T08:20:30.690 INFO:teuthology.orchestra.run.smithi094.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T08:20:30.691 INFO:teuthology.orchestra.run.smithi094.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T08:20:30.691 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T08:20:30.691 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T08:20:30.691 INFO:teuthology.orchestra.run.smithi094.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T08:20:30.691 INFO:teuthology.orchestra.run.smithi094.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T08:20:30.691 INFO:teuthology.orchestra.run.smithi094.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T08:20:30.691 INFO:teuthology.orchestra.run.smithi094.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T08:20:30.692 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T08:20:30.692 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T08:20:30.692 INFO:teuthology.orchestra.run.smithi094.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T08:20:30.692 INFO:teuthology.orchestra.run.smithi094.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T08:20:30.692 INFO:teuthology.orchestra.run.smithi094.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T08:20:30.692 INFO:teuthology.orchestra.run.smithi094.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T08:20:30.692 INFO:teuthology.orchestra.run.smithi094.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T08:20:30.693 INFO:teuthology.orchestra.run.smithi094.stdout:Suggested packages: 2023-10-09T08:20:30.693 INFO:teuthology.orchestra.run.smithi094.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T08:20:30.693 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T08:20:30.693 INFO:teuthology.orchestra.run.smithi094.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T08:20:30.693 INFO:teuthology.orchestra.run.smithi094.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T08:20:30.694 INFO:teuthology.orchestra.run.smithi094.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T08:20:30.694 INFO:teuthology.orchestra.run.smithi094.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T08:20:30.694 INFO:teuthology.orchestra.run.smithi094.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T08:20:30.694 INFO:teuthology.orchestra.run.smithi094.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T08:20:30.694 INFO:teuthology.orchestra.run.smithi094.stdout:Recommended packages: 2023-10-09T08:20:30.694 INFO:teuthology.orchestra.run.smithi094.stdout: btrfs-tools 2023-10-09T08:20:30.799 INFO:teuthology.orchestra.run.smithi094.stdout:The following NEW packages will be installed: 2023-10-09T08:20:30.799 INFO:teuthology.orchestra.run.smithi094.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T08:20:30.799 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T08:20:30.799 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T08:20:30.799 INFO:teuthology.orchestra.run.smithi094.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T08:20:30.800 INFO:teuthology.orchestra.run.smithi094.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T08:20:30.800 INFO:teuthology.orchestra.run.smithi094.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T08:20:30.801 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T08:20:30.802 INFO:teuthology.orchestra.run.smithi094.stdout:The following packages will be upgraded: 2023-10-09T08:20:30.803 INFO:teuthology.orchestra.run.smithi094.stdout: librados2 librbd1 2023-10-09T08:20:30.869 INFO:teuthology.orchestra.run.smithi094.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T08:20:30.869 INFO:teuthology.orchestra.run.smithi094.stdout:Need to get 174 MB of archives. 2023-10-09T08:20:30.870 INFO:teuthology.orchestra.run.smithi094.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T08:20:30.870 INFO:teuthology.orchestra.run.smithi094.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T08:20:30.964 INFO:teuthology.orchestra.run.smithi094.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T08:20:30.965 INFO:teuthology.orchestra.run.smithi094.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T08:20:30.976 INFO:teuthology.orchestra.run.smithi094.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T08:20:30.983 INFO:teuthology.orchestra.run.smithi094.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T08:20:30.990 INFO:teuthology.orchestra.run.smithi094.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T08:20:31.011 INFO:teuthology.orchestra.run.smithi094.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T08:20:31.016 INFO:teuthology.orchestra.run.smithi094.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T08:20:31.017 INFO:teuthology.orchestra.run.smithi094.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T08:20:31.025 INFO:teuthology.orchestra.run.smithi094.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T08:20:31.026 INFO:teuthology.orchestra.run.smithi094.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T08:20:31.027 INFO:teuthology.orchestra.run.smithi094.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T08:20:31.031 INFO:teuthology.orchestra.run.smithi094.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T08:20:31.032 INFO:teuthology.orchestra.run.smithi094.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T08:20:31.032 INFO:teuthology.orchestra.run.smithi094.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T08:20:31.046 INFO:teuthology.orchestra.run.smithi094.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T08:20:31.048 INFO:teuthology.orchestra.run.smithi094.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T08:20:31.048 INFO:teuthology.orchestra.run.smithi094.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T08:20:31.048 INFO:teuthology.orchestra.run.smithi094.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T08:20:31.063 INFO:teuthology.orchestra.run.smithi094.stdout:Get:20 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T08:20:31.065 INFO:teuthology.orchestra.run.smithi094.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T08:20:31.066 INFO:teuthology.orchestra.run.smithi094.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T08:20:31.068 INFO:teuthology.orchestra.run.smithi094.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T08:20:31.071 INFO:teuthology.orchestra.run.smithi094.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T08:20:31.071 INFO:teuthology.orchestra.run.smithi094.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T08:20:31.076 INFO:teuthology.orchestra.run.smithi094.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T08:20:31.084 INFO:teuthology.orchestra.run.smithi094.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T08:20:31.085 INFO:teuthology.orchestra.run.smithi094.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T08:20:31.086 INFO:teuthology.orchestra.run.smithi094.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T08:20:31.088 INFO:teuthology.orchestra.run.smithi094.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T08:20:31.104 INFO:teuthology.orchestra.run.smithi094.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T08:20:31.106 INFO:teuthology.orchestra.run.smithi094.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T08:20:31.107 INFO:teuthology.orchestra.run.smithi094.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T08:20:31.110 INFO:teuthology.orchestra.run.smithi094.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T08:20:31.112 INFO:teuthology.orchestra.run.smithi094.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T08:20:31.113 INFO:teuthology.orchestra.run.smithi094.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T08:20:31.126 INFO:teuthology.orchestra.run.smithi094.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T08:20:31.126 INFO:teuthology.orchestra.run.smithi094.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T08:20:31.270 INFO:teuthology.orchestra.run.smithi094.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T08:20:31.285 INFO:teuthology.orchestra.run.smithi094.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T08:20:31.286 INFO:teuthology.orchestra.run.smithi094.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T08:20:31.305 INFO:teuthology.orchestra.run.smithi094.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T08:20:31.305 INFO:teuthology.orchestra.run.smithi094.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T08:20:31.306 INFO:teuthology.orchestra.run.smithi094.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T08:20:31.306 INFO:teuthology.orchestra.run.smithi094.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T08:20:31.307 INFO:teuthology.orchestra.run.smithi094.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T08:20:31.307 INFO:teuthology.orchestra.run.smithi094.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T08:20:31.308 INFO:teuthology.orchestra.run.smithi094.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T08:20:31.313 INFO:teuthology.orchestra.run.smithi094.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T08:20:31.314 INFO:teuthology.orchestra.run.smithi094.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T08:20:31.333 INFO:teuthology.orchestra.run.smithi094.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T08:20:31.340 INFO:teuthology.orchestra.run.smithi094.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T08:20:31.344 INFO:teuthology.orchestra.run.smithi094.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T08:20:31.346 INFO:teuthology.orchestra.run.smithi094.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T08:20:31.346 INFO:teuthology.orchestra.run.smithi094.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T08:20:31.349 INFO:teuthology.orchestra.run.smithi094.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T08:20:31.351 INFO:teuthology.orchestra.run.smithi094.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T08:20:31.354 INFO:teuthology.orchestra.run.smithi094.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T08:20:31.354 INFO:teuthology.orchestra.run.smithi094.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T08:20:31.354 INFO:teuthology.orchestra.run.smithi094.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T08:20:31.372 INFO:teuthology.orchestra.run.smithi094.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T08:20:31.374 INFO:teuthology.orchestra.run.smithi094.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T08:20:31.390 INFO:teuthology.orchestra.run.smithi094.stdout:Get:63 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T08:20:31.393 INFO:teuthology.orchestra.run.smithi094.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T08:20:31.393 INFO:teuthology.orchestra.run.smithi094.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T08:20:31.393 INFO:teuthology.orchestra.run.smithi094.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T08:20:31.394 INFO:teuthology.orchestra.run.smithi094.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T08:20:31.395 INFO:teuthology.orchestra.run.smithi094.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T08:20:31.396 INFO:teuthology.orchestra.run.smithi094.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T08:20:31.396 INFO:teuthology.orchestra.run.smithi094.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T08:20:31.396 INFO:teuthology.orchestra.run.smithi094.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T08:20:31.413 INFO:teuthology.orchestra.run.smithi094.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T08:20:31.506 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-10-09T08:20:31.530 INFO:teuthology.orchestra.run.smithi094.stdout:Get:73 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T08:20:31.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T08:20:31.548 INFO:teuthology.orchestra.run.smithi094.stdout:Get:74 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T08:20:31.569 INFO:teuthology.orchestra.run.smithi094.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T08:20:31.570 INFO:teuthology.orchestra.run.smithi094.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T08:20:31.573 INFO:teuthology.orchestra.run.smithi094.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T08:20:31.574 INFO:teuthology.orchestra.run.smithi094.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T08:20:31.586 INFO:teuthology.orchestra.run.smithi094.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T08:20:31.604 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-10-09T08:20:31.710 INFO:teuthology.orchestra.run.smithi094.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T08:20:31.712 INFO:teuthology.orchestra.run.smithi094.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T08:20:31.723 INFO:teuthology.orchestra.run.smithi094.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T08:20:31.809 INFO:teuthology.orchestra.run.smithi016.stdout:Building dependency tree... 2023-10-09T08:20:31.810 INFO:teuthology.orchestra.run.smithi016.stdout:Reading state information... 2023-10-09T08:20:31.993 INFO:teuthology.orchestra.run.smithi016.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T08:20:31.995 INFO:teuthology.orchestra.run.smithi016.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T08:20:31.995 INFO:teuthology.orchestra.run.smithi016.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T08:20:31.995 INFO:teuthology.orchestra.run.smithi016.stdout:The following additional packages will be installed: 2023-10-09T08:20:31.995 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T08:20:31.995 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T08:20:31.995 INFO:teuthology.orchestra.run.smithi016.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T08:20:31.995 INFO:teuthology.orchestra.run.smithi016.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T08:20:31.996 INFO:teuthology.orchestra.run.smithi016.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T08:20:31.996 INFO:teuthology.orchestra.run.smithi016.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T08:20:31.996 INFO:teuthology.orchestra.run.smithi016.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T08:20:31.996 INFO:teuthology.orchestra.run.smithi016.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T08:20:31.996 INFO:teuthology.orchestra.run.smithi016.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T08:20:31.996 INFO:teuthology.orchestra.run.smithi016.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T08:20:31.996 INFO:teuthology.orchestra.run.smithi016.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T08:20:31.997 INFO:teuthology.orchestra.run.smithi016.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T08:20:31.997 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T08:20:31.997 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T08:20:31.997 INFO:teuthology.orchestra.run.smithi016.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T08:20:31.997 INFO:teuthology.orchestra.run.smithi016.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T08:20:31.997 INFO:teuthology.orchestra.run.smithi016.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T08:20:31.998 INFO:teuthology.orchestra.run.smithi016.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T08:20:31.998 INFO:teuthology.orchestra.run.smithi016.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T08:20:31.998 INFO:teuthology.orchestra.run.smithi016.stdout:Suggested packages: 2023-10-09T08:20:31.998 INFO:teuthology.orchestra.run.smithi016.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout:Recommended packages: 2023-10-09T08:20:31.999 INFO:teuthology.orchestra.run.smithi016.stdout: btrfs-tools 2023-10-09T08:20:32.110 INFO:teuthology.orchestra.run.smithi016.stdout:The following NEW packages will be installed: 2023-10-09T08:20:32.110 INFO:teuthology.orchestra.run.smithi016.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T08:20:32.110 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T08:20:32.110 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T08:20:32.110 INFO:teuthology.orchestra.run.smithi016.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T08:20:32.110 INFO:teuthology.orchestra.run.smithi016.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T08:20:32.111 INFO:teuthology.orchestra.run.smithi016.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T08:20:32.111 INFO:teuthology.orchestra.run.smithi016.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T08:20:32.111 INFO:teuthology.orchestra.run.smithi016.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T08:20:32.112 INFO:teuthology.orchestra.run.smithi016.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T08:20:32.113 INFO:teuthology.orchestra.run.smithi016.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T08:20:32.113 INFO:teuthology.orchestra.run.smithi016.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T08:20:32.113 INFO:teuthology.orchestra.run.smithi016.stdout:The following packages will be upgraded: 2023-10-09T08:20:32.113 INFO:teuthology.orchestra.run.smithi016.stdout: librados2 librbd1 2023-10-09T08:20:32.176 INFO:teuthology.orchestra.run.smithi016.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T08:20:32.176 INFO:teuthology.orchestra.run.smithi016.stdout:Need to get 174 MB of archives. 2023-10-09T08:20:32.176 INFO:teuthology.orchestra.run.smithi016.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T08:20:32.177 INFO:teuthology.orchestra.run.smithi016.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T08:20:32.258 INFO:teuthology.orchestra.run.smithi016.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T08:20:32.259 INFO:teuthology.orchestra.run.smithi016.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T08:20:32.268 INFO:teuthology.orchestra.run.smithi016.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T08:20:32.274 INFO:teuthology.orchestra.run.smithi016.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T08:20:32.298 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T08:20:32.301 INFO:teuthology.orchestra.run.smithi016.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T08:20:32.303 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T08:20:32.309 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T08:20:32.311 INFO:teuthology.orchestra.run.smithi016.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T08:20:32.312 INFO:teuthology.orchestra.run.smithi016.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T08:20:32.315 INFO:teuthology.orchestra.run.smithi016.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T08:20:32.315 INFO:teuthology.orchestra.run.smithi016.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T08:20:32.316 INFO:teuthology.orchestra.run.smithi016.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T08:20:32.330 INFO:teuthology.orchestra.run.smithi016.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T08:20:32.332 INFO:teuthology.orchestra.run.smithi016.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T08:20:32.332 INFO:teuthology.orchestra.run.smithi016.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T08:20:32.333 INFO:teuthology.orchestra.run.smithi016.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T08:20:32.347 INFO:teuthology.orchestra.run.smithi016.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T08:20:32.348 INFO:teuthology.orchestra.run.smithi016.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T08:20:32.349 INFO:teuthology.orchestra.run.smithi016.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T08:20:32.350 INFO:teuthology.orchestra.run.smithi016.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T08:20:32.351 INFO:teuthology.orchestra.run.smithi016.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T08:20:32.356 INFO:teuthology.orchestra.run.smithi016.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T08:20:32.358 INFO:teuthology.orchestra.run.smithi016.stdout:Get:25 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T08:20:32.364 INFO:teuthology.orchestra.run.smithi016.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T08:20:32.364 INFO:teuthology.orchestra.run.smithi016.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T08:20:32.365 INFO:teuthology.orchestra.run.smithi016.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T08:20:32.366 INFO:teuthology.orchestra.run.smithi016.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T08:20:32.381 INFO:teuthology.orchestra.run.smithi016.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T08:20:32.383 INFO:teuthology.orchestra.run.smithi016.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T08:20:32.384 INFO:teuthology.orchestra.run.smithi016.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T08:20:32.386 INFO:teuthology.orchestra.run.smithi016.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T08:20:32.389 INFO:teuthology.orchestra.run.smithi016.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T08:20:32.391 INFO:teuthology.orchestra.run.smithi016.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T08:20:32.397 INFO:teuthology.orchestra.run.smithi016.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T08:20:32.398 INFO:teuthology.orchestra.run.smithi016.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T08:20:32.481 INFO:teuthology.orchestra.run.smithi016.stdout:Get:38 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T08:20:32.489 INFO:teuthology.orchestra.run.smithi016.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T08:20:32.503 INFO:teuthology.orchestra.run.smithi016.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T08:20:32.505 INFO:teuthology.orchestra.run.smithi016.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T08:20:32.517 INFO:teuthology.orchestra.run.smithi016.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T08:20:32.517 INFO:teuthology.orchestra.run.smithi016.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T08:20:32.518 INFO:teuthology.orchestra.run.smithi016.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T08:20:32.518 INFO:teuthology.orchestra.run.smithi016.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T08:20:32.518 INFO:teuthology.orchestra.run.smithi016.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T08:20:32.569 INFO:teuthology.orchestra.run.smithi094.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T08:20:32.602 INFO:teuthology.orchestra.run.smithi016.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T08:20:32.602 INFO:teuthology.orchestra.run.smithi016.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T08:20:32.602 INFO:teuthology.orchestra.run.smithi016.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T08:20:32.603 INFO:teuthology.orchestra.run.smithi016.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T08:20:32.603 INFO:teuthology.orchestra.run.smithi016.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T08:20:32.605 INFO:teuthology.orchestra.run.smithi016.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T08:20:32.606 INFO:teuthology.orchestra.run.smithi016.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T08:20:32.618 INFO:teuthology.orchestra.run.smithi016.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T08:20:32.619 INFO:teuthology.orchestra.run.smithi016.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T08:20:32.636 INFO:teuthology.orchestra.run.smithi016.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T08:20:32.641 INFO:teuthology.orchestra.run.smithi016.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T08:20:32.646 INFO:teuthology.orchestra.run.smithi016.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T08:20:32.646 INFO:teuthology.orchestra.run.smithi016.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T08:20:32.646 INFO:teuthology.orchestra.run.smithi016.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T08:20:32.647 INFO:teuthology.orchestra.run.smithi016.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T08:20:32.648 INFO:teuthology.orchestra.run.smithi016.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T08:20:32.661 INFO:teuthology.orchestra.run.smithi016.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T08:20:32.661 INFO:teuthology.orchestra.run.smithi016.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T08:20:32.662 INFO:teuthology.orchestra.run.smithi016.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T08:20:32.669 INFO:teuthology.orchestra.run.smithi016.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T08:20:32.670 INFO:teuthology.orchestra.run.smithi016.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T08:20:32.671 INFO:teuthology.orchestra.run.smithi016.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T08:20:32.671 INFO:teuthology.orchestra.run.smithi016.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T08:20:32.672 INFO:teuthology.orchestra.run.smithi016.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T08:20:32.673 INFO:teuthology.orchestra.run.smithi016.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T08:20:32.677 INFO:teuthology.orchestra.run.smithi094.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T08:20:32.681 INFO:teuthology.orchestra.run.smithi094.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T08:20:32.687 INFO:teuthology.orchestra.run.smithi094.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T08:20:32.808 INFO:teuthology.orchestra.run.smithi094.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T08:20:33.029 INFO:teuthology.orchestra.run.smithi094.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T08:20:33.251 INFO:teuthology.orchestra.run.smithi016.stdout:Get:72 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T08:20:33.872 INFO:teuthology.orchestra.run.smithi016.stdout:Get:73 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T08:20:33.994 INFO:teuthology.orchestra.run.smithi016.stdout:Get:74 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T08:20:34.088 INFO:teuthology.orchestra.run.smithi016.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T08:20:34.091 INFO:teuthology.orchestra.run.smithi016.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T08:20:34.134 INFO:teuthology.orchestra.run.smithi016.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T08:20:34.135 INFO:teuthology.orchestra.run.smithi016.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T08:20:34.158 INFO:teuthology.orchestra.run.smithi094.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T08:20:34.159 INFO:teuthology.orchestra.run.smithi094.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T08:20:34.165 INFO:teuthology.orchestra.run.smithi094.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T08:20:34.194 INFO:teuthology.orchestra.run.smithi016.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T08:20:34.197 INFO:teuthology.orchestra.run.smithi094.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T08:20:34.199 INFO:teuthology.orchestra.run.smithi094.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T08:20:34.203 INFO:teuthology.orchestra.run.smithi094.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T08:20:34.321 INFO:teuthology.orchestra.run.smithi094.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T08:20:34.626 INFO:teuthology.orchestra.run.smithi094.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T08:20:34.626 INFO:teuthology.orchestra.run.smithi094.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T08:20:34.947 INFO:teuthology.orchestra.run.smithi016.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T08:20:34.968 INFO:teuthology.orchestra.run.smithi016.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T08:20:35.066 INFO:teuthology.orchestra.run.smithi016.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T08:20:36.886 INFO:teuthology.orchestra.run.smithi094.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T08:20:36.886 INFO:teuthology.orchestra.run.smithi094.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T08:20:37.261 INFO:teuthology.orchestra.run.smithi094.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T08:20:37.772 INFO:teuthology.orchestra.run.smithi016.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T08:20:37.902 INFO:teuthology.orchestra.run.smithi094.stdout:Fetched 174 MB in 6s (26.9 MB/s) 2023-10-09T08:20:38.010 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libjaeger. 2023-10-09T08:20:38.058 INFO:teuthology.orchestra.run.smithi094.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T08:20:38.061 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:38.084 INFO:teuthology.orchestra.run.smithi016.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T08:20:38.087 INFO:teuthology.orchestra.run.smithi016.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T08:20:38.100 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:38.103 INFO:teuthology.orchestra.run.smithi016.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T08:20:38.186 INFO:teuthology.orchestra.run.smithi016.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T08:20:38.361 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T08:20:38.375 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T08:20:38.462 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:20:38.584 INFO:teuthology.orchestra.run.smithi016.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T08:20:38.799 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:38.966 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:20:39.720 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:39.847 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:20:40.128 INFO:teuthology.orchestra.run.smithi016.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T08:20:40.129 INFO:teuthology.orchestra.run.smithi016.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T08:20:40.134 INFO:teuthology.orchestra.run.smithi016.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T08:20:40.257 INFO:teuthology.orchestra.run.smithi016.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T08:20:40.258 INFO:teuthology.orchestra.run.smithi016.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T08:20:40.274 INFO:teuthology.orchestra.run.smithi016.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T08:20:40.405 INFO:teuthology.orchestra.run.smithi016.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T08:20:40.577 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T08:20:40.586 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:40.636 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:40.798 INFO:teuthology.orchestra.run.smithi016.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T08:20:40.798 INFO:teuthology.orchestra.run.smithi016.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T08:20:40.994 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-rados. 2023-10-09T08:20:41.003 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:41.053 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:41.422 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T08:20:41.431 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:20:41.472 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:41.809 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T08:20:41.822 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:41.875 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:42.214 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T08:20:42.227 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:20:42.264 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:42.584 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T08:20:42.598 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T08:20:42.642 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T08:20:42.895 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T08:20:42.910 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:42.945 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:43.270 INFO:teuthology.orchestra.run.smithi016.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T08:20:43.270 INFO:teuthology.orchestra.run.smithi016.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T08:20:43.298 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T08:20:43.311 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T08:20:43.356 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:20:43.634 INFO:teuthology.orchestra.run.smithi016.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T08:20:43.710 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T08:20:43.723 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T08:20:43.760 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:20:44.072 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T08:20:44.085 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T08:20:44.113 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:20:44.302 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 174 MB in 12s (15.1 MB/s) 2023-10-09T08:20:44.419 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjaeger. 2023-10-09T08:20:44.466 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package librgw2. 2023-10-09T08:20:44.467 INFO:teuthology.orchestra.run.smithi016.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T08:20:44.471 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:44.480 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:44.517 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:44.541 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:44.853 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T08:20:44.867 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T08:20:44.954 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:20:45.135 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T08:20:45.150 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:45.194 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:45.321 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:45.458 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:20:45.514 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T08:20:45.527 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T08:20:45.563 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:20:45.908 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package liboath0. 2023-10-09T08:20:45.916 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T08:20:45.958 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T08:20:46.279 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T08:20:46.287 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:46.329 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:46.417 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:46.571 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:20:47.016 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-common. 2023-10-09T08:20:47.029 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:47.074 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:47.241 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T08:20:47.255 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:47.299 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:47.628 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rados. 2023-10-09T08:20:47.641 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:47.678 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:48.048 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T08:20:48.061 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:20:48.106 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:48.401 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T08:20:48.414 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:48.468 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:48.810 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T08:20:48.824 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:20:48.868 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:49.162 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T08:20:49.177 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T08:20:49.221 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T08:20:49.322 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-base. 2023-10-09T08:20:49.335 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:49.389 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:49.499 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T08:20:49.513 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:49.566 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:49.944 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T08:20:49.957 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T08:20:49.993 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:20:50.338 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T08:20:50.353 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T08:20:50.368 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T08:20:50.382 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T08:20:50.396 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:20:50.418 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:20:50.798 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T08:20:50.799 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T08:20:50.811 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T08:20:50.813 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T08:20:50.848 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T08:20:50.866 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:20:51.125 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T08:20:51.141 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:20:51.183 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:51.303 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package librgw2. 2023-10-09T08:20:51.317 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:51.361 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:51.539 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T08:20:51.553 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T08:20:51.597 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:20:51.900 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T08:20:51.914 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T08:20:51.947 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T08:20:51.958 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:20:51.961 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:52.005 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:52.304 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T08:20:52.317 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T08:20:52.362 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:20:52.374 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T08:20:52.389 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T08:20:52.441 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:20:52.674 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T08:20:52.687 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T08:20:52.732 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T08:20:52.852 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liboath0. 2023-10-09T08:20:52.868 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T08:20:52.919 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T08:20:53.035 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T08:20:53.048 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T08:20:53.093 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:20:53.247 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T08:20:53.261 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:53.305 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:53.380 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-mako. 2023-10-09T08:20:53.394 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T08:20:53.438 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:20:53.758 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T08:20:53.758 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-common. 2023-10-09T08:20:53.771 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T08:20:53.773 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:53.816 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:20:53.816 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:54.119 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T08:20:54.134 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T08:20:54.177 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:20:54.489 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-webob. 2023-10-09T08:20:54.502 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T08:20:54.547 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T08:20:54.858 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T08:20:54.873 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T08:20:54.916 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:20:55.219 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T08:20:55.233 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T08:20:55.278 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T08:20:55.631 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T08:20:55.646 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T08:20:55.692 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:20:56.043 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T08:20:56.056 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T08:20:56.073 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-base. 2023-10-09T08:20:56.087 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:56.101 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T08:20:56.141 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:56.412 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-paste. 2023-10-09T08:20:56.425 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T08:20:56.462 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:20:56.774 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T08:20:56.789 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T08:20:56.832 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:20:57.110 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T08:20:57.123 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T08:20:57.153 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T08:20:57.168 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T08:20:57.168 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:20:57.211 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:20:57.464 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T08:20:57.478 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T08:20:57.521 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T08:20:57.615 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T08:20:57.629 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T08:20:57.681 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T08:20:57.833 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T08:20:57.846 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T08:20:57.891 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:20:58.001 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T08:20:58.014 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:20:58.067 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:58.238 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T08:20:58.253 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T08:20:58.296 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:20:58.439 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T08:20:58.452 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T08:20:58.497 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:20:58.691 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T08:20:58.706 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:58.749 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:58.808 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T08:20:58.822 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T08:20:58.858 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:20:59.103 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T08:20:59.116 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:59.161 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:59.196 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T08:20:59.209 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T08:20:59.237 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:20:59.548 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T08:20:59.564 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T08:20:59.607 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T08:20:59.656 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T08:20:59.669 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:20:59.706 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:20:59.909 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T08:20:59.923 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T08:20:59.968 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:21:00.271 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-mako. 2023-10-09T08:21:00.285 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T08:21:00.329 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:21:00.649 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T08:21:00.664 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T08:21:00.707 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:21:00.785 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T08:21:00.799 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:00.843 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:01.001 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T08:21:01.015 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T08:21:01.060 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:21:01.354 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-webob. 2023-10-09T08:21:01.368 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T08:21:01.412 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T08:21:02.716 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T08:21:02.730 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T08:21:02.766 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:21:03.052 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T08:21:03.066 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T08:21:03.110 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T08:21:03.447 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T08:21:03.460 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T08:21:03.499 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:21:03.825 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T08:21:03.832 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph. 2023-10-09T08:21:03.841 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T08:21:03.846 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:03.883 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T08:21:03.890 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:04.186 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-paste. 2023-10-09T08:21:04.201 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T08:21:04.210 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T08:21:04.223 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:04.244 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:21:04.285 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:04.581 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T08:21:04.597 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T08:21:04.639 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:21:04.697 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T08:21:04.710 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:04.755 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:04.925 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T08:21:04.939 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T08:21:05.017 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:21:05.283 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package cephadm. 2023-10-09T08:21:05.297 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:05.320 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T08:21:05.333 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T08:21:05.341 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:05.378 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T08:21:05.653 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T08:21:05.666 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T08:21:05.711 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T08:21:05.723 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T08:21:05.737 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T08:21:05.781 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:21:06.014 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T08:21:06.029 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:06.072 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:06.144 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T08:21:06.158 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T08:21:06.202 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:21:06.409 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T08:21:06.425 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T08:21:06.467 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T08:21:06.580 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T08:21:06.596 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:06.638 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:06.720 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T08:21:06.734 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T08:21:06.778 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:21:06.975 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T08:21:06.988 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:07.033 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:07.097 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T08:21:07.111 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T08:21:07.155 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:21:07.458 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-routes. 2023-10-09T08:21:07.472 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T08:21:07.486 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T08:21:07.500 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:07.533 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T08:21:07.536 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:07.845 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T08:21:07.858 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:07.903 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:08.598 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T08:21:08.613 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:08.656 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:08.979 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T08:21:08.993 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T08:21:09.037 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:21:09.331 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T08:21:09.345 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T08:21:09.389 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:21:09.709 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T08:21:09.722 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T08:21:09.767 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T08:21:11.094 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T08:21:11.109 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T08:21:11.151 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:21:11.337 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph. 2023-10-09T08:21:11.353 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:11.395 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:11.672 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T08:21:11.685 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T08:21:11.698 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T08:21:11.713 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:11.730 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T08:21:11.756 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:12.092 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T08:21:12.106 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T08:21:12.150 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:21:12.176 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T08:21:12.189 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:12.234 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:12.737 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package cephadm. 2023-10-09T08:21:12.753 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:12.795 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:12.876 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T08:21:12.890 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:12.934 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:13.090 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T08:21:13.105 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T08:21:13.148 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T08:21:13.467 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T08:21:13.481 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:13.525 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:13.904 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T08:21:13.918 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T08:21:13.962 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T08:21:14.053 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T08:21:14.069 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T08:21:14.111 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T08:21:14.215 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T08:21:14.228 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T08:21:14.273 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:21:14.415 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T08:21:14.429 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T08:21:14.473 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:21:14.542 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T08:21:14.556 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T08:21:14.600 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:21:14.818 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T08:21:14.832 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T08:21:14.870 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-routes. 2023-10-09T08:21:14.876 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:21:14.884 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T08:21:14.928 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T08:21:15.170 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T08:21:15.185 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T08:21:15.229 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T08:21:15.273 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T08:21:15.286 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:15.322 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:15.566 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T08:21:15.582 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T08:21:15.624 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T08:21:15.944 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T08:21:15.957 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T08:21:16.002 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:21:16.305 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T08:21:16.318 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T08:21:16.357 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T08:21:16.363 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:21:16.373 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T08:21:16.415 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:21:16.675 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T08:21:16.690 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T08:21:16.718 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T08:21:16.733 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:21:16.733 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T08:21:16.768 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:21:17.078 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T08:21:17.087 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T08:21:17.094 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T08:21:17.102 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T08:21:17.136 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:21:17.145 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T08:21:17.431 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T08:21:17.446 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T08:21:17.488 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:21:18.028 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T08:21:18.042 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:18.085 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:18.405 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T08:21:18.418 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T08:21:18.463 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:21:18.943 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T08:21:18.957 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:21:18.998 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T08:21:19.013 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T08:21:19.055 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:21:19.241 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:21:19.519 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package jq. 2023-10-09T08:21:19.532 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:21:19.576 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T08:21:19.576 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:21:19.590 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T08:21:19.634 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T08:21:19.812 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package socat. 2023-10-09T08:21:19.827 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T08:21:19.870 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T08:21:20.012 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T08:21:20.028 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T08:21:20.071 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:21:20.198 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T08:21:20.212 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T08:21:20.256 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:21:20.696 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package ceph-test. 2023-10-09T08:21:20.709 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:20.746 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T08:21:20.754 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:20.762 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:20.804 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:21.907 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T08:21:21.922 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T08:21:21.965 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T08:21:22.302 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T08:21:22.317 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T08:21:22.360 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:21:22.688 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T08:21:22.702 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T08:21:22.746 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:21:23.074 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T08:21:23.088 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T08:21:23.132 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T08:21:23.461 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T08:21:23.475 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T08:21:23.519 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T08:21:23.822 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T08:21:23.837 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T08:21:23.888 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:21:24.183 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T08:21:24.198 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T08:21:24.241 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:21:24.561 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T08:21:24.575 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T08:21:24.611 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:21:24.931 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T08:21:24.941 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T08:21:24.946 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T08:21:24.949 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:24.989 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:21:24.991 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:25.309 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T08:21:25.325 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T08:21:25.328 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T08:21:25.342 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T08:21:25.367 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:21:25.393 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:21:25.754 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T08:21:25.770 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T08:21:25.812 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:21:25.922 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T08:21:25.938 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:21:25.980 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:26.098 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-attr. 2023-10-09T08:21:26.112 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T08:21:26.163 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T08:21:26.308 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T08:21:26.322 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T08:21:26.366 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:21:26.508 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T08:21:26.522 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T08:21:26.557 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:21:26.720 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T08:21:26.734 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:21:26.777 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:21:26.852 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T08:21:26.866 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T08:21:26.910 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T08:21:27.047 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package jq. 2023-10-09T08:21:27.062 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:21:27.105 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:21:27.281 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T08:21:27.293 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T08:21:27.337 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:21:27.341 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package socat. 2023-10-09T08:21:27.357 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T08:21:27.399 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T08:21:27.718 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T08:21:27.734 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T08:21:27.776 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:21:27.791 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T08:21:27.806 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T08:21:27.841 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T08:21:28.152 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T08:21:28.166 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T08:21:28.210 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T08:21:28.217 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-test. 2023-10-09T08:21:28.233 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:28.275 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:28.530 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T08:21:28.546 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T08:21:28.588 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T08:21:28.882 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T08:21:28.898 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T08:21:28.940 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:21:29.277 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-py. 2023-10-09T08:21:29.293 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T08:21:29.335 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:21:29.654 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T08:21:29.668 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T08:21:29.704 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:21:30.040 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T08:21:30.054 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T08:21:30.098 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:21:30.426 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T08:21:30.442 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T08:21:30.492 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T08:21:30.879 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T08:21:30.893 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T08:21:30.946 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:21:31.466 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package radosgw. 2023-10-09T08:21:31.481 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:31.532 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:32.437 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T08:21:32.453 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:32.495 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:32.802 INFO:teuthology.orchestra.run.smithi094.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T08:21:32.816 INFO:teuthology.orchestra.run.smithi094.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:32.860 INFO:teuthology.orchestra.run.smithi094.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:32.899 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T08:21:32.908 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T08:21:32.965 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:21:33.169 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T08:21:33.331 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T08:21:33.345 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T08:21:33.389 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:21:33.474 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:21:33.600 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:21:33.705 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-attr. 2023-10-09T08:21:33.718 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T08:21:33.726 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:21:33.762 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T08:21:33.927 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:34.074 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T08:21:34.090 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T08:21:34.132 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:21:34.477 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T08:21:34.491 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T08:21:34.535 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T08:21:34.547 INFO:teuthology.orchestra.run.smithi094.stdout:Adding system user cephadm....done 2023-10-09T08:21:34.691 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:21:34.896 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T08:21:34.910 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T08:21:34.954 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:21:35.026 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T08:21:35.336 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T08:21:35.399 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T08:21:35.414 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T08:21:35.457 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T08:21:35.484 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:21:35.760 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T08:21:35.774 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T08:21:35.805 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T08:21:35.818 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T08:21:36.147 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T08:21:36.162 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T08:21:36.205 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T08:21:36.215 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:21:36.508 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T08:21:36.525 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T08:21:36.541 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T08:21:36.584 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:21:36.784 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:21:36.912 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-py. 2023-10-09T08:21:36.925 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T08:21:36.970 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:21:37.060 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:21:37.273 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T08:21:37.287 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T08:21:37.331 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:21:37.361 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:37.634 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T08:21:37.650 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T08:21:37.654 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:21:37.692 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:21:37.780 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:21:37.978 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T08:21:37.987 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T08:21:38.028 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T08:21:38.072 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:21:38.357 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:21:38.364 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T08:21:38.374 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T08:21:38.431 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:21:38.724 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:21:38.851 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package radosgw. 2023-10-09T08:21:38.861 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:38.901 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:38.992 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:21:39.327 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:21:39.645 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:39.770 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T08:21:40.097 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T08:21:40.105 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T08:21:40.110 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:21:40.154 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:40.398 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T08:21:40.457 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T08:21:40.690 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:21:40.760 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:21:40.817 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:21:40.869 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:21:40.995 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:21:41.117 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:21:41.196 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:41.511 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:21:41.809 INFO:teuthology.orchestra.run.smithi016.stdout:Adding system user cephadm....done 2023-10-09T08:21:41.871 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:21:41.971 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:21:42.223 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T08:21:42.314 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T08:21:42.349 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:42.598 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T08:21:42.675 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T08:21:42.725 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:21:43.034 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:21:43.060 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T08:21:43.169 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:21:43.295 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T08:21:43.453 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:21:43.579 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:21:43.754 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T08:21:43.897 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:21:44.039 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:21:44.040 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:21:44.323 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:21:44.367 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:21:44.608 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:44.634 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T08:21:44.909 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:21:44.936 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:21:45.035 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:21:45.061 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:21:45.328 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:21:45.337 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T08:21:45.612 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:21:45.638 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:21:45.939 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:21:45.988 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:21:46.273 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:21:46.313 INFO:teuthology.orchestra.run.smithi094.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T08:21:46.562 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:21:46.640 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:21:46.688 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:21:46.942 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:47.031 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:21:47.068 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T08:21:47.390 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:21:47.394 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T08:21:47.516 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:21:47.642 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T08:21:47.679 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T08:21:47.976 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:21:47.980 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:21:48.106 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:21:48.261 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:48.387 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T08:21:48.390 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:21:48.801 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:21:49.352 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:21:49.695 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T08:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:50.123 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T08:21:50.246 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:50.483 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:21:50.485 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:50.609 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:21:50.611 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:21:50.738 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T08:21:50.768 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T08:21:51.054 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:21:51.108 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T08:21:51.380 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:21:51.393 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:21:51.506 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:21:51.694 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T08:21:51.824 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:21:52.037 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:52.100 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T08:21:52.163 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:21:52.401 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:21:52.475 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:52.527 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:21:52.598 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:21:52.812 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T08:21:52.952 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:21:53.105 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:21:53.406 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:21:53.822 INFO:teuthology.orchestra.run.smithi016.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T08:21:53.990 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:54.076 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:21:54.116 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:21:54.194 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:21:54.242 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:21:54.368 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T08:21:54.503 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:21:54.661 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T08:21:54.871 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:21:54.970 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:54.997 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:21:55.096 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:55.119 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T08:21:55.342 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:21:55.342 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:21:55.447 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:21:55.723 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:55.831 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:55.832 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T08:21:55.956 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:21:56.241 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T08:21:56.550 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:21:56.911 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:57.045 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:57.171 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:57.297 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T08:21:57.363 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:57.489 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:57.590 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T08:21:57.599 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:21:57.733 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T08:21:57.969 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:58.118 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T08:21:58.136 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:58.428 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:21:58.447 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:21:58.721 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T08:21:59.022 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:59.149 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:21:59.377 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:59.480 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:21:59.623 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:21:59.923 INFO:teuthology.orchestra.run.smithi094.stdout:Adding group ceph....done 2023-10-09T08:21:59.957 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:22:00.320 INFO:teuthology.orchestra.run.smithi094.stdout:Adding system user ceph....done 2023-10-09T08:22:00.488 INFO:teuthology.orchestra.run.smithi094.stdout:Setting system user ceph properties....done 2023-10-09T08:22:00.508 INFO:teuthology.orchestra.run.smithi094.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T08:22:00.654 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T08:22:00.967 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T08:22:01.047 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:01.173 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:22:01.301 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:22:01.428 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T08:22:01.450 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:01.568 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:01.729 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T08:22:01.839 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:22:01.839 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:22:02.021 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:02.131 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:02.357 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:22:02.357 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:22:02.506 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:02.857 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:02.865 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T08:22:03.016 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:22:03.308 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:03.309 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T08:22:03.528 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:22:03.529 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:22:03.627 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:22:03.944 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:04.005 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:04.114 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:04.172 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:22:04.172 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:22:04.240 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:04.366 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T08:22:04.628 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:04.667 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T08:22:05.063 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:05.113 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:22:05.113 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:22:05.180 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:05.457 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:22:05.572 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:05.732 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:05.875 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:06.094 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:22:06.095 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:22:06.419 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:06.552 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:06.686 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:06.812 INFO:teuthology.orchestra.run.smithi094.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:06.956 INFO:teuthology.orchestra.run.smithi094.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T08:22:06.967 INFO:teuthology.orchestra.run.smithi016.stdout:Adding group ceph....done 2023-10-09T08:22:07.240 INFO:teuthology.orchestra.run.smithi094.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:22:07.369 INFO:teuthology.orchestra.run.smithi016.stdout:Adding system user ceph....done 2023-10-09T08:22:07.546 INFO:teuthology.orchestra.run.smithi016.stdout:Setting system user ceph properties....done 2023-10-09T08:22:07.564 INFO:teuthology.orchestra.run.smithi016.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T08:22:07.718 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T08:22:08.047 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T08:22:08.533 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:08.720 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:08.972 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:22:08.972 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:22:09.640 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:10.024 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T08:22:10.194 INFO:teuthology.orchestra.run.smithi094.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:22:10.509 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:10.647 INFO:teuthology.orchestra.run.smithi094.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T08:22:10.650 DEBUG:teuthology.parallel:result is None 2023-10-09T08:22:10.733 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:22:10.733 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:22:11.227 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:11.457 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:22:11.457 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:22:11.970 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:12.485 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:22:12.485 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:22:12.957 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:13.083 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:13.218 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:13.438 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:22:13.438 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:22:13.911 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:14.054 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:14.171 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:22:14.307 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T08:22:14.616 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:22:17.645 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:22:18.104 INFO:teuthology.orchestra.run.smithi016.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T08:22:18.106 DEBUG:teuthology.parallel:result is None 2023-10-09T08:22:18.107 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:22:18.107 INFO:teuthology.packaging:ref: None 2023-10-09T08:22:18.107 INFO:teuthology.packaging:tag: None 2023-10-09T08:22:18.107 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:22:18.107 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:22:18.107 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:22:18.344 DEBUG:teuthology.orchestra.run.smithi016:> dpkg-query -W -f '${Version}' ceph 2023-10-09T08:22:18.367 INFO:teuthology.orchestra.run.smithi016.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T08:22:18.367 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:22:18.368 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T08:22:18.368 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:22:18.368 INFO:teuthology.packaging:ref: None 2023-10-09T08:22:18.368 INFO:teuthology.packaging:tag: None 2023-10-09T08:22:18.368 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:22:18.368 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:22:18.369 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:22:18.592 DEBUG:teuthology.orchestra.run.smithi094:> dpkg-query -W -f '${Version}' ceph 2023-10-09T08:22:18.615 INFO:teuthology.orchestra.run.smithi094.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T08:22:18.615 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:22:18.616 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T08:22:18.616 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T08:22:18.616 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:22:18.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T08:22:18.633 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:22:18.633 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T08:22:18.671 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T08:22:18.671 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:22:18.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T08:22:18.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T08:22:18.752 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:22:18.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T08:22:18.765 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T08:22:18.824 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T08:22:18.824 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:22:18.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T08:22:18.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T08:22:18.894 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:22:18.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T08:22:18.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T08:22:18.967 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T08:22:18.967 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:22:18.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T08:22:18.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T08:22:19.044 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:22:19.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T08:22:19.059 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T08:22:19.114 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T08:22:19.187 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T08:22:19.187 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T08:22:19.188 INFO:tasks.cephadm:Cluster fsid is f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:22:19.188 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T08:22:19.188 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.16', 'mon.c': '[v2:172.21.15.16:3301,v1:172.21.15.16:6790]', 'mon.b': '172.21.15.94'} 2023-10-09T08:22:19.188 INFO:tasks.cephadm:First mon is mon.a on smithi016 2023-10-09T08:22:19.189 INFO:tasks.cephadm:First mgr is y 2023-10-09T08:22:19.189 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T08:22:19.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo hostname $(hostname -s) 2023-10-09T08:22:19.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo hostname $(hostname -s) 2023-10-09T08:22:19.220 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-09T08:22:19.220 DEBUG:teuthology.orchestra.run.smithi016:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T08:22:19.614 INFO:teuthology.orchestra.run.smithi016.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 08:22 /home/ubuntu/cephtest/cephadm 2023-10-09T08:22:19.614 DEBUG:teuthology.orchestra.run.smithi094:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T08:22:19.861 INFO:teuthology.orchestra.run.smithi094.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 08:22 /home/ubuntu/cephtest/cephadm 2023-10-09T08:22:19.861 DEBUG:teuthology.orchestra.run.smithi016:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T08:22:19.871 DEBUG:teuthology.orchestra.run.smithi094:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T08:22:19.889 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-09T08:22:19.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T08:22:19.914 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T08:22:20.184 INFO:teuthology.orchestra.run.smithi094.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T08:22:20.192 INFO:teuthology.orchestra.run.smithi016.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T08:23:01.990 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:23:01.990 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T08:23:01.990 INFO:teuthology.orchestra.run.smithi016.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T08:23:01.990 INFO:teuthology.orchestra.run.smithi016.stdout: "repo_digests": [ 2023-10-09T08:23:01.990 INFO:teuthology.orchestra.run.smithi016.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T08:23:01.991 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:23:01.991 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:23:05.423 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:23:05.423 INFO:teuthology.orchestra.run.smithi094.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T08:23:05.423 INFO:teuthology.orchestra.run.smithi094.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T08:23:05.423 INFO:teuthology.orchestra.run.smithi094.stdout: "repo_digests": [ 2023-10-09T08:23:05.424 INFO:teuthology.orchestra.run.smithi094.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T08:23:05.424 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:23:05.424 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:23:05.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo mkdir -p /etc/ceph 2023-10-09T08:23:05.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo mkdir -p /etc/ceph 2023-10-09T08:23:05.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 777 /etc/ceph 2023-10-09T08:23:05.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod 777 /etc/ceph 2023-10-09T08:23:05.540 INFO:tasks.cephadm:Writing seed config... 2023-10-09T08:23:05.541 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T08:23:05.541 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T08:23:05.541 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T08:23:05.542 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T08:23:05.543 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:23:05.543 DEBUG:teuthology.orchestra.run.smithi016:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T08:23:05.575 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = f674ade6-667c-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T08:23:05.575 DEBUG:teuthology.orchestra.run.smithi016:mon.a> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T08:23:05.618 DEBUG:teuthology.orchestra.run.smithi016:mgr.y> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T08:23:05.662 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T08:23:05.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.16 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T08:23:05.867 INFO:teuthology.orchestra.run.smithi016.stderr:-------------------------------------------------------------------------------- 2023-10-09T08:23:05.867 INFO:teuthology.orchestra.run.smithi016.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', 'f674ade6-667c-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.16', '--skip-admin-label'] 2023-10-09T08:23:05.867 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying podman|docker is present... 2023-10-09T08:23:05.867 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying lvm2 is present... 2023-10-09T08:23:05.868 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying time synchronization is in place... 2023-10-09T08:23:05.875 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T08:23:05.881 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.885 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T08:23:05.891 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.896 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: masked 2023-10-09T08:23:05.902 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T08:23:05.914 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.921 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: enabled 2023-10-09T08:23:05.927 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: active 2023-10-09T08:23:05.928 INFO:teuthology.orchestra.run.smithi016.stderr:Unit ntp.service is enabled and running 2023-10-09T08:23:05.928 INFO:teuthology.orchestra.run.smithi016.stderr:Repeating the final host check... 2023-10-09T08:23:05.929 INFO:teuthology.orchestra.run.smithi016.stderr:docker (/usr/bin/docker) is present 2023-10-09T08:23:05.929 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl is present 2023-10-09T08:23:05.929 INFO:teuthology.orchestra.run.smithi016.stderr:lvcreate is present 2023-10-09T08:23:05.935 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T08:23:05.941 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.948 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T08:23:05.954 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.961 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: masked 2023-10-09T08:23:05.967 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.972 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T08:23:05.978 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-10-09T08:23:05.984 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: enabled 2023-10-09T08:23:05.990 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: active 2023-10-09T08:23:05.991 INFO:teuthology.orchestra.run.smithi016.stderr:Unit ntp.service is enabled and running 2023-10-09T08:23:05.991 INFO:teuthology.orchestra.run.smithi016.stderr:Host looks OK 2023-10-09T08:23:05.991 INFO:teuthology.orchestra.run.smithi016.stderr:Cluster fsid: f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:05.991 INFO:teuthology.orchestra.run.smithi016.stderr:Acquiring lock 139979299352528 on /run/cephadm/f674ade6-667c-11ee-8db5-212e2dc638e7.lock 2023-10-09T08:23:05.991 INFO:teuthology.orchestra.run.smithi016.stderr:Lock 139979299352528 acquired on /run/cephadm/f674ade6-667c-11ee-8db5-212e2dc638e7.lock 2023-10-09T08:23:05.992 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying IP 172.21.15.16 port 3300 ... 2023-10-09T08:23:05.992 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying IP 172.21.15.16 port 6789 ... 2023-10-09T08:23:05.992 INFO:teuthology.orchestra.run.smithi016.stderr:Base mon IP(s) is [172.21.15.16:3300, 172.21.15.16:6789], mon addrv is [v2:172.21.15.16:3300,v1:172.21.15.16:6789] 2023-10-09T08:23:05.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: default via 172.21.15.254 dev ens1f0 2023-10-09T08:23:05.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-10-09T08:23:05.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.16 2023-10-09T08:23:06.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T08:23:06.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-10-09T08:23:06.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1793sec hoplimit 64 pref medium 2023-10-09T08:23:06.005 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T08:23:06.005 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-10-09T08:23:06.005 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T08:23:06.005 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2023-10-09T08:23:06.005 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:febd:1564/64 scope link 2023-10-09T08:23:06.005 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T08:23:06.006 INFO:teuthology.orchestra.run.smithi016.stderr:Mon IP `172.21.15.16` is in CIDR network `172.21.0.0/20` 2023-10-09T08:23:06.006 INFO:teuthology.orchestra.run.smithi016.stderr:Mon IP `172.21.15.16` is in CIDR network `172.21.0.0/20` 2023-10-09T08:23:06.007 INFO:teuthology.orchestra.run.smithi016.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T08:23:06.007 INFO:teuthology.orchestra.run.smithi016.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T08:23:06.008 INFO:teuthology.orchestra.run.smithi016.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T08:23:06.081 INFO:journalctl@ceph.mgr.y.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:23:06.083 INFO:journalctl@ceph.mon.a.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:23:06.226 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-10-09T08:23:06.256 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: Digest: sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3 2023-10-09T08:23:06.256 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T08:23:06.281 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T08:23:07.260 INFO:teuthology.orchestra.run.smithi016.stderr:ceph: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T08:23:07.868 INFO:teuthology.orchestra.run.smithi016.stderr:Ceph version: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T08:23:07.868 INFO:teuthology.orchestra.run.smithi016.stderr:Extracting ceph user uid/gid from container image... 2023-10-09T08:23:08.689 INFO:teuthology.orchestra.run.smithi016.stderr:stat: 167 167 2023-10-09T08:23:09.338 INFO:teuthology.orchestra.run.smithi016.stderr:Creating initial keys... 2023-10-09T08:23:10.141 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-authtool: AQBuuCNlJcpKCBAAwQS9MZu3X6s1ToKSGAD3Bw== 2023-10-09T08:23:11.530 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-authtool: AQBvuCNlL42AHxAAZEv+fgdTiOqcIS5vUfG0Ng== 2023-10-09T08:23:12.895 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-authtool: AQBwuCNl6fBANRAAro7sDPo2UKcyh3cY3QhVvQ== 2023-10-09T08:23:13.454 INFO:teuthology.orchestra.run.smithi016.stderr:Creating initial monmap... 2023-10-09T08:23:14.303 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T08:23:14.303 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:14.303 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T08:23:14.881 INFO:teuthology.orchestra.run.smithi016.stderr:monmaptool for a [v2:172.21.15.16:3300,v1:172.21.15.16:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T08:23:14.881 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: set fsid to f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:14.881 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T08:23:14.882 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:23:14.882 INFO:teuthology.orchestra.run.smithi016.stderr:Creating mon... 2023-10-09T08:23:15.689 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.682+0000 7f7a398a6880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T08:23:15.689 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.682+0000 7f7a398a6880 1 imported monmap: 2023-10-09T08:23:15.690 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-09T08:23:15.690 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:15.690 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: last_changed 2023-10-09T08:23:14.300421+0000 2023-10-09T08:23:15.690 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: created 2023-10-09T08:23:14.300421+0000 2023-10-09T08:23:15.691 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] mon.a 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.682+0000 7f7a398a6880 0 /usr/bin/ceph-mon: set fsid to f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: DB SUMMARY 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.696 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.create_if_missing: 1 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.env: 0x5570f73c1080 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.fs: Posix File System 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.info_log: 0x5570f8433aa0 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.statistics: (nil) 2023-10-09T08:23:15.697 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.use_fsync: 0 2023-10-09T08:23:15.698 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T08:23:15.698 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T08:23:15.698 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T08:23:15.698 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T08:23:15.698 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T08:23:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T08:23:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.db_log_dir: 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T08:23:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.write_buffer_manager: 0x5570f843b890 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T08:23:15.721 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.unordered_write: 0 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.row_cache: None 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.wal_filter: None 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T08:23:15.722 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T08:23:15.723 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_open_files: -1 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T08:23:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Compression algorithms supported: 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kZSTD supported: 0 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-09T08:23:15.725 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.merge_operator: 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_filter: None 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T08:23:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5570f8349d20) 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache: 0x5570f8380d10 2023-10-09T08:23:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-09T08:23:15.728 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T08:23:15.729 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compression: NoCompression 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.num_levels: 7 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T08:23:15.730 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T08:23:15.731 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T08:23:15.732 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T08:23:15.733 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T08:23:15.734 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.ttl: 2592000 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a398a6880 4 rocksdb: DB pointer 0x5570f8449800 2023-10-09T08:23:15.735 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a2272e700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.686+0000 7f7a2272e700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:15.736 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:23:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:23:15.738 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:15.739 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:23:15.740 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.740 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:23:15.740 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-10-09T08:23:15.740 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.690+0000 7f7a398a6880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-09T08:23:15.740 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.690+0000 7f7a398a6880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-09T08:23:15.740 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:23:15.690+0000 7f7a398a6880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-09T08:23:16.329 INFO:teuthology.orchestra.run.smithi016.stderr:create mon.a on 2023-10-09T08:23:16.352 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T08:23:16.545 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-10-09T08:23:16.779 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7.target. 2023-10-09T08:23:16.779 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7.target. 2023-10-09T08:23:17.203 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to reset failed state of unit ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T08:23:17.211 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7.target.wants/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@.service. 2023-10-09T08:23:17.422 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-10-09T08:23:17.422 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T08:23:17.422 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mon to start... 2023-10-09T08:23:17.422 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mon... 2023-10-09T08:23:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:18 smithi016 bash[26900]: cluster 2023-10-09T08:23:18.612915+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T08:23:18.830 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: cluster: 2023-10-09T08:23:18.831 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: id: f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: services: 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.207888s) 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:18.832 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: data: 2023-10-09T08:23:18.833 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-09T08:23:18.833 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-09T08:23:18.833 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-09T08:23:18.833 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: pgs: 2023-10-09T08:23:18.833 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:19.446 INFO:teuthology.orchestra.run.smithi016.stderr:mon is available 2023-10-09T08:23:19.446 INFO:teuthology.orchestra.run.smithi016.stderr:Assimilating anything we can from ceph.conf... 2023-10-09T08:23:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:19 smithi016 bash[26900]: cluster 2023-10-09T08:23:18.620484+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T08:23:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:19 smithi016 bash[26900]: cluster 2023-10-09T08:23:18.620794+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0]} removed_ranks: {} 2023-10-09T08:23:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:19 smithi016 bash[26900]: cluster 2023-10-09T08:23:18.623849+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-09T08:23:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:19 smithi016 bash[26900]: cluster 2023-10-09T08:23:18.626249+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T08:23:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:19 smithi016 bash[26900]: cluster 2023-10-09T08:23:18.626866+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T08:23:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:19 smithi016 bash[26900]: audit 2023-10-09T08:23:18.828247+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.16:0/3693287897' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T08:23:20.509 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:20.509 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [global] 2023-10-09T08:23:20.509 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: fsid = f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.16:3300,v1:172.21.15.16:6789] 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [mgr] 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:20.510 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [osd] 2023-10-09T08:23:20.511 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T08:23:20.511 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T08:23:21.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:20 smithi016 bash[26900]: audit 2023-10-09T08:23:20.502292+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.16:0/1222059304' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T08:23:21.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:20 smithi016 bash[26900]: audit 2023-10-09T08:23:20.506062+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.16:0/1222059304' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T08:23:21.323 INFO:teuthology.orchestra.run.smithi016.stderr:Generating new minimal ceph.conf... 2023-10-09T08:23:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:22 smithi016 bash[26900]: audit 2023-10-09T08:23:22.351901+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.16:0/1887335940' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:23:22.906 INFO:teuthology.orchestra.run.smithi016.stderr:Restarting the monitor... 2023-10-09T08:23:23.181 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:22 smithi016 systemd[1]: Stopping Ceph mon.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:23:23.181 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:22 smithi016 bash[27300]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon.a 2023-10-09T08:23:23.181 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:23 smithi016 bash[26900]: debug 2023-10-09T08:23:23.085+0000 7f9b58696700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T08:23:23.181 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:23 smithi016 bash[26900]: debug 2023-10-09T08:23:23.085+0000 7f9b58696700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T08:23:23.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:23 smithi016 bash[27336]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T08:23:23.761 INFO:teuthology.orchestra.run.smithi016.stderr:Setting mon public_network to 172.21.0.0/20 2023-10-09T08:23:23.811 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:23 smithi016 bash[27413]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T08:23:23.812 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:23 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T08:23:23.812 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:23 smithi016 systemd[1]: Stopped Ceph mon.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:23:23.812 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:23 smithi016 systemd[1]: Started Ceph mon.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.925+0000 7f4b6f258880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.925+0000 7f4b6f258880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.925+0000 7f4b6f258880 0 pidfile_write: ignore empty --pid-file 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 0 load: jerasure load: lrc load: isa 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T08:23:25.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: DB SUMMARY 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86707 ; 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.env: 0x558ee29c1080 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.fs: Posix File System 2023-10-09T08:23:25.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.info_log: 0x558ee4947a80 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.statistics: (nil) 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.use_fsync: 0 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T08:23:25.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T08:23:25.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T08:23:25.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T08:23:25.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T08:23:25.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T08:23:25.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.db_log_dir: 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T08:23:25.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.write_buffer_manager: 0x558ee5644330 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T08:23:25.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.unordered_write: 0 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.row_cache: None 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.wal_filter: None 2023-10-09T08:23:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T08:23:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_open_files: -1 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T08:23:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Compression algorithms supported: 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kZSTD supported: 0 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T08:23:25.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.merge_operator: 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_filter: None 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558ee48f5d18) 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: cache_index_and_filter_blocks: 1 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T08:23:25.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: pin_top_level_index_and_filter: 1 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: index_type: 0 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: data_block_index_type: 0 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: index_shortening: 1 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: hash_index_allow_collision: 1 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: checksum: 1 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: no_block_cache: 0 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_cache: 0x558ee492cf10 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_cache_name: BinnedLRUCache 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_cache_options: 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: capacity : 536870912 2023-10-09T08:23:25.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: num_shard_bits : 4 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: strict_capacity_limit : 0 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: high_pri_pool_ratio: 0.000 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_cache_compressed: (nil) 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: persistent_cache: (nil) 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_size: 4096 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_size_deviation: 10 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_restart_interval: 16 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: index_block_restart_interval: 1 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: metadata_block_size: 4096 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: partition_filters: 0 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: use_delta_encoding: 1 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T08:23:25.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: whole_key_filtering: 1 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: verify_compression: 0 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: read_amp_bytes_per_bit: 0 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: format_version: 2 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: enable_index_compression: 1 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: block_align: 0 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compression: NoCompression 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T08:23:25.258 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.num_levels: 7 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T08:23:25.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T08:23:25.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T08:23:25.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T08:23:25.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T08:23:25.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T08:23:25.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T08:23:25.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.ttl: 2592000 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.929+0000 7f4b6f258880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696839804936932, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696839804938054, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83307, "table_properties": {"data_size": 81580, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75901, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696839804, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696839804938590, "job": 1, "event": "recovery_finished"} 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b6f258880 4 rocksdb: DB pointer 0x558ee49f5800 2023-10-09T08:23:25.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b5605f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.933+0000 7f4b5605f700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ** DB Stats ** 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ** Compaction Stats [default] ** 2023-10-09T08:23:25.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 110.7 0.00 0.00 1 0.001 0 0 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 110.7 0.00 0.00 1 0.001 0 0 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 110.7 0.00 0.00 1 0.001 0 0 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ** Compaction Stats [default] ** 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 110.7 0.00 0.00 1 0.001 0 0 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:25.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Cumulative compaction: 0.00 GB write, 18.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Interval compaction: 0.00 GB write, 18.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ** Compaction Stats [default] ** 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:25.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 110.7 0.00 0.00 1 0.001 0 0 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 110.7 0.00 0.00 1 0.001 0 0 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ** Compaction Stats [default] ** 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 110.7 0.00 0.00 1 0.001 0 0 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:23:25.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Cumulative compaction: 0.00 GB write, 18.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] at bind addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 1 mon.a@-1(???) e1 preinit fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 0 mon.a@-1(???).mds e1 new map 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 0 mon.a@-1(???).mds e1 print_map 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: e1 2023-10-09T08:23:25.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: legacy client fscid: -1 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: No filesystems configured 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: audit 2023-10-09T08:23:22.351901+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.16:0/1887335940' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:24 smithi016 bash[27454]: debug 2023-10-09T08:23:24.937+0000 7f4b6f258880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T08:23:25.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:25 smithi016 bash[27454]: cluster 2023-10-09T08:23:24.972599+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T08:23:25.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:25 smithi016 bash[27454]: cluster 2023-10-09T08:23:24.972709+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0]} removed_ranks: {} 2023-10-09T08:23:25.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:25 smithi016 bash[27454]: cluster 2023-10-09T08:23:24.973370+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-09T08:23:25.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:25 smithi016 bash[27454]: cluster 2023-10-09T08:23:24.973431+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T08:23:25.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:25 smithi016 bash[27454]: cluster 2023-10-09T08:23:24.974003+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T08:23:25.847 INFO:teuthology.orchestra.run.smithi016.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-09T08:23:25.847 INFO:teuthology.orchestra.run.smithi016.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T08:23:25.847 INFO:teuthology.orchestra.run.smithi016.stderr:Creating mgr... 2023-10-09T08:23:25.847 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying port 9283 ... 2023-10-09T08:23:26.083 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to reset failed state of unit ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T08:23:26.093 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7.target.wants/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@.service. 2023-10-09T08:23:26.317 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-10-09T08:23:26.318 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T08:23:26.318 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-10-09T08:23:26.318 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T08:23:26.318 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr to start... 2023-10-09T08:23:26.318 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr... 2023-10-09T08:23:26.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:26 smithi016 bash[27454]: audit 2023-10-09T08:23:25.189036+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.16:0/831828972' entity='client.admin' 2023-10-09T08:23:27.834 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:27.834 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-10-09T08:23:27.834 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsid": "f674ade6-667c-11ee-8db5-212e2dc638e7", 2023-10-09T08:23:27.834 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "health": { 2023-10-09T08:23:27.834 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T08:23:27.834 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 0 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "a" 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T08:23:27.835 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T08:23:27.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T08:23:27.839 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T08:23:27.839 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T08:23:27.840 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:27.840 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T08:23:27.840 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T08:23:27.840 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T08:23:27.840 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T08:23:27.841 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": false, 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "iostat", 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "nfs", 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "restful" 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modified": "2023-10-09T08:23:18.622787+0000", 2023-10-09T08:23:27.842 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:23:27.843 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:27.843 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T08:23:27.843 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-10-09T08:23:28.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:27 smithi016 bash[27454]: audit 2023-10-09T08:23:27.829680+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.16:0/3604159703' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T08:23:28.052 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:27 smithi016 bash[27723]: debug 2023-10-09T08:23:27.905+0000 7f7502384680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:23:28.532 INFO:teuthology.orchestra.run.smithi016.stderr:mgr not available, waiting (1/15)... 2023-10-09T08:23:28.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:28 smithi016 bash[27723]: debug 2023-10-09T08:23:28.393+0000 7f7502384680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:23:29.737 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:29 smithi016 bash[27723]: debug 2023-10-09T08:23:29.369+0000 7f7502384680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:23:29.737 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:29 smithi016 bash[27723]: debug 2023-10-09T08:23:29.469+0000 7f7502384680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:23:30.914 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:29 smithi016 bash[27723]: debug 2023-10-09T08:23:29.729+0000 7f7502384680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:23:31.184 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:30 smithi016 bash[27723]: debug 2023-10-09T08:23:30.909+0000 7f7502384680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:23:31.184 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:31 smithi016 bash[27723]: debug 2023-10-09T08:23:31.077+0000 7f7502384680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:23:31.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:31 smithi016 bash[27723]: debug 2023-10-09T08:23:31.177+0000 7f7502384680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:23:31.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:31 smithi016 bash[27723]: debug 2023-10-09T08:23:31.297+0000 7f7502384680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:23:31.751 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:31.751 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-10-09T08:23:31.751 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsid": "f674ade6-667c-11ee-8db5-212e2dc638e7", 2023-10-09T08:23:31.751 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "health": { 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 0 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T08:23:31.752 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "a" 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T08:23:31.753 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T08:23:31.754 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T08:23:31.757 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T08:23:31.757 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T08:23:31.757 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:31.757 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T08:23:31.757 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T08:23:31.757 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T08:23:31.757 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T08:23:31.758 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": false, 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "iostat", 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "nfs", 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "restful" 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T08:23:31.759 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:31.760 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modified": "2023-10-09T08:23:18.622787+0000", 2023-10-09T08:23:31.760 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:23:31.760 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:31.760 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T08:23:31.760 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-10-09T08:23:31.796 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:31 smithi016 bash[27723]: debug 2023-10-09T08:23:31.525+0000 7f7502384680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:23:32.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:31 smithi016 bash[27454]: audit 2023-10-09T08:23:31.746070+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.16:0/933649551' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T08:23:32.250 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:31 smithi016 bash[27723]: debug 2023-10-09T08:23:31.905+0000 7f7502384680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:23:32.250 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:32 smithi016 bash[27723]: debug 2023-10-09T08:23:32.029+0000 7f7502384680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:23:32.344 INFO:teuthology.orchestra.run.smithi016.stderr:mgr not available, waiting (2/15)... 2023-10-09T08:23:33.250 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:32 smithi016 bash[27723]: debug 2023-10-09T08:23:32.921+0000 7f7502384680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:23:33.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:33 smithi016 bash[27723]: debug 2023-10-09T08:23:33.033+0000 7f7502384680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:23:33.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:33 smithi016 bash[27723]: debug 2023-10-09T08:23:33.153+0000 7f7502384680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:23:33.681 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:33 smithi016 bash[27723]: debug 2023-10-09T08:23:33.381+0000 7f7502384680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:23:34.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:33 smithi016 bash[27723]: debug 2023-10-09T08:23:33.673+0000 7f7502384680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:23:34.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:33 smithi016 bash[27723]: debug 2023-10-09T08:23:33.849+0000 7f7502384680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:23:34.362 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27723]: debug 2023-10-09T08:23:34.025+0000 7f7502384680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:23:34.363 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27723]: debug 2023-10-09T08:23:34.245+0000 7f7502384680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:23:34.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27723]: debug 2023-10-09T08:23:34.357+0000 7f7502384680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:23:34.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: cluster 2023-10-09T08:23:34.381886+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-09T08:23:34.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: cluster 2023-10-09T08:23:34.387057+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00535293s) 2023-10-09T08:23:34.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.388738+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.388937+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.389109+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.389266+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.389461+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.389638+0000 mon.a (mon.0) 16 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.389820+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.390164+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: cluster 2023-10-09T08:23:34.411452+0000 mon.a (mon.0) 19 : cluster [INF] Manager daemon y is now available 2023-10-09T08:23:34.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:34 smithi016 bash[27454]: audit 2023-10-09T08:23:34.425242+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' 2023-10-09T08:23:35.489 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:35.493 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsid": "f674ade6-667c-11ee-8db5-212e2dc638e7", 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "health": { 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 0 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:35.494 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "a" 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_age": 10, 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:35.495 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T08:23:35.496 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T08:23:35.497 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T08:23:35.497 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T08:23:35.497 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T08:23:35.499 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T08:23:35.499 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:35.499 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T08:23:35.499 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:35.499 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": true, 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "iostat", 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "nfs", 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "restful" 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:23:35.500 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:35.501 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T08:23:35.501 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:23:35.501 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modified": "2023-10-09T08:23:18.622787+0000", 2023-10-09T08:23:35.501 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:23:35.501 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-10-09T08:23:35.501 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T08:23:35.501 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-10-09T08:23:35.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:35 smithi016 bash[27454]: audit 2023-10-09T08:23:34.436803+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:23:35.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:35 smithi016 bash[27454]: audit 2023-10-09T08:23:34.441080+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' 2023-10-09T08:23:35.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:35 smithi016 bash[27454]: audit 2023-10-09T08:23:34.441277+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:23:35.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:35 smithi016 bash[27454]: audit 2023-10-09T08:23:34.444658+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.14100 172.21.15.16:0/3673607432' entity='mgr.y' 2023-10-09T08:23:35.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:35 smithi016 bash[27454]: cluster 2023-10-09T08:23:35.390906+0000 mon.a (mon.0) 25 : cluster [DBG] mgrmap e3: y(active, since 1.00921s) 2023-10-09T08:23:36.051 INFO:teuthology.orchestra.run.smithi016.stderr:mgr is available 2023-10-09T08:23:36.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:36 smithi016 bash[27454]: audit 2023-10-09T08:23:35.486961+0000 mon.a (mon.0) 26 : audit [DBG] from='client.? 172.21.15.16:0/685627550' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T08:23:37.329 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:37.329 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [global] 2023-10-09T08:23:37.329 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: fsid = f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:37.329 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T08:23:37.330 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T08:23:37.330 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T08:23:37.330 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [mgr] 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [osd] 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T08:23:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T08:23:37.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:37 smithi016 bash[27454]: cluster 2023-10-09T08:23:36.440358+0000 mon.a (mon.0) 27 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-10-09T08:23:37.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:37 smithi016 bash[27454]: audit 2023-10-09T08:23:37.324920+0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.16:0/1779905052' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T08:23:37.972 INFO:teuthology.orchestra.run.smithi016.stderr:Enabling cephadm module... 2023-10-09T08:23:39.692 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:39 smithi016 bash[27454]: audit 2023-10-09T08:23:39.322612+0000 mon.a (mon.0) 29 : audit [INF] from='client.? 172.21.15.16:0/3042864268' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T08:23:39.693 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:39 smithi016 bash[27723]: ignoring --setuser ceph since I am not root 2023-10-09T08:23:39.693 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:39 smithi016 bash[27723]: ignoring --setgroup ceph since I am not root 2023-10-09T08:23:39.693 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:39 smithi016 bash[27723]: debug 2023-10-09T08:23:39.581+0000 7fc8dff67680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:23:39.693 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:39 smithi016 bash[27723]: debug 2023-10-09T08:23:39.685+0000 7fc8dff67680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:23:40.382 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:40 smithi016 bash[27723]: debug 2023-10-09T08:23:40.173+0000 7fc8dff67680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:23:40.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:40 smithi016 bash[27454]: audit 2023-10-09T08:23:39.379246+0000 mon.a (mon.0) 30 : audit [INF] from='client.? 172.21.15.16:0/3042864268' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T08:23:40.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:40 smithi016 bash[27454]: cluster 2023-10-09T08:23:39.379397+0000 mon.a (mon.0) 31 : cluster [DBG] mgrmap e5: y(active, since 4s) 2023-10-09T08:23:41.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:41 smithi016 bash[27723]: debug 2023-10-09T08:23:41.133+0000 7fc8dff67680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:23:41.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:41 smithi016 bash[27723]: debug 2023-10-09T08:23:41.237+0000 7fc8dff67680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:23:41.540 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-10-09T08:23:41.541 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-09T08:23:41.541 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": true, 2023-10-09T08:23:41.541 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T08:23:41.541 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T08:23:41.541 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-10-09T08:23:41.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:41 smithi016 bash[27454]: audit 2023-10-09T08:23:41.537787+0000 mon.a (mon.0) 32 : audit [DBG] from='client.? 172.21.15.16:0/757236299' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T08:23:41.775 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:41 smithi016 bash[27723]: debug 2023-10-09T08:23:41.493+0000 7fc8dff67680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:23:42.142 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for the mgr to restart... 2023-10-09T08:23:42.142 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr epoch 5... 2023-10-09T08:23:42.957 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:42 smithi016 bash[27723]: debug 2023-10-09T08:23:42.673+0000 7fc8dff67680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:23:42.957 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:42 smithi016 bash[27723]: debug 2023-10-09T08:23:42.849+0000 7fc8dff67680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:23:43.250 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:42 smithi016 bash[27723]: debug 2023-10-09T08:23:42.949+0000 7fc8dff67680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:23:43.250 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:43 smithi016 bash[27723]: debug 2023-10-09T08:23:43.069+0000 7fc8dff67680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:23:43.688 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:43 smithi016 bash[27723]: debug 2023-10-09T08:23:43.297+0000 7fc8dff67680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:23:44.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:43 smithi016 bash[27723]: debug 2023-10-09T08:23:43.681+0000 7fc8dff67680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:23:44.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:43 smithi016 bash[27723]: debug 2023-10-09T08:23:43.805+0000 7fc8dff67680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:23:45.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:44 smithi016 bash[27723]: debug 2023-10-09T08:23:44.705+0000 7fc8dff67680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:23:45.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:44 smithi016 bash[27723]: debug 2023-10-09T08:23:44.817+0000 7fc8dff67680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:23:45.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:44 smithi016 bash[27723]: debug 2023-10-09T08:23:44.941+0000 7fc8dff67680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:23:45.496 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:45 smithi016 bash[27723]: debug 2023-10-09T08:23:45.165+0000 7fc8dff67680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:23:45.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:45 smithi016 bash[27723]: debug 2023-10-09T08:23:45.489+0000 7fc8dff67680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:23:45.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:45 smithi016 bash[27723]: debug 2023-10-09T08:23:45.657+0000 7fc8dff67680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:23:46.180 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:45 smithi016 bash[27723]: debug 2023-10-09T08:23:45.837+0000 7fc8dff67680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:23:46.180 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27723]: debug 2023-10-09T08:23:46.061+0000 7fc8dff67680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:23:46.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27723]: debug 2023-10-09T08:23:46.173+0000 7fc8dff67680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:23:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: cluster 2023-10-09T08:23:46.198451+0000 mon.a (mon.0) 33 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: cluster 2023-10-09T08:23:46.198877+0000 mon.a (mon.0) 34 : cluster [INF] Activating manager daemon y 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: cluster 2023-10-09T08:23:46.206229+0000 mon.a (mon.0) 35 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: cluster 2023-10-09T08:23:46.206527+0000 mon.a (mon.0) 36 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00780599s) 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: audit 2023-10-09T08:23:46.207822+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: audit 2023-10-09T08:23:46.209053+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: audit 2023-10-09T08:23:46.209928+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: audit 2023-10-09T08:23:46.210287+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:23:46.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: audit 2023-10-09T08:23:46.210577+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:23:46.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:46 smithi016 bash[27454]: cluster 2023-10-09T08:23:46.236136+0000 mon.a (mon.0) 42 : cluster [INF] Manager daemon y is now available 2023-10-09T08:23:47.226 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-10-09T08:23:47.227 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-09T08:23:47.227 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T08:23:47.227 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-10-09T08:23:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: cephadm 2023-10-09T08:23:46.254652+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: audit 2023-10-09T08:23:46.259044+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: audit 2023-10-09T08:23:46.389899+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: audit 2023-10-09T08:23:46.392059+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: audit 2023-10-09T08:23:46.395138+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: audit 2023-10-09T08:23:46.405943+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: audit 2023-10-09T08:23:46.422701+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: audit 2023-10-09T08:23:46.428392+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:23:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:47 smithi016 bash[27454]: cluster 2023-10-09T08:23:47.209233+0000 mon.a (mon.0) 50 : cluster [DBG] mgrmap e7: y(active, since 1.01051s) 2023-10-09T08:23:47.939 INFO:teuthology.orchestra.run.smithi016.stderr:mgr epoch 5 is available 2023-10-09T08:23:47.939 INFO:teuthology.orchestra.run.smithi016.stderr:Setting orchestrator backend to cephadm... 2023-10-09T08:23:48.595 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:48 smithi016 bash[27454]: audit 2023-10-09T08:23:47.208799+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T08:23:48.595 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:48 smithi016 bash[27454]: audit 2023-10-09T08:23:47.224157+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T08:23:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:49 smithi016 bash[27454]: cluster 2023-10-09T08:23:48.264423+0000 mon.a (mon.0) 51 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-09T08:23:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:49 smithi016 bash[27454]: audit 2023-10-09T08:23:49.235804+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:23:50.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:50 smithi016 bash[27454]: audit 2023-10-09T08:23:49.225790+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:23:50.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:50 smithi016 bash[27454]: audit 2023-10-09T08:23:49.281512+0000 mon.a (mon.0) 53 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:23:51.186 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: value unchanged 2023-10-09T08:23:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:51 smithi016 bash[27454]: audit 2023-10-09T08:23:51.183707+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:23:51.800 INFO:teuthology.orchestra.run.smithi016.stderr:Generating ssh key... 2023-10-09T08:23:53.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: Generating public/private rsa key pair. 2023-10-09T08:23:53.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: Your identification has been saved in /tmp/tmpzcwyi0dy/key. 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: Your public key has been saved in /tmp/tmpzcwyi0dy/key.pub. 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: The key fingerprint is: 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: SHA256:vb7s7qvbZOfXydeq3KUzVTH64OdKJlC6Y5TB/d/0lr8 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: The key's randomart image is: 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: +---[RSA 3072]----+ 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | | 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | . . o | 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | o o . o| 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | .= .o .| 2023-10-09T08:23:53.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | S=. ..o o| 2023-10-09T08:23:53.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | . o. ..+=| 2023-10-09T08:23:53.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | ++..o+=B| 2023-10-09T08:23:53.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | .*.++.+B=| 2023-10-09T08:23:53.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: | o*Xo++=E+| 2023-10-09T08:23:53.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:53 smithi016 bash[27723]: +----[SHA256]-----+ 2023-10-09T08:23:54.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:54 smithi016 bash[27454]: audit 2023-10-09T08:23:53.148464+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:23:54.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:54 smithi016 bash[27454]: cephadm 2023-10-09T08:23:53.148733+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-10-09T08:23:54.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:54 smithi016 bash[27454]: audit 2023-10-09T08:23:53.235642+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:23:54.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:54 smithi016 bash[27454]: audit 2023-10-09T08:23:53.238634+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:23:55.174 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULNjOonlTEKj2CAxSbnys7MqBPNb8tUXtNqnvbJteFuFpCvsQ3pTdvrSa30lYgwm+zc+utcfOn/JnwgnANJ5UHB0XR9ZPWNCwvjsxfNgi/5cMWfyw4sQligtvq4KdQfGAmDiXpcVkLL3kFS8xbEMhYRpmcp8A+7v7IGJe5IF7KiJzWGv5n8QxQgKgfmPKa674dQK3i0Rej7lGwYdmdZnEPs8SSD0T5Hhssm2Lbb3YFC9lIWW375oFao48J57JNPmLUvBeIqfQ9X1zWLqx4zrz6IQXqfzi/MmIwy+vZR0kxveA7cuA5ZGVKSHDaHHzKnZHF3WmF5feVG0gzCvodsBZzUnExmmYZe/OSdC3x5QwigSn0q8hs74bTUEhW8GTw3NnBTublUlr2rifQGUeGwacCtB6TRl+iS37T7UtYErdgNb7Hz4D/5jf+zOsN3WsAVevQKNbkgIv50wppWj0nMsD030Rqy7tmmOcIXkkt60YaQ69W2py9JebC4+Qors9nTM= ceph-f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:23:55.793 INFO:teuthology.orchestra.run.smithi016.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T08:23:55.793 INFO:teuthology.orchestra.run.smithi016.stderr:Adding key to root@localhost authorized_keys... 2023-10-09T08:23:55.794 INFO:teuthology.orchestra.run.smithi016.stderr:Adding host smithi016... 2023-10-09T08:23:56.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:55 smithi016 bash[27454]: audit 2023-10-09T08:23:55.171667+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:23:57.481 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:23:57 smithi016 bash[27723]: Warning: Permanently added '172.21.15.16' (ECDSA) to the list of known hosts. 2023-10-09T08:23:57.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:57 smithi016 bash[27454]: audit 2023-10-09T08:23:57.042099+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi016", "addr": "172.21.15.16", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:23:57.918 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Added host 'smithi016' with addr '172.21.15.16' 2023-10-09T08:23:58.797 INFO:teuthology.orchestra.run.smithi016.stderr:Deploying unmanaged mon service... 2023-10-09T08:23:59.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:58 smithi016 bash[27454]: audit 2023-10-09T08:23:57.916155+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:23:59.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:58 smithi016 bash[27454]: cephadm 2023-10-09T08:23:57.916746+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi016 2023-10-09T08:23:59.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:23:58 smithi016 bash[27454]: audit 2023-10-09T08:23:57.925521+0000 mon.a (mon.0) 57 : audit [DBG] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:00.748 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-09T08:24:01.417 INFO:teuthology.orchestra.run.smithi016.stderr:Deploying unmanaged mgr service... 2023-10-09T08:24:01.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:01 smithi016 bash[27454]: audit 2023-10-09T08:24:00.738814+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:24:02.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:01 smithi016 bash[27454]: cephadm 2023-10-09T08:24:00.740833+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-09T08:24:02.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:01 smithi016 bash[27454]: audit 2023-10-09T08:24:00.745649+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:24:02.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:01 smithi016 bash[27454]: audit 2023-10-09T08:24:01.028365+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:24:02.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:01 smithi016 bash[27454]: audit 2023-10-09T08:24:01.294950+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:24:02.756 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-09T08:24:04.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:03 smithi016 bash[27454]: audit 2023-10-09T08:24:02.747861+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:24:04.071 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:03 smithi016 bash[27454]: cephadm 2023-10-09T08:24:02.748964+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-09T08:24:04.071 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:03 smithi016 bash[27454]: audit 2023-10-09T08:24:02.753811+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:24:05.991 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:05 smithi016 bash[27454]: audit 2023-10-09T08:24:04.699575+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.16:0/3377918642' entity='client.admin' 2023-10-09T08:24:07.301 INFO:teuthology.orchestra.run.smithi016.stderr:Enabling the dashboard module... 2023-10-09T08:24:07.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:07 smithi016 bash[27454]: cluster 2023-10-09T08:24:06.211774+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:07.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:07 smithi016 bash[27454]: audit 2023-10-09T08:24:06.365175+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.14116 172.21.15.16:0/1983948403' entity='mgr.y' 2023-10-09T08:24:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:07 smithi016 bash[27454]: audit 2023-10-09T08:24:06.615375+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.16:0/4009164043' entity='client.admin' 2023-10-09T08:24:09.629 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:09 smithi016 bash[27723]: ignoring --setuser ceph since I am not root 2023-10-09T08:24:09.629 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:09 smithi016 bash[27723]: ignoring --setgroup ceph since I am not root 2023-10-09T08:24:09.629 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:09 smithi016 bash[27723]: debug 2023-10-09T08:24:09.589+0000 7fecf2818680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:24:09.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:09 smithi016 bash[27454]: cluster 2023-10-09T08:24:08.212209+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:09.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:09 smithi016 bash[27454]: audit 2023-10-09T08:24:08.693494+0000 mon.a (mon.0) 65 : audit [INF] from='client.? 172.21.15.16:0/3634608983' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T08:24:09.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:09 smithi016 bash[27723]: debug 2023-10-09T08:24:09.693+0000 7fecf2818680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:24:10.372 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:10 smithi016 bash[27723]: debug 2023-10-09T08:24:10.169+0000 7fecf2818680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:24:10.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:10 smithi016 bash[27454]: audit 2023-10-09T08:24:09.375481+0000 mon.a (mon.0) 66 : audit [INF] from='client.? 172.21.15.16:0/3634608983' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T08:24:10.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:10 smithi016 bash[27454]: cluster 2023-10-09T08:24:09.375537+0000 mon.a (mon.0) 67 : cluster [DBG] mgrmap e9: y(active, since 23s) 2023-10-09T08:24:11.494 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:11 smithi016 bash[27723]: debug 2023-10-09T08:24:11.117+0000 7fecf2818680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:24:11.494 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:11 smithi016 bash[27723]: debug 2023-10-09T08:24:11.221+0000 7fecf2818680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:24:11.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-10-09T08:24:11.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-09T08:24:11.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": true, 2023-10-09T08:24:11.523 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T08:24:11.523 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T08:24:11.523 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-10-09T08:24:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:11 smithi016 bash[27454]: audit 2023-10-09T08:24:11.520361+0000 mon.a (mon.0) 68 : audit [DBG] from='client.? 172.21.15.16:0/4257034973' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T08:24:11.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:11 smithi016 bash[27723]: debug 2023-10-09T08:24:11.489+0000 7fecf2818680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:24:12.177 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for the mgr to restart... 2023-10-09T08:24:12.177 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr epoch 9... 2023-10-09T08:24:12.941 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:12 smithi016 bash[27723]: debug 2023-10-09T08:24:12.661+0000 7fecf2818680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:24:12.941 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:12 smithi016 bash[27723]: debug 2023-10-09T08:24:12.829+0000 7fecf2818680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:24:13.249 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:12 smithi016 bash[27723]: debug 2023-10-09T08:24:12.937+0000 7fecf2818680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:24:13.249 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:13 smithi016 bash[27723]: debug 2023-10-09T08:24:13.053+0000 7fecf2818680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:24:13.666 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:13 smithi016 bash[27723]: debug 2023-10-09T08:24:13.277+0000 7fecf2818680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:24:13.666 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:13 smithi016 bash[27723]: debug 2023-10-09T08:24:13.661+0000 7fecf2818680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:24:13.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:13 smithi016 bash[27723]: debug 2023-10-09T08:24:13.785+0000 7fecf2818680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:24:14.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:14 smithi016 bash[27723]: debug 2023-10-09T08:24:14.677+0000 7fecf2818680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:24:14.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:14 smithi016 bash[27723]: debug 2023-10-09T08:24:14.789+0000 7fecf2818680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:24:15.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:14 smithi016 bash[27723]: debug 2023-10-09T08:24:14.913+0000 7fecf2818680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:24:15.451 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:15 smithi016 bash[27723]: debug 2023-10-09T08:24:15.149+0000 7fecf2818680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:24:15.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:15 smithi016 bash[27723]: debug 2023-10-09T08:24:15.445+0000 7fecf2818680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:24:15.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:15 smithi016 bash[27723]: debug 2023-10-09T08:24:15.617+0000 7fecf2818680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:24:16.142 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:15 smithi016 bash[27723]: debug 2023-10-09T08:24:15.797+0000 7fecf2818680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:24:16.142 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27723]: debug 2023-10-09T08:24:16.021+0000 7fecf2818680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:24:16.392 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27723]: debug 2023-10-09T08:24:16.137+0000 7fecf2818680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:24:16.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: cluster 2023-10-09T08:24:16.162077+0000 mon.a (mon.0) 69 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:24:16.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: cluster 2023-10-09T08:24:16.162471+0000 mon.a (mon.0) 70 : cluster [INF] Activating manager daemon y 2023-10-09T08:24:16.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: cluster 2023-10-09T08:24:16.168409+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-09T08:24:16.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: cluster 2023-10-09T08:24:16.168641+0000 mon.a (mon.0) 72 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00632453s) 2023-10-09T08:24:16.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: audit 2023-10-09T08:24:16.171470+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:24:16.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: audit 2023-10-09T08:24:16.172848+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:24:16.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: audit 2023-10-09T08:24:16.173754+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:24:16.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: audit 2023-10-09T08:24:16.173968+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:24:16.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: audit 2023-10-09T08:24:16.174183+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:24:16.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27454]: cluster 2023-10-09T08:24:16.196096+0000 mon.a (mon.0) 78 : cluster [INF] Manager daemon y is now available 2023-10-09T08:24:16.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:16 smithi016 bash[27723]: Warning: Permanently added '172.21.15.16' (ECDSA) to the list of known hosts. 2023-10-09T08:24:17.188 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-10-09T08:24:17.189 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-09T08:24:17.189 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T08:24:17.189 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-10-09T08:24:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:17 smithi016 bash[27454]: audit 2023-10-09T08:24:16.340029+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:17 smithi016 bash[27454]: audit 2023-10-09T08:24:16.345742+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:17 smithi016 bash[27454]: audit 2023-10-09T08:24:16.386983+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:24:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:17 smithi016 bash[27454]: audit 2023-10-09T08:24:16.392461+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:24:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:17 smithi016 bash[27454]: cluster 2023-10-09T08:24:17.172249+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e11: y(active, since 1.00992s) 2023-10-09T08:24:17.824 INFO:teuthology.orchestra.run.smithi016.stderr:mgr epoch 9 is available 2023-10-09T08:24:17.825 INFO:teuthology.orchestra.run.smithi016.stderr:Generating a dashboard self-signed certificate... 2023-10-09T08:24:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:18 smithi016 bash[27454]: audit 2023-10-09T08:24:17.172188+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T08:24:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:18 smithi016 bash[27454]: audit 2023-10-09T08:24:17.187093+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T08:24:19.212 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-09T08:24:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:19 smithi016 bash[27454]: cluster 2023-10-09T08:24:18.221886+0000 mon.a (mon.0) 84 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-09T08:24:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:19 smithi016 bash[27454]: audit 2023-10-09T08:24:19.206143+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:19 smithi016 bash[27454]: audit 2023-10-09T08:24:19.210558+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:19.888 INFO:teuthology.orchestra.run.smithi016.stderr:Creating initial admin user... 2023-10-09T08:24:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:20 smithi016 bash[27454]: audit 2023-10-09T08:24:19.130950+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:24:21.515 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$plRs8n3P1TtfYMTcA7d4Tuic0TGYr81HVDSFwndoG4yScOoiChAgK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696839861, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T08:24:21.851 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:21 smithi016 bash[27454]: audit 2023-10-09T08:24:21.245156+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:24:22.118 INFO:teuthology.orchestra.run.smithi016.stderr:Fetching dashboard port number... 2023-10-09T08:24:22.873 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:22 smithi016 bash[27454]: audit 2023-10-09T08:24:21.508233+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:23.468 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 8443 2023-10-09T08:24:23.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:23 smithi016 bash[27454]: cluster 2023-10-09T08:24:22.513223+0000 mon.a (mon.0) 88 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-09T08:24:23.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:23 smithi016 bash[27454]: audit 2023-10-09T08:24:23.170784+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:23.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:23 smithi016 bash[27454]: audit 2023-10-09T08:24:23.466901+0000 mon.a (mon.0) 90 : audit [DBG] from='client.? 172.21.15.16:0/3645178404' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T08:24:24.200 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-10-09T08:24:24.200 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T08:24:24.204 INFO:teuthology.orchestra.run.smithi016.stderr:Ceph Dashboard is now available at: 2023-10-09T08:24:24.204 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:24.204 INFO:teuthology.orchestra.run.smithi016.stderr: URL: https://smithi016.front.sepia.ceph.com:8443/ 2023-10-09T08:24:24.204 INFO:teuthology.orchestra.run.smithi016.stderr: User: admin 2023-10-09T08:24:24.204 INFO:teuthology.orchestra.run.smithi016.stderr: Password: ltx1t71mj8 2023-10-09T08:24:24.205 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:24.205 INFO:teuthology.orchestra.run.smithi016.stderr:Enabling autotune for osd_memory_target 2023-10-09T08:24:27.840 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-09T08:24:28.913 INFO:teuthology.orchestra.run.smithi016.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr:Or, if you are only running a single cluster on this host: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: ceph telemetry on 2023-10-09T08:24:28.914 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.915 INFO:teuthology.orchestra.run.smithi016.stderr:For more information see: 2023-10-09T08:24:28.915 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.915 INFO:teuthology.orchestra.run.smithi016.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-09T08:24:28.915 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-10-09T08:24:28.915 INFO:teuthology.orchestra.run.smithi016.stderr:Bootstrap complete. 2023-10-09T08:24:28.946 INFO:tasks.cephadm:Fetching config... 2023-10-09T08:24:28.947 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:24:28.947 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T08:24:28.953 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T08:24:28.953 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:24:28.953 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T08:24:29.000 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T08:24:29.001 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:24:29.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T08:24:29.057 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T08:24:29.058 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:24:29.058 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T08:24:29.105 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T08:24:29.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULNjOonlTEKj2CAxSbnys7MqBPNb8tUXtNqnvbJteFuFpCvsQ3pTdvrSa30lYgwm+zc+utcfOn/JnwgnANJ5UHB0XR9ZPWNCwvjsxfNgi/5cMWfyw4sQligtvq4KdQfGAmDiXpcVkLL3kFS8xbEMhYRpmcp8A+7v7IGJe5IF7KiJzWGv5n8QxQgKgfmPKa674dQK3i0Rej7lGwYdmdZnEPs8SSD0T5Hhssm2Lbb3YFC9lIWW375oFao48J57JNPmLUvBeIqfQ9X1zWLqx4zrz6IQXqfzi/MmIwy+vZR0kxveA7cuA5ZGVKSHDaHHzKnZHF3WmF5feVG0gzCvodsBZzUnExmmYZe/OSdC3x5QwigSn0q8hs74bTUEhW8GTw3NnBTublUlr2rifQGUeGwacCtB6TRl+iS37T7UtYErdgNb7Hz4D/5jf+zOsN3WsAVevQKNbkgIv50wppWj0nMsD030Rqy7tmmOcIXkkt60YaQ69W2py9JebC4+Qors9nTM= ceph-f674ade6-667c-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T08:24:29.156 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:28 smithi016 bash[27454]: audit 2023-10-09T08:24:27.835659+0000 mon.a (mon.0) 91 : audit [INF] from='client.? 172.21.15.16:0/311967970' entity='client.admin' 2023-10-09T08:24:29.164 INFO:teuthology.orchestra.run.smithi016.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULNjOonlTEKj2CAxSbnys7MqBPNb8tUXtNqnvbJteFuFpCvsQ3pTdvrSa30lYgwm+zc+utcfOn/JnwgnANJ5UHB0XR9ZPWNCwvjsxfNgi/5cMWfyw4sQligtvq4KdQfGAmDiXpcVkLL3kFS8xbEMhYRpmcp8A+7v7IGJe5IF7KiJzWGv5n8QxQgKgfmPKa674dQK3i0Rej7lGwYdmdZnEPs8SSD0T5Hhssm2Lbb3YFC9lIWW375oFao48J57JNPmLUvBeIqfQ9X1zWLqx4zrz6IQXqfzi/MmIwy+vZR0kxveA7cuA5ZGVKSHDaHHzKnZHF3WmF5feVG0gzCvodsBZzUnExmmYZe/OSdC3x5QwigSn0q8hs74bTUEhW8GTw3NnBTublUlr2rifQGUeGwacCtB6TRl+iS37T7UtYErdgNb7Hz4D/5jf+zOsN3WsAVevQKNbkgIv50wppWj0nMsD030Rqy7tmmOcIXkkt60YaQ69W2py9JebC4+Qors9nTM= ceph-f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:24:29.176 DEBUG:teuthology.orchestra.run.smithi094:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULNjOonlTEKj2CAxSbnys7MqBPNb8tUXtNqnvbJteFuFpCvsQ3pTdvrSa30lYgwm+zc+utcfOn/JnwgnANJ5UHB0XR9ZPWNCwvjsxfNgi/5cMWfyw4sQligtvq4KdQfGAmDiXpcVkLL3kFS8xbEMhYRpmcp8A+7v7IGJe5IF7KiJzWGv5n8QxQgKgfmPKa674dQK3i0Rej7lGwYdmdZnEPs8SSD0T5Hhssm2Lbb3YFC9lIWW375oFao48J57JNPmLUvBeIqfQ9X1zWLqx4zrz6IQXqfzi/MmIwy+vZR0kxveA7cuA5ZGVKSHDaHHzKnZHF3WmF5feVG0gzCvodsBZzUnExmmYZe/OSdC3x5QwigSn0q8hs74bTUEhW8GTw3NnBTublUlr2rifQGUeGwacCtB6TRl+iS37T7UtYErdgNb7Hz4D/5jf+zOsN3WsAVevQKNbkgIv50wppWj0nMsD030Rqy7tmmOcIXkkt60YaQ69W2py9JebC4+Qors9nTM= ceph-f674ade6-667c-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T08:24:29.203 INFO:teuthology.orchestra.run.smithi094.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULNjOonlTEKj2CAxSbnys7MqBPNb8tUXtNqnvbJteFuFpCvsQ3pTdvrSa30lYgwm+zc+utcfOn/JnwgnANJ5UHB0XR9ZPWNCwvjsxfNgi/5cMWfyw4sQligtvq4KdQfGAmDiXpcVkLL3kFS8xbEMhYRpmcp8A+7v7IGJe5IF7KiJzWGv5n8QxQgKgfmPKa674dQK3i0Rej7lGwYdmdZnEPs8SSD0T5Hhssm2Lbb3YFC9lIWW375oFao48J57JNPmLUvBeIqfQ9X1zWLqx4zrz6IQXqfzi/MmIwy+vZR0kxveA7cuA5ZGVKSHDaHHzKnZHF3WmF5feVG0gzCvodsBZzUnExmmYZe/OSdC3x5QwigSn0q8hs74bTUEhW8GTw3NnBTublUlr2rifQGUeGwacCtB6TRl+iS37T7UtYErdgNb7Hz4D/5jf+zOsN3WsAVevQKNbkgIv50wppWj0nMsD030Rqy7tmmOcIXkkt60YaQ69W2py9JebC4+Qors9nTM= ceph-f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:24:29.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T08:24:33.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:32.647786+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:33.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:32.964402+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi016", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:24:33.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:32.969178+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:33.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:32.971920+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:32.976882+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:33.530799+0000 mon.a (mon.0) 97 : audit [INF] from='client.? 172.21.15.16:0/264435903' entity='client.admin' 2023-10-09T08:24:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:33.593885+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:33 smithi016 bash[27454]: audit 2023-10-09T08:24:33.599568+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:34.316 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T08:24:34.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T08:24:37.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:37 smithi016 bash[27454]: cluster 2023-10-09T08:24:36.175861+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:38.441 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi094 2023-10-09T08:24:38.441 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:24:38.441 DEBUG:teuthology.orchestra.run.smithi094:> dd of=/etc/ceph/ceph.conf 2023-10-09T08:24:38.448 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:24:38.449 DEBUG:teuthology.orchestra.run.smithi094:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:24:38.499 INFO:tasks.cephadm:Adding host smithi094 to orchestrator... 2023-10-09T08:24:38.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi094 2023-10-09T08:24:38.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: audit 2023-10-09T08:24:37.746697+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: audit 2023-10-09T08:24:37.775325+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: audit 2023-10-09T08:24:37.777888+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:24:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: audit 2023-10-09T08:24:37.779180+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:24:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: cephadm 2023-10-09T08:24:37.781317+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:24:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: cephadm 2023-10-09T08:24:37.802987+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:24:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: audit 2023-10-09T08:24:37.846382+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:39.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: audit 2023-10-09T08:24:37.849963+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:39.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:38 smithi016 bash[27454]: cluster 2023-10-09T08:24:38.176259+0000 mgr.y (mgr.14146) 9 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:41.229 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:40 smithi016 bash[27454]: cluster 2023-10-09T08:24:40.176695+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:41.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:41 smithi016 bash[27723]: Warning: Permanently added 'smithi094,172.21.15.94' (ECDSA) to the list of known hosts. 2023-10-09T08:24:42.439 INFO:teuthology.orchestra.run.smithi016.stdout:Added host 'smithi094' with addr '172.21.15.94' 2023-10-09T08:24:42.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:24:42 smithi016 bash[27723]: Warning: Permanently added '172.21.15.94' (ECDSA) to the list of known hosts. 2023-10-09T08:24:43.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T08:24:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:43 smithi016 bash[27454]: audit 2023-10-09T08:24:41.596297+0000 mgr.y (mgr.14146) 11 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi094", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:24:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:43 smithi016 bash[27454]: cluster 2023-10-09T08:24:42.177293+0000 mgr.y (mgr.14146) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:43 smithi016 bash[27454]: audit 2023-10-09T08:24:42.437739+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:43 smithi016 bash[27454]: cephadm 2023-10-09T08:24:42.438636+0000 mgr.y (mgr.14146) 13 : cephadm [INF] Added host smithi094 2023-10-09T08:24:43.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:43 smithi016 bash[27454]: audit 2023-10-09T08:24:42.482678+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:43.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:43 smithi016 bash[27454]: audit 2023-10-09T08:24:43.040785+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:45.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:45 smithi016 bash[27454]: cluster 2023-10-09T08:24:44.177840+0000 mgr.y (mgr.14146) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:45.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:45 smithi016 bash[27454]: audit 2023-10-09T08:24:44.301465+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:46.327 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:24:46.327 INFO:teuthology.orchestra.run.smithi016.stdout:[{"addr": "172.21.15.16", "hostname": "smithi016", "labels": [], "status": ""}, {"addr": "172.21.15.94", "hostname": "smithi094", "labels": [], "status": ""}] 2023-10-09T08:24:47.006 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T08:24:47.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T08:24:47.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:47 smithi016 bash[27454]: cluster 2023-10-09T08:24:46.178271+0000 mgr.y (mgr.14146) 15 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:47.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:47 smithi016 bash[27454]: audit 2023-10-09T08:24:46.324892+0000 mgr.y (mgr.14146) 16 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:24:49.704 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:49 smithi016 bash[27454]: cluster 2023-10-09T08:24:48.178751+0000 mgr.y (mgr.14146) 17 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:51.143 INFO:teuthology.orchestra.run.smithi016.stderr:adjusted tunables profile to default 2023-10-09T08:24:51.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.131396+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:51.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.145468+0000 mon.a (mon.0) 111 : audit [INF] from='client.? 172.21.15.16:0/1378944410' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T08:24:51.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: cluster 2023-10-09T08:24:50.179163+0000 mgr.y (mgr.14146) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:51.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.448571+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.450194+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi094", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:24:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.452159+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:24:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.453342+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:24:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.507926+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:51 smithi016 bash[27454]: audit 2023-10-09T08:24:50.511575+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:51.847 INFO:tasks.cephadm:Adding mon.a on smithi016 2023-10-09T08:24:51.847 INFO:tasks.cephadm:Adding mon.c on smithi016 2023-10-09T08:24:51.847 INFO:tasks.cephadm:Adding mon.b on smithi094 2023-10-09T08:24:51.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi016:172.21.15.16=a;smithi016:[v2:172.21.15.16:3301,v1:172.21.15.16:6790]=c;smithi094:172.21.15.94=b' 2023-10-09T08:24:52.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:52 smithi016 bash[27454]: cephadm 2023-10-09T08:24:50.455174+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:24:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:52 smithi016 bash[27454]: cephadm 2023-10-09T08:24:50.477721+0000 mgr.y (mgr.14146) 20 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:24:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:52 smithi016 bash[27454]: audit 2023-10-09T08:24:51.137864+0000 mon.a (mon.0) 118 : audit [INF] from='client.? 172.21.15.16:0/1378944410' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T08:24:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:52 smithi016 bash[27454]: cluster 2023-10-09T08:24:51.137991+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:24:53.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:53 smithi016 bash[27454]: cluster 2023-10-09T08:24:52.179627+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:55.133 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled mon update... 2023-10-09T08:24:55.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: cluster 2023-10-09T08:24:54.180062+0000 mgr.y (mgr.14146) 22 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:55.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: audit 2023-10-09T08:24:55.131138+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:55.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: audit 2023-10-09T08:24:55.139716+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:24:55.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: audit 2023-10-09T08:24:55.143173+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:24:55.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: audit 2023-10-09T08:24:55.144620+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:24:55.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: audit 2023-10-09T08:24:55.150407+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:24:55.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: audit 2023-10-09T08:24:55.153575+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:24:55.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:55 smithi016 bash[27454]: audit 2023-10-09T08:24:55.154871+0000 mon.a (mon.0) 126 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:24:56.120 DEBUG:teuthology.orchestra.run.smithi016:mon.c> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T08:24:56.123 DEBUG:teuthology.orchestra.run.smithi094:mon.b> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T08:24:56.128 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T08:24:56.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T08:24:56.473 INFO:journalctl@ceph.mon.c.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:24:56.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:56 smithi016 bash[27454]: audit 2023-10-09T08:24:55.120483+0000 mgr.y (mgr.14146) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi016:172.21.15.16=a;smithi016:[v2:172.21.15.16:3301,v1:172.21.15.16:6790]=c;smithi094:172.21.15.94=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:24:56.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:56 smithi016 bash[27454]: cephadm 2023-10-09T08:24:55.126588+0000 mgr.y (mgr.14146) 24 : cephadm [INF] Saving service mon spec with placement smithi016:172.21.15.16=a;smithi016:[v2:172.21.15.16:3301,v1:172.21.15.16:6790]=c;smithi094:172.21.15.94=b;count:3 2023-10-09T08:24:56.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:56 smithi016 bash[27454]: cephadm 2023-10-09T08:24:55.156059+0000 mgr.y (mgr.14146) 25 : cephadm [INF] Deploying daemon mon.b on smithi094 2023-10-09T08:24:57.425 INFO:journalctl@ceph.mon.b.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:24:57.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:57 smithi016 bash[27454]: cluster 2023-10-09T08:24:56.180330+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:24:59.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:24:59 smithi016 bash[27454]: cluster 2023-10-09T08:24:58.180724+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:01.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:00 smithi016 bash[27454]: cluster 2023-10-09T08:25:00.181130+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:01.558 INFO:teuthology.orchestra.run.smithi094.stdout: 2023-10-09T08:25:01.558 INFO:teuthology.orchestra.run.smithi094.stdout:{"epoch":1,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","modified":"2023-10-09T08:23:14.300421Z","created":"2023-10-09T08:23:14.300421Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:3300","nonce":0},{"type":"v1","addr":"172.21.15.16:6789","nonce":0}]},"addr":"172.21.15.16:6789/0","public_addr":"172.21.15.16:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T08:25:01.563 INFO:teuthology.orchestra.run.smithi094.stderr:dumped monmap epoch 1 2023-10-09T08:25:02.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:01 smithi016 bash[27454]: audit 2023-10-09T08:25:01.556988+0000 mon.a (mon.0) 127 : audit [DBG] from='client.? 172.21.15.94:0/3019412863' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:25:03.485 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T08:25:03.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T08:25:03.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:03 smithi094 bash[27803]: debug 2023-10-09T08:25:03.373+0000 7fb4ee113700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T08:25:03.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:03 smithi016 bash[27454]: cluster 2023-10-09T08:25:02.181584+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:03.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:03 smithi016 bash[27454]: audit 2023-10-09T08:25:02.182069+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:03.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:03 smithi016 bash[27454]: audit 2023-10-09T08:25:02.184047+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:03.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:03 smithi016 bash[27454]: audit 2023-10-09T08:25:02.185759+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:03.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:03 smithi016 bash[27454]: cephadm 2023-10-09T08:25:02.187246+0000 mgr.y (mgr.14146) 30 : cephadm [INF] Deploying daemon mon.c on smithi016 2023-10-09T08:25:07.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:07 smithi016 bash[33851]: debug 2023-10-09T08:25:07.133+0000 7fa9b103a700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T08:25:08.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:03.388430+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:03.388863+0000 mon.a (mon.0) 133 : cluster [INF] mon.a calling monitor election 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:03.412873+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:04.182142+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:04.385158+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:05.385215+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:05.386668+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:06.182715+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:06.385642+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:07.142805+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:07.385788+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:08.143298+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:08.183194+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:08.385784+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:08.434407+0000 mon.a (mon.0) 142 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:08.440272+0000 mon.a (mon.0) 143 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0]} removed_ranks: {} 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:08.460825+0000 mon.a (mon.0) 144 : cluster [DBG] fsmap 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:08.460905+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:08.461275+0000 mon.a (mon.0) 146 : cluster [DBG] mgrmap e13: y(active, since 52s) 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: cluster 2023-10-09T08:25:08.467693+0000 mon.a (mon.0) 147 : cluster [INF] overall HEALTH_OK 2023-10-09T08:25:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:08.472360+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:08.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:08 smithi094 bash[27803]: audit 2023-10-09T08:25:08.474241+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:08.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:03.388430+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:25:08.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:03.388863+0000 mon.a (mon.0) 133 : cluster [INF] mon.a calling monitor election 2023-10-09T08:25:08.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:03.412873+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:08.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:04.182142+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:08.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:04.385158+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:05.385215+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:05.386668+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:06.182715+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:06.385642+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:07.142805+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:07.385788+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:08.143298+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:08.183194+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:08.385784+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:08.434407+0000 mon.a (mon.0) 142 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:25:09.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:08.440272+0000 mon.a (mon.0) 143 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0]} removed_ranks: {} 2023-10-09T08:25:09.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:08.460825+0000 mon.a (mon.0) 144 : cluster [DBG] fsmap 2023-10-09T08:25:09.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:08.460905+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:25:09.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:08.461275+0000 mon.a (mon.0) 146 : cluster [DBG] mgrmap e13: y(active, since 52s) 2023-10-09T08:25:09.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: cluster 2023-10-09T08:25:08.467693+0000 mon.a (mon.0) 147 : cluster [INF] overall HEALTH_OK 2023-10-09T08:25:09.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:08.472360+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:09.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:08 smithi016 bash[27454]: audit 2023-10-09T08:25:08.474241+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:09.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:25:09 smithi016 bash[27723]: debug 2023-10-09T08:25:09.381+0000 7fecae01d700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T08:25:14.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:09.150191+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:25:14.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:09.150577+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:09.150861+0000 mon.a (mon.0) 153 : cluster [INF] mon.a calling monitor election 2023-10-09T08:25:14.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:09.151634+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T08:25:14.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:09.169432+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:10.143679+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:10.183611+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:11.144279+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:11.146617+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:12.143860+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:12.184054+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:13.144168+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:14.144279+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:14.191604+0000 mon.a (mon.0) 160 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:14.197723+0000 mon.a (mon.0) 161 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:25:14.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:14.216869+0000 mon.a (mon.0) 162 : cluster [DBG] fsmap 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:14.216976+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:14.217399+0000 mon.a (mon.0) 164 : cluster [DBG] mgrmap e13: y(active, since 58s) 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: cluster 2023-10-09T08:25:14.227191+0000 mon.a (mon.0) 165 : cluster [INF] overall HEALTH_OK 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:14.230319+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:14.233627+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:14.237265+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:14.238416+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:14.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[27454]: audit 2023-10-09T08:25:14.239685+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:03.388430+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:03.388863+0000 mon.a (mon.0) 133 : cluster [INF] mon.a calling monitor election 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:03.412873+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:04.182142+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:04.385158+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:05.385215+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:05.386668+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:06.182715+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:06.385642+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:07.142805+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:07.385788+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:08.143298+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:08.183194+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:08.385784+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:08.434407+0000 mon.a (mon.0) 142 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:08.440272+0000 mon.a (mon.0) 143 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0]} removed_ranks: {} 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:08.460825+0000 mon.a (mon.0) 144 : cluster [DBG] fsmap 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:08.460905+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:08.461275+0000 mon.a (mon.0) 146 : cluster [DBG] mgrmap e13: y(active, since 52s) 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:08.467693+0000 mon.a (mon.0) 147 : cluster [INF] overall HEALTH_OK 2023-10-09T08:25:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:08.472360+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:08.474241+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:09.150191+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:09.150577+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:09.150861+0000 mon.a (mon.0) 153 : cluster [INF] mon.a calling monitor election 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:09.151634+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:09.169432+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:10.143679+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:10.183611+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:11.144279+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:11.146617+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:12.143860+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:12.184054+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:13.144168+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:14.144279+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:14.191604+0000 mon.a (mon.0) 160 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:14.197723+0000 mon.a (mon.0) 161 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:14.216869+0000 mon.a (mon.0) 162 : cluster [DBG] fsmap 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:14.216976+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:14.217399+0000 mon.a (mon.0) 164 : cluster [DBG] mgrmap e13: y(active, since 58s) 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: cluster 2023-10-09T08:25:14.227191+0000 mon.a (mon.0) 165 : cluster [INF] overall HEALTH_OK 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:14.230319+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:14.233627+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:14.237265+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:14.238416+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:14.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:14 smithi016 bash[33851]: audit 2023-10-09T08:25:14.239685+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:14.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:09.150191+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:25:14.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:09.150577+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:25:14.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:09.150861+0000 mon.a (mon.0) 153 : cluster [INF] mon.a calling monitor election 2023-10-09T08:25:14.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:09.151634+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T08:25:14.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:09.169432+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:10.143679+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:10.183611+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:11.144279+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:11.146617+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:12.143860+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:12.184054+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:13.144168+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:14.144279+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:14.191604+0000 mon.a (mon.0) 160 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:25:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:14.197723+0000 mon.a (mon.0) 161 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:14.216869+0000 mon.a (mon.0) 162 : cluster [DBG] fsmap 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:14.216976+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:14.217399+0000 mon.a (mon.0) 164 : cluster [DBG] mgrmap e13: y(active, since 58s) 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: cluster 2023-10-09T08:25:14.227191+0000 mon.a (mon.0) 165 : cluster [INF] overall HEALTH_OK 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:14.230319+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:14.233627+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:14.237265+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:14.238416+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:14 smithi094 bash[27803]: audit 2023-10-09T08:25:14.239685+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:15.625 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: cluster 2023-10-09T08:25:14.184438+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: cephadm 2023-10-09T08:25:14.241326+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: cephadm 2023-10-09T08:25:14.241806+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: audit 2023-10-09T08:25:14.270002+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: audit 2023-10-09T08:25:14.274300+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: audit 2023-10-09T08:25:14.285993+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: cephadm 2023-10-09T08:25:14.287865+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: audit 2023-10-09T08:25:14.288250+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: audit 2023-10-09T08:25:14.289248+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: audit 2023-10-09T08:25:14.290168+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:15.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: cephadm 2023-10-09T08:25:14.291016+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring daemon mon.a on smithi016 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[33851]: audit 2023-10-09T08:25:15.143721+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: cluster 2023-10-09T08:25:14.184438+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: cephadm 2023-10-09T08:25:14.241326+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: cephadm 2023-10-09T08:25:14.241806+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: audit 2023-10-09T08:25:14.270002+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: audit 2023-10-09T08:25:14.274300+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: audit 2023-10-09T08:25:14.285993+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: cephadm 2023-10-09T08:25:14.287865+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T08:25:15.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: audit 2023-10-09T08:25:14.288250+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:15.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: audit 2023-10-09T08:25:14.289248+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:15.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: audit 2023-10-09T08:25:14.290168+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:15.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: cephadm 2023-10-09T08:25:14.291016+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring daemon mon.a on smithi016 2023-10-09T08:25:15.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:15 smithi016 bash[27454]: audit 2023-10-09T08:25:15.143721+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: cluster 2023-10-09T08:25:14.184438+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: cephadm 2023-10-09T08:25:14.241326+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: cephadm 2023-10-09T08:25:14.241806+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: audit 2023-10-09T08:25:14.270002+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: audit 2023-10-09T08:25:14.274300+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: audit 2023-10-09T08:25:14.285993+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: cephadm 2023-10-09T08:25:14.287865+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T08:25:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: audit 2023-10-09T08:25:14.288250+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: audit 2023-10-09T08:25:14.289248+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: audit 2023-10-09T08:25:14.290168+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: cephadm 2023-10-09T08:25:14.291016+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring daemon mon.a on smithi016 2023-10-09T08:25:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:15 smithi094 bash[27803]: audit 2023-10-09T08:25:15.143721+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:25:16.026 INFO:teuthology.orchestra.run.smithi094.stdout: 2023-10-09T08:25:16.026 INFO:teuthology.orchestra.run.smithi094.stdout:{"epoch":3,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","modified":"2023-10-09T08:25:09.144664Z","created":"2023-10-09T08:23:14.300421Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:3300","nonce":0},{"type":"v1","addr":"172.21.15.16:6789","nonce":0}]},"addr":"172.21.15.16:6789/0","public_addr":"172.21.15.16:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:3300","nonce":0},{"type":"v1","addr":"172.21.15.94:6789","nonce":0}]},"addr":"172.21.15.94:6789/0","public_addr":"172.21.15.94:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:3301","nonce":0},{"type":"v1","addr":"172.21.15.16:6790","nonce":0}]},"addr":"172.21.15.16:6790/0","public_addr":"172.21.15.16:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T08:25:16.030 INFO:teuthology.orchestra.run.smithi094.stderr:dumped monmap epoch 3 2023-10-09T08:25:16.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:16 smithi094 bash[27803]: audit 2023-10-09T08:25:16.025305+0000 mon.a (mon.0) 178 : audit [DBG] from='client.? 172.21.15.94:0/2479632844' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:25:16.542 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:16 smithi016 bash[33851]: audit 2023-10-09T08:25:16.025305+0000 mon.a (mon.0) 178 : audit [DBG] from='client.? 172.21.15.94:0/2479632844' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:25:16.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:16 smithi016 bash[27454]: audit 2023-10-09T08:25:16.025305+0000 mon.a (mon.0) 178 : audit [DBG] from='client.? 172.21.15.94:0/2479632844' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:25:16.915 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T08:25:16.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T08:25:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[27454]: cluster 2023-10-09T08:25:16.184840+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[27454]: audit 2023-10-09T08:25:16.323118+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[27454]: cephadm 2023-10-09T08:25:16.324093+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[27454]: audit 2023-10-09T08:25:16.324480+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[27454]: audit 2023-10-09T08:25:16.325510+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[27454]: audit 2023-10-09T08:25:16.326549+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[27454]: cephadm 2023-10-09T08:25:16.327571+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring daemon mon.c on smithi016 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[33851]: cluster 2023-10-09T08:25:16.184840+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[33851]: audit 2023-10-09T08:25:16.323118+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:17.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[33851]: cephadm 2023-10-09T08:25:16.324093+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T08:25:17.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[33851]: audit 2023-10-09T08:25:16.324480+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:17.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[33851]: audit 2023-10-09T08:25:16.325510+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:17.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[33851]: audit 2023-10-09T08:25:16.326549+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:17.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:17 smithi016 bash[33851]: cephadm 2023-10-09T08:25:16.327571+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring daemon mon.c on smithi016 2023-10-09T08:25:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:17 smithi094 bash[27803]: cluster 2023-10-09T08:25:16.184840+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:17 smithi094 bash[27803]: audit 2023-10-09T08:25:16.323118+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:17 smithi094 bash[27803]: cephadm 2023-10-09T08:25:16.324093+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T08:25:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:17 smithi094 bash[27803]: audit 2023-10-09T08:25:16.324480+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:17 smithi094 bash[27803]: audit 2023-10-09T08:25:16.325510+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:17 smithi094 bash[27803]: audit 2023-10-09T08:25:16.326549+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:17 smithi094 bash[27803]: cephadm 2023-10-09T08:25:16.327571+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring daemon mon.c on smithi016 2023-10-09T08:25:19.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:19 smithi094 bash[27803]: cluster 2023-10-09T08:25:18.185362+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:19 smithi094 bash[27803]: audit 2023-10-09T08:25:18.851844+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:19 smithi094 bash[27803]: audit 2023-10-09T08:25:18.853205+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:19 smithi094 bash[27803]: audit 2023-10-09T08:25:18.854136+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:19 smithi094 bash[27803]: audit 2023-10-09T08:25:18.855082+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[27454]: cluster 2023-10-09T08:25:18.185362+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[27454]: audit 2023-10-09T08:25:18.851844+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[27454]: audit 2023-10-09T08:25:18.853205+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[27454]: audit 2023-10-09T08:25:18.854136+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[27454]: audit 2023-10-09T08:25:18.855082+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:19.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[33851]: cluster 2023-10-09T08:25:18.185362+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:19.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[33851]: audit 2023-10-09T08:25:18.851844+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:19.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[33851]: audit 2023-10-09T08:25:18.853205+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:25:19.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[33851]: audit 2023-10-09T08:25:18.854136+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:25:19.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:19 smithi016 bash[33851]: audit 2023-10-09T08:25:18.855082+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:20 smithi094 bash[27803]: cephadm 2023-10-09T08:25:18.852878+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T08:25:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:20 smithi094 bash[27803]: cephadm 2023-10-09T08:25:18.855981+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mon.b on smithi094 2023-10-09T08:25:20.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:20 smithi016 bash[27454]: cephadm 2023-10-09T08:25:18.852878+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T08:25:20.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:20 smithi016 bash[27454]: cephadm 2023-10-09T08:25:18.855981+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mon.b on smithi094 2023-10-09T08:25:20.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:20 smithi016 bash[33851]: cephadm 2023-10-09T08:25:18.852878+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T08:25:20.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:20 smithi016 bash[33851]: cephadm 2023-10-09T08:25:18.855981+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mon.b on smithi094 2023-10-09T08:25:20.578 INFO:teuthology.orchestra.run.smithi016.stdout:# minimal ceph.conf for f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:25:20.578 INFO:teuthology.orchestra.run.smithi016.stdout:[global] 2023-10-09T08:25:20.578 INFO:teuthology.orchestra.run.smithi016.stdout: fsid = f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:25:20.578 INFO:teuthology.orchestra.run.smithi016.stdout: mon_host = [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] 2023-10-09T08:25:21.260 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T08:25:21.260 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:25:21.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T08:25:21.278 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:25:21.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:25:21.339 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:25:21.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T08:25:21.354 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:25:21.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:25:21.410 INFO:tasks.cephadm:Adding mgr.y on smithi016 2023-10-09T08:25:21.410 INFO:tasks.cephadm:Adding mgr.x on smithi094 2023-10-09T08:25:21.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi016=y;smithi094=x' 2023-10-09T08:25:21.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:21 smithi094 bash[27803]: cluster 2023-10-09T08:25:20.185821+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:21.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:21 smithi094 bash[27803]: audit 2023-10-09T08:25:20.577751+0000 mon.a (mon.0) 187 : audit [DBG] from='client.? 172.21.15.16:0/2050864202' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:21.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:21 smithi094 bash[27803]: audit 2023-10-09T08:25:21.008653+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:21.647 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:21 smithi094 bash[27803]: audit 2023-10-09T08:25:21.010733+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:21.647 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:21 smithi094 bash[27803]: audit 2023-10-09T08:25:21.013350+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:21.647 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:21 smithi094 bash[27803]: audit 2023-10-09T08:25:21.014576+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:21.647 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:21 smithi094 bash[27803]: audit 2023-10-09T08:25:21.021492+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[27454]: cluster 2023-10-09T08:25:20.185821+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[27454]: audit 2023-10-09T08:25:20.577751+0000 mon.a (mon.0) 187 : audit [DBG] from='client.? 172.21.15.16:0/2050864202' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[27454]: audit 2023-10-09T08:25:21.008653+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[27454]: audit 2023-10-09T08:25:21.010733+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[27454]: audit 2023-10-09T08:25:21.013350+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[27454]: audit 2023-10-09T08:25:21.014576+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[27454]: audit 2023-10-09T08:25:21.021492+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[33851]: cluster 2023-10-09T08:25:20.185821+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[33851]: audit 2023-10-09T08:25:20.577751+0000 mon.a (mon.0) 187 : audit [DBG] from='client.? 172.21.15.16:0/2050864202' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[33851]: audit 2023-10-09T08:25:21.008653+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[33851]: audit 2023-10-09T08:25:21.010733+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[33851]: audit 2023-10-09T08:25:21.013350+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[33851]: audit 2023-10-09T08:25:21.014576+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:21 smithi016 bash[33851]: audit 2023-10-09T08:25:21.021492+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:23.605 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:23 smithi094 bash[27803]: cluster 2023-10-09T08:25:22.186332+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:23.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:23 smithi016 bash[27454]: cluster 2023-10-09T08:25:22.186332+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:23.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:23 smithi016 bash[33851]: cluster 2023-10-09T08:25:22.186332+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:24.901 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled mgr update... 2023-10-09T08:25:25.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: cluster 2023-10-09T08:25:24.186756+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.900253+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.949436+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.953128+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.954995+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.962855+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.964843+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.968895+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.970950+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:25:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:25 smithi094 bash[27803]: audit 2023-10-09T08:25:24.974494+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: cluster 2023-10-09T08:25:24.186756+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.900253+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.949436+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.953128+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.954995+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.962855+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.964843+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:25:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.968895+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.970950+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[27454]: audit 2023-10-09T08:25:24.974494+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: cluster 2023-10-09T08:25:24.186756+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.900253+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.949436+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.953128+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.954995+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.962855+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:25.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.964843+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:25:25.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.968895+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T08:25:25.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.970950+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:25:25.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:25 smithi016 bash[33851]: audit 2023-10-09T08:25:24.974494+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:26.047 DEBUG:teuthology.orchestra.run.smithi094:mgr.x> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T08:25:26.049 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T08:25:26.049 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:25:26.049 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T08:25:26.057 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T08:25:26.057 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_1 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 492 Links: 1 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-10-09 08:18:42.819123286 +0000 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-10-09 08:18:42.495130757 +0000 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-10-09 08:18:42.495130757 +0000 2023-10-09T08:25:26.107 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-10-09T08:25:26.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T08:25:26.162 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-10-09T08:25:26.162 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-10-09T08:25:26.162 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000299437 s, 1.7 MB/s 2023-10-09T08:25:26.162 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T08:25:26.212 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_2 2023-10-09T08:25:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T08:25:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:26.263 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 497 Links: 1 2023-10-09T08:25:26.263 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:26.263 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-10-09 08:18:43.303112124 +0000 2023-10-09T08:25:26.263 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-10-09 08:18:43.015118765 +0000 2023-10-09T08:25:26.263 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-10-09 08:18:43.015118765 +0000 2023-10-09T08:25:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-10-09T08:25:26.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T08:25:26.320 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-10-09T08:25:26.320 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-10-09T08:25:26.320 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000308166 s, 1.7 MB/s 2023-10-09T08:25:26.321 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T08:25:26.373 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_3 2023-10-09T08:25:26.422 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T08:25:26.423 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:26.423 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 502 Links: 1 2023-10-09T08:25:26.423 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:26.423 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-10-09 08:18:43.815100317 +0000 2023-10-09T08:25:26.423 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-10-09 08:18:43.503107512 +0000 2023-10-09T08:25:26.423 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-10-09 08:18:43.503107512 +0000 2023-10-09T08:25:26.423 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-10-09T08:25:26.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T08:25:26.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:26 smithi016 bash[33851]: audit 2023-10-09T08:25:24.889920+0000 mgr.y (mgr.14146) 50 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi016=y;smithi094=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:25:26.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:26 smithi016 bash[33851]: cephadm 2023-10-09T08:25:24.893471+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Saving service mgr spec with placement smithi016=y;smithi094=x;count:2 2023-10-09T08:25:26.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:26 smithi016 bash[33851]: cephadm 2023-10-09T08:25:24.975607+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:25:26.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:26 smithi016 bash[27454]: audit 2023-10-09T08:25:24.889920+0000 mgr.y (mgr.14146) 50 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi016=y;smithi094=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:25:26.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:26 smithi016 bash[27454]: cephadm 2023-10-09T08:25:24.893471+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Saving service mgr spec with placement smithi016=y;smithi094=x;count:2 2023-10-09T08:25:26.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:26 smithi016 bash[27454]: cephadm 2023-10-09T08:25:24.975607+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:25:26.478 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-10-09T08:25:26.478 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-10-09T08:25:26.478 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000324296 s, 1.6 MB/s 2023-10-09T08:25:26.479 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T08:25:26.532 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_4 2023-10-09T08:25:26.582 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T08:25:26.583 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:26.583 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-10-09T08:25:26.583 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:26.583 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-10-09 08:18:44.027095428 +0000 2023-10-09T08:25:26.583 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-10-09 08:18:44.023095520 +0000 2023-10-09T08:25:26.583 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-10-09 08:18:44.023095520 +0000 2023-10-09T08:25:26.583 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-10-09T08:25:26.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T08:25:26.600 INFO:journalctl@ceph.mgr.x.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:25:26.636 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-10-09T08:25:26.636 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-10-09T08:25:26.636 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000271283 s, 1.9 MB/s 2023-10-09T08:25:26.637 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T08:25:26.692 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:25:26.692 DEBUG:teuthology.orchestra.run.smithi094:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T08:25:26.697 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T08:25:26.698 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_1 2023-10-09T08:25:26.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:26 smithi094 bash[27803]: audit 2023-10-09T08:25:24.889920+0000 mgr.y (mgr.14146) 50 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi016=y;smithi094=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:25:26.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:26 smithi094 bash[27803]: cephadm 2023-10-09T08:25:24.893471+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Saving service mgr spec with placement smithi016=y;smithi094=x;count:2 2023-10-09T08:25:26.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:26 smithi094 bash[27803]: cephadm 2023-10-09T08:25:24.975607+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:25:26.742 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T08:25:26.742 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:26.742 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 492 Links: 1 2023-10-09T08:25:26.742 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:26.742 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2023-10-09 08:18:43.413024397 +0000 2023-10-09T08:25:26.742 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2023-10-09 08:18:43.081032052 +0000 2023-10-09T08:25:26.743 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2023-10-09 08:18:43.081032052 +0000 2023-10-09T08:25:26.743 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: - 2023-10-09T08:25:26.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T08:25:26.798 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2023-10-09T08:25:26.798 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2023-10-09T08:25:26.798 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 0.000219355 s, 2.3 MB/s 2023-10-09T08:25:26.799 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T08:25:26.848 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_2 2023-10-09T08:25:26.899 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T08:25:26.899 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:26.899 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 496 Links: 1 2023-10-09T08:25:26.899 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:26.899 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2023-10-09 08:18:43.941012220 +0000 2023-10-09T08:25:26.899 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2023-10-09 08:18:43.629019415 +0000 2023-10-09T08:25:26.899 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2023-10-09 08:18:43.629019415 +0000 2023-10-09T08:25:26.900 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: - 2023-10-09T08:25:26.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T08:25:26.954 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2023-10-09T08:25:26.954 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2023-10-09T08:25:26.955 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 0.000297656 s, 1.7 MB/s 2023-10-09T08:25:26.956 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T08:25:27.008 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_3 2023-10-09T08:25:27.057 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T08:25:27.057 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:27.057 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 503 Links: 1 2023-10-09T08:25:27.058 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:27.058 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2023-10-09 08:18:44.476999860 +0000 2023-10-09T08:25:27.058 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2023-10-09 08:18:44.153007331 +0000 2023-10-09T08:25:27.058 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2023-10-09 08:18:44.153007331 +0000 2023-10-09T08:25:27.058 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: - 2023-10-09T08:25:27.059 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T08:25:27.113 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2023-10-09T08:25:27.114 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2023-10-09T08:25:27.114 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 0.000138968 s, 3.7 MB/s 2023-10-09T08:25:27.115 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T08:25:27.165 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_4 2023-10-09T08:25:27.215 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T08:25:27.215 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:25:27.215 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 510 Links: 1 2023-10-09T08:25:27.215 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:25:27.215 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2023-10-09 08:18:44.688994970 +0000 2023-10-09T08:25:27.215 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2023-10-09 08:18:44.684995063 +0000 2023-10-09T08:25:27.216 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2023-10-09 08:18:44.684995063 +0000 2023-10-09T08:25:27.216 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: - 2023-10-09T08:25:27.216 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T08:25:27.267 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2023-10-09T08:25:27.268 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2023-10-09T08:25:27.268 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 0.000131253 s, 3.9 MB/s 2023-10-09T08:25:27.269 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T08:25:27.320 INFO:tasks.cephadm:Deploying osd.0 on smithi016 with /dev/vg_nvme/lv_4... 2023-10-09T08:25:27.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T08:25:27.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:27 smithi094 bash[27803]: cluster 2023-10-09T08:25:26.187174+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:27.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:27 smithi016 bash[27454]: cluster 2023-10-09T08:25:26.187174+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:27 smithi016 bash[33851]: cluster 2023-10-09T08:25:26.187174+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:29.241 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:29 smithi094 bash[28656]: debug 2023-10-09T08:25:29.077+0000 7f187d52b700 1 -- 172.21.15.94:0/3947503027 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e31284c4e0 con 0x55e312940c00 2023-10-09T08:25:29.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:29 smithi094 bash[27803]: audit 2023-10-09T08:25:28.009991+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:29.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:29 smithi094 bash[27803]: audit 2023-10-09T08:25:28.012622+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:29.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:29 smithi094 bash[27803]: cluster 2023-10-09T08:25:28.187688+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:29.498 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:29 smithi016 bash[33851]: audit 2023-10-09T08:25:28.009991+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:29 smithi016 bash[33851]: audit 2023-10-09T08:25:28.012622+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:29 smithi016 bash[33851]: cluster 2023-10-09T08:25:28.187688+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:29 smithi016 bash[27454]: audit 2023-10-09T08:25:28.009991+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:29 smithi016 bash[27454]: audit 2023-10-09T08:25:28.012622+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:29 smithi016 bash[27454]: cluster 2023-10-09T08:25:28.187688+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:29.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:29 smithi094 bash[28656]: debug 2023-10-09T08:25:29.245+0000 7f188db11680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:25:29.741 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:29 smithi094 bash[28656]: debug 2023-10-09T08:25:29.345+0000 7f188db11680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:25:30.239 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:29 smithi094 bash[28656]: debug 2023-10-09T08:25:29.801+0000 7f188db11680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:25:30.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:30 smithi094 bash[27803]: audit 2023-10-09T08:25:29.238170+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:30.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:30 smithi016 bash[27454]: audit 2023-10-09T08:25:29.238170+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:30.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:30 smithi016 bash[33851]: audit 2023-10-09T08:25:29.238170+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:30.635 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T08:25:30.635 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T08:25:30.635 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-10-09T08:25:30.635 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-10-09T08:25:30.636 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0140386 s, 747 MB/s 2023-10-09T08:25:30.636 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-10-09T08:25:30.990 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:30 smithi094 bash[28656]: debug 2023-10-09T08:25:30.717+0000 7f188db11680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:25:30.991 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:30 smithi094 bash[28656]: debug 2023-10-09T08:25:30.813+0000 7f188db11680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:25:31.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_4 2023-10-09T08:25:31.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:31 smithi094 bash[27803]: cluster 2023-10-09T08:25:30.188156+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:31.491 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:31 smithi094 bash[28656]: debug 2023-10-09T08:25:31.061+0000 7f188db11680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:25:31.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:31 smithi016 bash[27454]: cluster 2023-10-09T08:25:30.188156+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:31.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:31 smithi016 bash[33851]: cluster 2023-10-09T08:25:30.188156+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:32.449 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:32 smithi094 bash[28656]: debug 2023-10-09T08:25:32.181+0000 7f188db11680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:25:32.450 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:32 smithi094 bash[28656]: debug 2023-10-09T08:25:32.345+0000 7f188db11680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:25:32.741 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:32 smithi094 bash[28656]: debug 2023-10-09T08:25:32.445+0000 7f188db11680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:25:32.741 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:32 smithi094 bash[28656]: debug 2023-10-09T08:25:32.557+0000 7f188db11680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:25:33.195 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:32 smithi094 bash[28656]: debug 2023-10-09T08:25:32.777+0000 7f188db11680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:25:33.490 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[28656]: debug 2023-10-09T08:25:33.189+0000 7f188db11680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:25:33.491 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[28656]: debug 2023-10-09T08:25:33.309+0000 7f188db11680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:25:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: cluster 2023-10-09T08:25:32.188517+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: audit 2023-10-09T08:25:33.056047+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: audit 2023-10-09T08:25:33.057416+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: audit 2023-10-09T08:25:33.059029+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: audit 2023-10-09T08:25:33.064947+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: audit 2023-10-09T08:25:33.068636+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:25:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: audit 2023-10-09T08:25:33.069788+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:25:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:33 smithi094 bash[27803]: audit 2023-10-09T08:25:33.070698+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:33.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: cluster 2023-10-09T08:25:32.188517+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:33.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: audit 2023-10-09T08:25:33.056047+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:33.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: audit 2023-10-09T08:25:33.057416+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:33.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: audit 2023-10-09T08:25:33.059029+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:33.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: audit 2023-10-09T08:25:33.064947+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:33.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: audit 2023-10-09T08:25:33.068636+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:25:33.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: audit 2023-10-09T08:25:33.069788+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:25:33.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[27454]: audit 2023-10-09T08:25:33.070698+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:33.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: cluster 2023-10-09T08:25:32.188517+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:33.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: audit 2023-10-09T08:25:33.056047+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:33.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: audit 2023-10-09T08:25:33.057416+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:33.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: audit 2023-10-09T08:25:33.059029+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:33.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: audit 2023-10-09T08:25:33.064947+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:33.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: audit 2023-10-09T08:25:33.068636+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:25:33.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: audit 2023-10-09T08:25:33.069788+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:25:33.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:33 smithi016 bash[33851]: audit 2023-10-09T08:25:33.070698+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:34.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:34 smithi094 bash[27803]: cephadm 2023-10-09T08:25:33.068187+0000 mgr.y (mgr.14146) 57 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T08:25:34.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:34 smithi094 bash[27803]: cephadm 2023-10-09T08:25:33.071528+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring daemon mgr.y on smithi016 2023-10-09T08:25:34.491 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:34 smithi094 bash[28656]: debug 2023-10-09T08:25:34.169+0000 7f188db11680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:25:34.491 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:34 smithi094 bash[28656]: debug 2023-10-09T08:25:34.269+0000 7f188db11680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:25:34.491 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:34 smithi094 bash[28656]: debug 2023-10-09T08:25:34.381+0000 7f188db11680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:25:34.563 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:34 smithi016 bash[27454]: cephadm 2023-10-09T08:25:33.068187+0000 mgr.y (mgr.14146) 57 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T08:25:34.563 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:34 smithi016 bash[27454]: cephadm 2023-10-09T08:25:33.071528+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring daemon mgr.y on smithi016 2023-10-09T08:25:34.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:34 smithi016 bash[33851]: cephadm 2023-10-09T08:25:33.068187+0000 mgr.y (mgr.14146) 57 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T08:25:34.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:34 smithi016 bash[33851]: cephadm 2023-10-09T08:25:33.071528+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring daemon mgr.y on smithi016 2023-10-09T08:25:34.888 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:34 smithi094 bash[28656]: debug 2023-10-09T08:25:34.597+0000 7f188db11680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:25:35.217 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:34 smithi094 bash[28656]: debug 2023-10-09T08:25:34.881+0000 7f188db11680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:25:35.218 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[28656]: debug 2023-10-09T08:25:35.045+0000 7f188db11680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:25:35.490 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[28656]: debug 2023-10-09T08:25:35.213+0000 7f188db11680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:25:35.491 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[28656]: debug 2023-10-09T08:25:35.425+0000 7f188db11680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:25:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[27803]: cluster 2023-10-09T08:25:34.188918+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[27803]: audit 2023-10-09T08:25:34.914598+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:25:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[27803]: audit 2023-10-09T08:25:34.919465+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:25:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[27803]: audit 2023-10-09T08:25:34.920892+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:35.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[33851]: cluster 2023-10-09T08:25:34.188918+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:35.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[33851]: audit 2023-10-09T08:25:34.914598+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:25:35.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[33851]: audit 2023-10-09T08:25:34.919465+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:25:35.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[33851]: audit 2023-10-09T08:25:34.920892+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:35.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[27454]: cluster 2023-10-09T08:25:34.188918+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:35.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[27454]: audit 2023-10-09T08:25:34.914598+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:25:35.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[27454]: audit 2023-10-09T08:25:34.919465+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:25:35.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:35 smithi016 bash[27454]: audit 2023-10-09T08:25:34.920892+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:35.991 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:25:35 smithi094 bash[28656]: debug 2023-10-09T08:25:35.525+0000 7f188db11680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:34.911259+0000 mgr.y (mgr.14146) 60 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.429929+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.432034+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: cluster 2023-10-09T08:25:35.556408+0000 mon.a (mon.0) 217 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.558278+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.559117+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.561221+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:25:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.562094+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:25:36.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.726506+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:36.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.727861+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:36.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:36 smithi094 bash[27803]: audit 2023-10-09T08:25:35.735251+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:36.748 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:34.911259+0000 mgr.y (mgr.14146) 60 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.429929+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.432034+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: cluster 2023-10-09T08:25:35.556408+0000 mon.a (mon.0) 217 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.558278+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.559117+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.561221+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.562094+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.726506+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.727861+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:36.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[33851]: audit 2023-10-09T08:25:35.735251+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:34.911259+0000 mgr.y (mgr.14146) 60 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.429929+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.432034+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: cluster 2023-10-09T08:25:35.556408+0000 mon.a (mon.0) 217 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.558278+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.559117+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.561221+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.562094+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.94:0/1248600579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.726506+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.727861+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:25:36.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:36 smithi016 bash[27454]: audit 2023-10-09T08:25:35.735251+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:25:37.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:37 smithi094 bash[27803]: cluster 2023-10-09T08:25:36.189385+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:37.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:37 smithi094 bash[27803]: cluster 2023-10-09T08:25:36.443919+0000 mon.a (mon.0) 221 : cluster [DBG] mgrmap e14: y(active, since 80s), standbys: x 2023-10-09T08:25:37.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:37 smithi094 bash[27803]: audit 2023-10-09T08:25:36.444203+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:25:37.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:37 smithi016 bash[27454]: cluster 2023-10-09T08:25:36.189385+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:37.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:37 smithi016 bash[27454]: cluster 2023-10-09T08:25:36.443919+0000 mon.a (mon.0) 221 : cluster [DBG] mgrmap e14: y(active, since 80s), standbys: x 2023-10-09T08:25:37.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:37 smithi016 bash[27454]: audit 2023-10-09T08:25:36.444203+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:25:37.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:37 smithi016 bash[33851]: cluster 2023-10-09T08:25:36.189385+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:37.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:37 smithi016 bash[33851]: cluster 2023-10-09T08:25:36.443919+0000 mon.a (mon.0) 221 : cluster [DBG] mgrmap e14: y(active, since 80s), standbys: x 2023-10-09T08:25:37.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:37 smithi016 bash[33851]: audit 2023-10-09T08:25:36.444203+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:25:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:39 smithi094 bash[27803]: cluster 2023-10-09T08:25:38.189989+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:39.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:39 smithi016 bash[27454]: cluster 2023-10-09T08:25:38.189989+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:39.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:39 smithi016 bash[33851]: cluster 2023-10-09T08:25:38.189989+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:41.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:40 smithi094 bash[27803]: cluster 2023-10-09T08:25:40.190589+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:41.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:40 smithi016 bash[27454]: cluster 2023-10-09T08:25:40.190589+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:41.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:41 smithi016 bash[33851]: cluster 2023-10-09T08:25:40.190589+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[27454]: cluster 2023-10-09T08:25:42.191115+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[27454]: audit 2023-10-09T08:25:43.107189+0000 mon.a (mon.0) 223 : audit [INF] from='client.? 172.21.15.16:0/1962041116' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0334a36b-f8d4-486c-a3f7-6b5278148938"}]: dispatch 2023-10-09T08:25:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[27454]: audit 2023-10-09T08:25:43.114157+0000 mon.a (mon.0) 224 : audit [INF] from='client.? 172.21.15.16:0/1962041116' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0334a36b-f8d4-486c-a3f7-6b5278148938"}]': finished 2023-10-09T08:25:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[27454]: cluster 2023-10-09T08:25:43.114331+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T08:25:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[27454]: audit 2023-10-09T08:25:43.114525+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:25:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[33851]: cluster 2023-10-09T08:25:42.191115+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[33851]: audit 2023-10-09T08:25:43.107189+0000 mon.a (mon.0) 223 : audit [INF] from='client.? 172.21.15.16:0/1962041116' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0334a36b-f8d4-486c-a3f7-6b5278148938"}]: dispatch 2023-10-09T08:25:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[33851]: audit 2023-10-09T08:25:43.114157+0000 mon.a (mon.0) 224 : audit [INF] from='client.? 172.21.15.16:0/1962041116' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0334a36b-f8d4-486c-a3f7-6b5278148938"}]': finished 2023-10-09T08:25:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[33851]: cluster 2023-10-09T08:25:43.114331+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T08:25:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:43 smithi016 bash[33851]: audit 2023-10-09T08:25:43.114525+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:25:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:43 smithi094 bash[27803]: cluster 2023-10-09T08:25:42.191115+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:43 smithi094 bash[27803]: audit 2023-10-09T08:25:43.107189+0000 mon.a (mon.0) 223 : audit [INF] from='client.? 172.21.15.16:0/1962041116' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0334a36b-f8d4-486c-a3f7-6b5278148938"}]: dispatch 2023-10-09T08:25:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:43 smithi094 bash[27803]: audit 2023-10-09T08:25:43.114157+0000 mon.a (mon.0) 224 : audit [INF] from='client.? 172.21.15.16:0/1962041116' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0334a36b-f8d4-486c-a3f7-6b5278148938"}]': finished 2023-10-09T08:25:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:43 smithi094 bash[27803]: cluster 2023-10-09T08:25:43.114331+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T08:25:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:43 smithi094 bash[27803]: audit 2023-10-09T08:25:43.114525+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:25:44.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:44 smithi094 bash[27803]: audit 2023-10-09T08:25:43.822448+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.16:0/3135848085' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:25:44.748 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:44 smithi016 bash[33851]: audit 2023-10-09T08:25:43.822448+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.16:0/3135848085' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:25:44.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:44 smithi016 bash[27454]: audit 2023-10-09T08:25:43.822448+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.16:0/3135848085' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:25:45.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:45 smithi094 bash[27803]: cluster 2023-10-09T08:25:44.191566+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:45.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:45 smithi016 bash[27454]: cluster 2023-10-09T08:25:44.191566+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:45.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:45 smithi016 bash[33851]: cluster 2023-10-09T08:25:44.191566+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:47.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:47 smithi094 bash[27803]: cluster 2023-10-09T08:25:46.192006+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:47.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:47 smithi016 bash[27454]: cluster 2023-10-09T08:25:46.192006+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:47.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:47 smithi016 bash[33851]: cluster 2023-10-09T08:25:46.192006+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:49 smithi094 bash[27803]: cluster 2023-10-09T08:25:48.192475+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:49.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:49 smithi016 bash[27454]: cluster 2023-10-09T08:25:48.192475+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:49.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:49 smithi016 bash[33851]: cluster 2023-10-09T08:25:48.192475+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:51.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:50 smithi094 bash[27803]: cluster 2023-10-09T08:25:50.192886+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:51.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:50 smithi016 bash[33851]: cluster 2023-10-09T08:25:50.192886+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:50 smithi016 bash[27454]: cluster 2023-10-09T08:25:50.192886+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:53.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:53 smithi016 bash[27454]: cluster 2023-10-09T08:25:52.193414+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:53.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:53 smithi016 bash[33851]: cluster 2023-10-09T08:25:52.193414+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:53 smithi094 bash[27803]: cluster 2023-10-09T08:25:52.193414+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:55.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:55 smithi016 bash[27454]: cluster 2023-10-09T08:25:54.193871+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:55.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:55 smithi016 bash[33851]: cluster 2023-10-09T08:25:54.193871+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:55.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:55 smithi094 bash[27803]: cluster 2023-10-09T08:25:54.193871+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:56.670 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:56 smithi016 bash[33851]: audit 2023-10-09T08:25:55.618107+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:25:56.671 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:56 smithi016 bash[33851]: audit 2023-10-09T08:25:55.619754+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:56.671 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:56 smithi016 bash[27454]: audit 2023-10-09T08:25:55.618107+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:25:56.671 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:56 smithi016 bash[27454]: audit 2023-10-09T08:25:55.619754+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:56.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:56 smithi094 bash[27803]: audit 2023-10-09T08:25:55.618107+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:25:56.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:56 smithi094 bash[27803]: audit 2023-10-09T08:25:55.619754+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:25:57.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:57 smithi016 bash[27454]: cephadm 2023-10-09T08:25:55.621125+0000 mgr.y (mgr.14146) 71 : cephadm [INF] Deploying daemon osd.0 on smithi016 2023-10-09T08:25:57.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:57 smithi016 bash[27454]: cluster 2023-10-09T08:25:56.194394+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:57.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:57 smithi016 bash[33851]: cephadm 2023-10-09T08:25:55.621125+0000 mgr.y (mgr.14146) 71 : cephadm [INF] Deploying daemon osd.0 on smithi016 2023-10-09T08:25:57.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:57 smithi016 bash[33851]: cluster 2023-10-09T08:25:56.194394+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:57.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:57 smithi094 bash[27803]: cephadm 2023-10-09T08:25:55.621125+0000 mgr.y (mgr.14146) 71 : cephadm [INF] Deploying daemon osd.0 on smithi016 2023-10-09T08:25:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:57 smithi094 bash[27803]: cluster 2023-10-09T08:25:56.194394+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:25:59 smithi094 bash[27803]: cluster 2023-10-09T08:25:58.194888+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:59.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:25:59 smithi016 bash[27454]: cluster 2023-10-09T08:25:58.194888+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:25:59.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:25:59 smithi016 bash[33851]: cluster 2023-10-09T08:25:58.194888+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:01.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:01 smithi016 bash[27454]: audit 2023-10-09T08:26:00.017487+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:01.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:01 smithi016 bash[27454]: audit 2023-10-09T08:26:00.021201+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:01.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:01 smithi016 bash[27454]: cluster 2023-10-09T08:26:00.195350+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:01.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:01 smithi016 bash[33851]: audit 2023-10-09T08:26:00.017487+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:01.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:01 smithi016 bash[33851]: audit 2023-10-09T08:26:00.021201+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:01.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:01 smithi016 bash[33851]: cluster 2023-10-09T08:26:00.195350+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:01.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:01 smithi094 bash[27803]: audit 2023-10-09T08:26:00.017487+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:01.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:01 smithi094 bash[27803]: audit 2023-10-09T08:26:00.021201+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:01.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:01 smithi094 bash[27803]: cluster 2023-10-09T08:26:00.195350+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:03.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:03 smithi016 bash[27454]: cluster 2023-10-09T08:26:02.195767+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:03.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:03 smithi016 bash[33851]: cluster 2023-10-09T08:26:02.195767+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:03.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:03 smithi094 bash[27803]: cluster 2023-10-09T08:26:02.195767+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:05 smithi094 bash[27803]: cluster 2023-10-09T08:26:04.196320+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:05 smithi094 bash[27803]: audit 2023-10-09T08:26:05.222660+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:05 smithi094 bash[27803]: audit 2023-10-09T08:26:05.223594+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:05 smithi094 bash[27803]: audit 2023-10-09T08:26:05.224767+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:05 smithi094 bash[27803]: audit 2023-10-09T08:26:05.230009+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[27454]: cluster 2023-10-09T08:26:04.196320+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[27454]: audit 2023-10-09T08:26:05.222660+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[27454]: audit 2023-10-09T08:26:05.223594+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[27454]: audit 2023-10-09T08:26:05.224767+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[27454]: audit 2023-10-09T08:26:05.230009+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:05.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[33851]: cluster 2023-10-09T08:26:04.196320+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[33851]: audit 2023-10-09T08:26:05.222660+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[33851]: audit 2023-10-09T08:26:05.223594+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[33851]: audit 2023-10-09T08:26:05.224767+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:05 smithi016 bash[33851]: audit 2023-10-09T08:26:05.230009+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:07 smithi094 bash[27803]: cluster 2023-10-09T08:26:06.196749+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:07 smithi094 bash[27803]: audit 2023-10-09T08:26:06.777753+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:26:07.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:07 smithi016 bash[27454]: cluster 2023-10-09T08:26:06.196749+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:07.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:07 smithi016 bash[27454]: audit 2023-10-09T08:26:06.777753+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:26:07.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:07 smithi016 bash[33851]: cluster 2023-10-09T08:26:06.196749+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:07.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:07 smithi016 bash[33851]: audit 2023-10-09T08:26:06.777753+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:26:08.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[27454]: audit 2023-10-09T08:26:07.261551+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:26:08.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[27454]: cluster 2023-10-09T08:26:07.261651+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T08:26:08.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[27454]: audit 2023-10-09T08:26:07.262164+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:08.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[27454]: audit 2023-10-09T08:26:07.263109+0000 mon.a (mon.0) 239 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:08.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[33851]: audit 2023-10-09T08:26:07.261551+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:26:08.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[33851]: cluster 2023-10-09T08:26:07.261651+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T08:26:08.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[33851]: audit 2023-10-09T08:26:07.262164+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:08.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:08 smithi016 bash[33851]: audit 2023-10-09T08:26:07.263109+0000 mon.a (mon.0) 239 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:08 smithi094 bash[27803]: audit 2023-10-09T08:26:07.261551+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:26:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:08 smithi094 bash[27803]: cluster 2023-10-09T08:26:07.261651+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T08:26:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:08 smithi094 bash[27803]: audit 2023-10-09T08:26:07.262164+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:08 smithi094 bash[27803]: audit 2023-10-09T08:26:07.263109+0000 mon.a (mon.0) 239 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:08.750 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 0 on host 'smithi016' 2023-10-09T08:26:09.468 DEBUG:teuthology.orchestra.run.smithi016:osd.0> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T08:26:09.486 INFO:tasks.cephadm:Deploying osd.1 on smithi016 with /dev/vg_nvme/lv_3... 2023-10-09T08:26:09.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T08:26:09.547 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[27454]: cluster 2023-10-09T08:26:08.197285+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:09.548 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[27454]: audit 2023-10-09T08:26:08.263301+0000 mon.a (mon.0) 240 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:26:09.548 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[27454]: cluster 2023-10-09T08:26:08.263409+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T08:26:09.548 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[27454]: audit 2023-10-09T08:26:08.263689+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:09.548 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[27454]: audit 2023-10-09T08:26:08.271220+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:09.548 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[27454]: audit 2023-10-09T08:26:08.748791+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:09.548 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[27454]: audit 2023-10-09T08:26:08.755614+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:09.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[33851]: cluster 2023-10-09T08:26:08.197285+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:09.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[33851]: audit 2023-10-09T08:26:08.263301+0000 mon.a (mon.0) 240 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:26:09.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[33851]: cluster 2023-10-09T08:26:08.263409+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T08:26:09.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[33851]: audit 2023-10-09T08:26:08.263689+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:09.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[33851]: audit 2023-10-09T08:26:08.271220+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:09.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[33851]: audit 2023-10-09T08:26:08.748791+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:09.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:09 smithi016 bash[33851]: audit 2023-10-09T08:26:08.755614+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:09.652 INFO:journalctl@ceph.osd.0.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:09 smithi094 bash[27803]: cluster 2023-10-09T08:26:08.197285+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:09 smithi094 bash[27803]: audit 2023-10-09T08:26:08.263301+0000 mon.a (mon.0) 240 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:09 smithi094 bash[27803]: cluster 2023-10-09T08:26:08.263409+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T08:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:09 smithi094 bash[27803]: audit 2023-10-09T08:26:08.263689+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:09 smithi094 bash[27803]: audit 2023-10-09T08:26:08.271220+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:09.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:09 smithi094 bash[27803]: audit 2023-10-09T08:26:08.748791+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:09.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:09 smithi094 bash[27803]: audit 2023-10-09T08:26:08.755614+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:10.585 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[27454]: cluster 2023-10-09T08:26:07.764610+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:26:10.585 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[27454]: cluster 2023-10-09T08:26:07.764750+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:26:10.585 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[27454]: audit 2023-10-09T08:26:09.274008+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[27454]: cluster 2023-10-09T08:26:09.279720+0000 mon.a (mon.0) 247 : cluster [INF] osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725] boot 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[27454]: cluster 2023-10-09T08:26:09.279774+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[27454]: audit 2023-10-09T08:26:09.280266+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[33851]: cluster 2023-10-09T08:26:07.764610+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[33851]: cluster 2023-10-09T08:26:07.764750+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[33851]: audit 2023-10-09T08:26:09.274008+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[33851]: cluster 2023-10-09T08:26:09.279720+0000 mon.a (mon.0) 247 : cluster [INF] osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725] boot 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[33851]: cluster 2023-10-09T08:26:09.279774+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T08:26:10.586 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:10 smithi016 bash[33851]: audit 2023-10-09T08:26:09.280266+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:10 smithi094 bash[27803]: cluster 2023-10-09T08:26:07.764610+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:26:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:10 smithi094 bash[27803]: cluster 2023-10-09T08:26:07.764750+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:26:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:10 smithi094 bash[27803]: audit 2023-10-09T08:26:09.274008+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:10 smithi094 bash[27803]: cluster 2023-10-09T08:26:09.279720+0000 mon.a (mon.0) 247 : cluster [INF] osd.0 [v2:172.21.15.16:6802/818359725,v1:172.21.15.16:6803/818359725] boot 2023-10-09T08:26:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:10 smithi094 bash[27803]: cluster 2023-10-09T08:26:09.279774+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T08:26:10.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:10 smithi094 bash[27803]: audit 2023-10-09T08:26:09.280266+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:26:11.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:11 smithi094 bash[27803]: cluster 2023-10-09T08:26:10.197718+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:11 smithi094 bash[27803]: cluster 2023-10-09T08:26:10.282919+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T08:26:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:11 smithi016 bash[27454]: cluster 2023-10-09T08:26:10.197718+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:11 smithi016 bash[27454]: cluster 2023-10-09T08:26:10.282919+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T08:26:11.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:11 smithi016 bash[33851]: cluster 2023-10-09T08:26:10.197718+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:26:11.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:11 smithi016 bash[33851]: cluster 2023-10-09T08:26:10.282919+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T08:26:13.003 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T08:26:13.003 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T08:26:13.003 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-10-09T08:26:13.004 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-10-09T08:26:13.004 INFO:teuthology.orchestra.run.smithi016.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.011342 s, 925 MB/s 2023-10-09T08:26:13.004 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-10-09T08:26:13.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:13 smithi016 bash[27454]: cluster 2023-10-09T08:26:12.198161+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:13.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:13 smithi016 bash[33851]: cluster 2023-10-09T08:26:12.198161+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:13.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_3 2023-10-09T08:26:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:13 smithi094 bash[27803]: cluster 2023-10-09T08:26:12.198161+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:15.309 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:15 smithi016 bash[27454]: cluster 2023-10-09T08:26:14.198715+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:15.309 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:15 smithi016 bash[33851]: cluster 2023-10-09T08:26:14.198715+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:15 smithi094 bash[27803]: cluster 2023-10-09T08:26:14.198715+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:17 smithi094 bash[27803]: cluster 2023-10-09T08:26:16.199258+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:17.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:17 smithi016 bash[27454]: cluster 2023-10-09T08:26:16.199258+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:17.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:17 smithi016 bash[33851]: cluster 2023-10-09T08:26:16.199258+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:18 smithi094 bash[27803]: audit 2023-10-09T08:26:18.215984+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:26:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:18 smithi094 bash[27803]: audit 2023-10-09T08:26:18.218975+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:26:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:18 smithi094 bash[27803]: audit 2023-10-09T08:26:18.219727+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:18.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:18 smithi016 bash[27454]: audit 2023-10-09T08:26:18.215984+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:26:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:18 smithi016 bash[27454]: audit 2023-10-09T08:26:18.218975+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:26:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:18 smithi016 bash[27454]: audit 2023-10-09T08:26:18.219727+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:18.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:18 smithi016 bash[33851]: audit 2023-10-09T08:26:18.215984+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:26:18.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:18 smithi016 bash[33851]: audit 2023-10-09T08:26:18.218975+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:26:18.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:18 smithi016 bash[33851]: audit 2023-10-09T08:26:18.219727+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:19 smithi094 bash[27803]: cluster 2023-10-09T08:26:18.199609+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:19 smithi094 bash[27803]: audit 2023-10-09T08:26:18.210760+0000 mgr.y (mgr.14146) 84 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:26:19.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:19 smithi016 bash[33851]: cluster 2023-10-09T08:26:18.199609+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:19.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:19 smithi016 bash[33851]: audit 2023-10-09T08:26:18.210760+0000 mgr.y (mgr.14146) 84 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:26:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:19 smithi016 bash[27454]: cluster 2023-10-09T08:26:18.199609+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:19 smithi016 bash[27454]: audit 2023-10-09T08:26:18.210760+0000 mgr.y (mgr.14146) 84 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:26:21.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: cephadm 2023-10-09T08:26:19.752054+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:26:21.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: audit 2023-10-09T08:26:19.761007+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: audit 2023-10-09T08:26:19.762623+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:26:21.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: cephadm 2023-10-09T08:26:19.763652+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 16245M 2023-10-09T08:26:21.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: audit 2023-10-09T08:26:19.768799+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: audit 2023-10-09T08:26:19.770250+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:21.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: audit 2023-10-09T08:26:19.771377+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: audit 2023-10-09T08:26:19.777084+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:20 smithi094 bash[27803]: cluster 2023-10-09T08:26:20.200085+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: cephadm 2023-10-09T08:26:19.752054+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:26:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: audit 2023-10-09T08:26:19.761007+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: audit 2023-10-09T08:26:19.762623+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:26:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: cephadm 2023-10-09T08:26:19.763652+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 16245M 2023-10-09T08:26:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: audit 2023-10-09T08:26:19.768799+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: audit 2023-10-09T08:26:19.770250+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: audit 2023-10-09T08:26:19.771377+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: audit 2023-10-09T08:26:19.777084+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[27454]: cluster 2023-10-09T08:26:20.200085+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: cephadm 2023-10-09T08:26:19.752054+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:26:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: audit 2023-10-09T08:26:19.761007+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: audit 2023-10-09T08:26:19.762623+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:26:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: cephadm 2023-10-09T08:26:19.763652+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 16245M 2023-10-09T08:26:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: audit 2023-10-09T08:26:19.768799+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: audit 2023-10-09T08:26:19.770250+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:21.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: audit 2023-10-09T08:26:19.771377+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:21.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: audit 2023-10-09T08:26:19.777084+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:21.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:20 smithi016 bash[33851]: cluster 2023-10-09T08:26:20.200085+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:23.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:23 smithi016 bash[27454]: cluster 2023-10-09T08:26:22.200551+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:23 smithi016 bash[33851]: cluster 2023-10-09T08:26:22.200551+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:23.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:23 smithi094 bash[27803]: cluster 2023-10-09T08:26:22.200551+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:25.578 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:25 smithi016 bash[27454]: cluster 2023-10-09T08:26:24.201021+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:25.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:25 smithi016 bash[33851]: cluster 2023-10-09T08:26:24.201021+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:25 smithi094 bash[27803]: cluster 2023-10-09T08:26:24.201021+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:27 smithi094 bash[27803]: cluster 2023-10-09T08:26:26.201496+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:27 smithi094 bash[27803]: audit 2023-10-09T08:26:26.514471+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.16:0/1792175231' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]: dispatch 2023-10-09T08:26:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:27 smithi094 bash[27803]: audit 2023-10-09T08:26:26.515091+0000 mon.a (mon.0) 260 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]: dispatch 2023-10-09T08:26:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:27 smithi094 bash[27803]: audit 2023-10-09T08:26:26.520608+0000 mon.a (mon.0) 261 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]': finished 2023-10-09T08:26:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:27 smithi094 bash[27803]: cluster 2023-10-09T08:26:26.520680+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T08:26:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:27 smithi094 bash[27803]: audit 2023-10-09T08:26:26.520927+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:27 smithi094 bash[27803]: audit 2023-10-09T08:26:27.217462+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.16:0/1821249086' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:26:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[33851]: cluster 2023-10-09T08:26:26.201496+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[33851]: audit 2023-10-09T08:26:26.514471+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.16:0/1792175231' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]: dispatch 2023-10-09T08:26:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[33851]: audit 2023-10-09T08:26:26.515091+0000 mon.a (mon.0) 260 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]: dispatch 2023-10-09T08:26:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[33851]: audit 2023-10-09T08:26:26.520608+0000 mon.a (mon.0) 261 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]': finished 2023-10-09T08:26:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[33851]: cluster 2023-10-09T08:26:26.520680+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T08:26:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[33851]: audit 2023-10-09T08:26:26.520927+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[33851]: audit 2023-10-09T08:26:27.217462+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.16:0/1821249086' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[27454]: cluster 2023-10-09T08:26:26.201496+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[27454]: audit 2023-10-09T08:26:26.514471+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.16:0/1792175231' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]: dispatch 2023-10-09T08:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[27454]: audit 2023-10-09T08:26:26.515091+0000 mon.a (mon.0) 260 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]: dispatch 2023-10-09T08:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[27454]: audit 2023-10-09T08:26:26.520608+0000 mon.a (mon.0) 261 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc3d1f7f-18df-4428-90b3-68c1d8296b11"}]': finished 2023-10-09T08:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[27454]: cluster 2023-10-09T08:26:26.520680+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T08:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[27454]: audit 2023-10-09T08:26:26.520927+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:27 smithi016 bash[27454]: audit 2023-10-09T08:26:27.217462+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.16:0/1821249086' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:26:29.715 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:29 smithi016 bash[27454]: cluster 2023-10-09T08:26:28.201864+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:29.715 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:29 smithi016 bash[33851]: cluster 2023-10-09T08:26:28.201864+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:29 smithi094 bash[27803]: cluster 2023-10-09T08:26:28.201864+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:31.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:30 smithi094 bash[27803]: cluster 2023-10-09T08:26:30.202360+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:31.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:30 smithi016 bash[33851]: cluster 2023-10-09T08:26:30.202360+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:31.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:30 smithi016 bash[27454]: cluster 2023-10-09T08:26:30.202360+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:33.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:33 smithi094 bash[27803]: cluster 2023-10-09T08:26:32.202795+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:33.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:33 smithi016 bash[33851]: cluster 2023-10-09T08:26:32.202795+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:33.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:33 smithi016 bash[27454]: cluster 2023-10-09T08:26:32.202795+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:35.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:35 smithi016 bash[27454]: cluster 2023-10-09T08:26:34.203275+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:35.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:35 smithi016 bash[33851]: cluster 2023-10-09T08:26:34.203275+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:35.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:35 smithi094 bash[27803]: cluster 2023-10-09T08:26:34.203275+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:37.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:37 smithi016 bash[27454]: cluster 2023-10-09T08:26:36.203746+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:37.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:37 smithi016 bash[33851]: cluster 2023-10-09T08:26:36.203746+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:37.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:37 smithi094 bash[27803]: cluster 2023-10-09T08:26:36.203746+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:39.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:39 smithi016 bash[27454]: cluster 2023-10-09T08:26:38.204318+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:39.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:39 smithi016 bash[27454]: audit 2023-10-09T08:26:38.989083+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:26:39.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:39 smithi016 bash[27454]: audit 2023-10-09T08:26:38.990583+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:39.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:39 smithi016 bash[33851]: cluster 2023-10-09T08:26:38.204318+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:39.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:39 smithi016 bash[33851]: audit 2023-10-09T08:26:38.989083+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:26:39.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:39 smithi016 bash[33851]: audit 2023-10-09T08:26:38.990583+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:39 smithi094 bash[27803]: cluster 2023-10-09T08:26:38.204318+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:39 smithi094 bash[27803]: audit 2023-10-09T08:26:38.989083+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:26:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:39 smithi094 bash[27803]: audit 2023-10-09T08:26:38.990583+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:40 smithi094 bash[27803]: cephadm 2023-10-09T08:26:38.991783+0000 mgr.y (mgr.14146) 97 : cephadm [INF] Deploying daemon osd.1 on smithi016 2023-10-09T08:26:40.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:40 smithi016 bash[27454]: cephadm 2023-10-09T08:26:38.991783+0000 mgr.y (mgr.14146) 97 : cephadm [INF] Deploying daemon osd.1 on smithi016 2023-10-09T08:26:40.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:40 smithi016 bash[33851]: cephadm 2023-10-09T08:26:38.991783+0000 mgr.y (mgr.14146) 97 : cephadm [INF] Deploying daemon osd.1 on smithi016 2023-10-09T08:26:41.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:41 smithi016 bash[27454]: cluster 2023-10-09T08:26:40.204786+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:41.573 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:41 smithi016 bash[33851]: cluster 2023-10-09T08:26:40.204786+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:41 smithi094 bash[27803]: cluster 2023-10-09T08:26:40.204786+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:43 smithi016 bash[27454]: cluster 2023-10-09T08:26:42.205274+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:43 smithi016 bash[27454]: audit 2023-10-09T08:26:43.154700+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:43 smithi016 bash[27454]: audit 2023-10-09T08:26:43.158486+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:43.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:43 smithi016 bash[33851]: cluster 2023-10-09T08:26:42.205274+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:43 smithi016 bash[33851]: audit 2023-10-09T08:26:43.154700+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:43 smithi016 bash[33851]: audit 2023-10-09T08:26:43.158486+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:43 smithi094 bash[27803]: cluster 2023-10-09T08:26:42.205274+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:43 smithi094 bash[27803]: audit 2023-10-09T08:26:43.154700+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:43 smithi094 bash[27803]: audit 2023-10-09T08:26:43.158486+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:45 smithi094 bash[27803]: cluster 2023-10-09T08:26:44.205646+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:45.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:45 smithi016 bash[27454]: cluster 2023-10-09T08:26:44.205646+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:45.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:45 smithi016 bash[33851]: cluster 2023-10-09T08:26:44.205646+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:47 smithi094 bash[27803]: cluster 2023-10-09T08:26:46.205933+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:47.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:47 smithi016 bash[27454]: cluster 2023-10-09T08:26:46.205933+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:47.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:47 smithi016 bash[33851]: cluster 2023-10-09T08:26:46.205933+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:49 smithi094 bash[27803]: cluster 2023-10-09T08:26:48.206390+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:49.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:49 smithi016 bash[27454]: cluster 2023-10-09T08:26:48.206390+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:49.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:49 smithi016 bash[33851]: cluster 2023-10-09T08:26:48.206390+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:50 smithi094 bash[27803]: audit 2023-10-09T08:26:49.422568+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:50 smithi094 bash[27803]: audit 2023-10-09T08:26:49.686284+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:50 smithi094 bash[27803]: audit 2023-10-09T08:26:49.687626+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:50 smithi094 bash[27803]: audit 2023-10-09T08:26:49.694011+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:50 smithi094 bash[27803]: audit 2023-10-09T08:26:49.975303+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:26:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:50 smithi094 bash[27803]: audit 2023-10-09T08:26:49.975867+0000 mon.a (mon.0) 272 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:26:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[27454]: audit 2023-10-09T08:26:49.422568+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[27454]: audit 2023-10-09T08:26:49.686284+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[27454]: audit 2023-10-09T08:26:49.687626+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[27454]: audit 2023-10-09T08:26:49.694011+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[27454]: audit 2023-10-09T08:26:49.975303+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:26:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[27454]: audit 2023-10-09T08:26:49.975867+0000 mon.a (mon.0) 272 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:26:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[33851]: audit 2023-10-09T08:26:49.422568+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[33851]: audit 2023-10-09T08:26:49.686284+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:26:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[33851]: audit 2023-10-09T08:26:49.687626+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:26:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[33851]: audit 2023-10-09T08:26:49.694011+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[33851]: audit 2023-10-09T08:26:49.975303+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:26:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:50 smithi016 bash[33851]: audit 2023-10-09T08:26:49.975867+0000 mon.a (mon.0) 272 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:26:51.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[33851]: cluster 2023-10-09T08:26:50.206951+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:51.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[33851]: audit 2023-10-09T08:26:50.697288+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:26:51.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[33851]: cluster 2023-10-09T08:26:50.697373+0000 mon.a (mon.0) 274 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T08:26:51.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[33851]: audit 2023-10-09T08:26:50.697571+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:51.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[33851]: audit 2023-10-09T08:26:50.700854+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:51.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[33851]: audit 2023-10-09T08:26:50.701746+0000 mon.a (mon.0) 276 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:51.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[27454]: cluster 2023-10-09T08:26:50.206951+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:51.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[27454]: audit 2023-10-09T08:26:50.697288+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:26:51.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[27454]: cluster 2023-10-09T08:26:50.697373+0000 mon.a (mon.0) 274 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T08:26:51.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[27454]: audit 2023-10-09T08:26:50.697571+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:51.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[27454]: audit 2023-10-09T08:26:50.700854+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:51.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:51 smithi016 bash[27454]: audit 2023-10-09T08:26:50.701746+0000 mon.a (mon.0) 276 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:51 smithi094 bash[27803]: cluster 2023-10-09T08:26:50.206951+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:51 smithi094 bash[27803]: audit 2023-10-09T08:26:50.697288+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:26:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:51 smithi094 bash[27803]: cluster 2023-10-09T08:26:50.697373+0000 mon.a (mon.0) 274 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T08:26:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:51 smithi094 bash[27803]: audit 2023-10-09T08:26:50.697571+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:51 smithi094 bash[27803]: audit 2023-10-09T08:26:50.700854+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:51 smithi094 bash[27803]: audit 2023-10-09T08:26:50.701746+0000 mon.a (mon.0) 276 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:26:52.986 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 1 on host 'smithi016' 2023-10-09T08:26:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:52 smithi094 bash[27803]: audit 2023-10-09T08:26:51.702874+0000 mon.a (mon.0) 277 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:26:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:52 smithi094 bash[27803]: cluster 2023-10-09T08:26:51.703133+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T08:26:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:52 smithi094 bash[27803]: audit 2023-10-09T08:26:51.703334+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:52 smithi094 bash[27803]: audit 2023-10-09T08:26:51.709065+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:52 smithi094 bash[27803]: cluster 2023-10-09T08:26:52.207403+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:52.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[33851]: audit 2023-10-09T08:26:51.702874+0000 mon.a (mon.0) 277 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:26:52.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[33851]: cluster 2023-10-09T08:26:51.703133+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T08:26:52.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[33851]: audit 2023-10-09T08:26:51.703334+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:52.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[33851]: audit 2023-10-09T08:26:51.709065+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:52.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[33851]: cluster 2023-10-09T08:26:52.207403+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:53.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[27454]: audit 2023-10-09T08:26:51.702874+0000 mon.a (mon.0) 277 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:26:53.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[27454]: cluster 2023-10-09T08:26:51.703133+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T08:26:53.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[27454]: audit 2023-10-09T08:26:51.703334+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:53.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[27454]: audit 2023-10-09T08:26:51.709065+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:53.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:52 smithi016 bash[27454]: cluster 2023-10-09T08:26:52.207403+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:26:53.662 DEBUG:teuthology.orchestra.run.smithi016:osd.1> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T08:26:53.664 INFO:tasks.cephadm:Deploying osd.2 on smithi016 with /dev/vg_nvme/lv_2... 2023-10-09T08:26:53.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T08:26:53.793 INFO:journalctl@ceph.osd.1.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:53 smithi094 bash[27803]: cluster 2023-10-09T08:26:50.992387+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:53 smithi094 bash[27803]: cluster 2023-10-09T08:26:50.992486+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:53 smithi094 bash[27803]: cluster 2023-10-09T08:26:52.705156+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092] boot 2023-10-09T08:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:53 smithi094 bash[27803]: cluster 2023-10-09T08:26:52.705330+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T08:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:53 smithi094 bash[27803]: audit 2023-10-09T08:26:52.706227+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:53 smithi094 bash[27803]: audit 2023-10-09T08:26:52.984987+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:53 smithi094 bash[27803]: audit 2023-10-09T08:26:53.020209+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[33851]: cluster 2023-10-09T08:26:50.992387+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:26:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[33851]: cluster 2023-10-09T08:26:50.992486+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:26:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[33851]: cluster 2023-10-09T08:26:52.705156+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092] boot 2023-10-09T08:26:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[33851]: cluster 2023-10-09T08:26:52.705330+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T08:26:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[33851]: audit 2023-10-09T08:26:52.706227+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[33851]: audit 2023-10-09T08:26:52.984987+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[33851]: audit 2023-10-09T08:26:53.020209+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[27454]: cluster 2023-10-09T08:26:50.992387+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[27454]: cluster 2023-10-09T08:26:50.992486+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[27454]: cluster 2023-10-09T08:26:52.705156+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.16:6810/664142092,v1:172.21.15.16:6811/664142092] boot 2023-10-09T08:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[27454]: cluster 2023-10-09T08:26:52.705330+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T08:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[27454]: audit 2023-10-09T08:26:52.706227+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[27454]: audit 2023-10-09T08:26:52.984987+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:53 smithi016 bash[27454]: audit 2023-10-09T08:26:53.020209+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:26:55.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:54 smithi016 bash[27454]: cluster 2023-10-09T08:26:53.989555+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T08:26:55.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:54 smithi016 bash[27454]: cluster 2023-10-09T08:26:54.207900+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:55.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:54 smithi016 bash[33851]: cluster 2023-10-09T08:26:53.989555+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T08:26:55.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:54 smithi016 bash[33851]: cluster 2023-10-09T08:26:54.207900+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:55.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:54 smithi094 bash[27803]: cluster 2023-10-09T08:26:53.989555+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T08:26:55.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:54 smithi094 bash[27803]: cluster 2023-10-09T08:26:54.207900+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:57.093 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T08:26:57.094 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T08:26:57.094 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-10-09T08:26:57.094 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-10-09T08:26:57.094 INFO:teuthology.orchestra.run.smithi016.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0143009 s, 733 MB/s 2023-10-09T08:26:57.094 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-10-09T08:26:57.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:57 smithi016 bash[27454]: cluster 2023-10-09T08:26:56.208396+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:57.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:57 smithi016 bash[33851]: cluster 2023-10-09T08:26:56.208396+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:57 smithi094 bash[27803]: cluster 2023-10-09T08:26:56.208396+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:57.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_2 2023-10-09T08:26:59.546 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:26:59 smithi016 bash[27454]: cluster 2023-10-09T08:26:58.208940+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:59.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:26:59 smithi016 bash[33851]: cluster 2023-10-09T08:26:58.208940+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:26:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:26:59 smithi094 bash[27803]: cluster 2023-10-09T08:26:58.208940+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:01.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:01 smithi016 bash[33851]: cluster 2023-10-09T08:27:00.209465+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:01.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:01 smithi016 bash[27454]: cluster 2023-10-09T08:27:00.209465+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:01.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:01 smithi094 bash[27803]: cluster 2023-10-09T08:27:00.209465+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:03.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[27454]: cluster 2023-10-09T08:27:02.209930+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:03.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[27454]: audit 2023-10-09T08:27:02.749585+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:27:03.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[27454]: audit 2023-10-09T08:27:02.754257+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:27:03.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[27454]: audit 2023-10-09T08:27:02.755524+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:03.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[33851]: cluster 2023-10-09T08:27:02.209930+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:03.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[33851]: audit 2023-10-09T08:27:02.749585+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:27:03.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[33851]: audit 2023-10-09T08:27:02.754257+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:27:03.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:03 smithi016 bash[33851]: audit 2023-10-09T08:27:02.755524+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:03.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:03 smithi094 bash[27803]: cluster 2023-10-09T08:27:02.209930+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:03.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:03 smithi094 bash[27803]: audit 2023-10-09T08:27:02.749585+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:27:03.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:03 smithi094 bash[27803]: audit 2023-10-09T08:27:02.754257+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:27:03.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:03 smithi094 bash[27803]: audit 2023-10-09T08:27:02.755524+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:02.744566+0000 mgr.y (mgr.14146) 110 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:27:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:04.029267+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:04.030789+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:04.032207+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:04.038312+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:04.039880+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:04.040986+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:04 smithi094 bash[27803]: audit 2023-10-09T08:27:04.046727+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:02.744566+0000 mgr.y (mgr.14146) 110 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:27:04.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:04.029267+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:04.030789+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:04.032207+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:04.038312+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:04.039880+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:04.040986+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[27454]: audit 2023-10-09T08:27:04.046727+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:02.744566+0000 mgr.y (mgr.14146) 110 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:27:04.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:04.029267+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:04.030789+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:04.032207+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:04.038312+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:04.039880+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:04.040986+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:04 smithi016 bash[33851]: audit 2023-10-09T08:27:04.046727+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:05 smithi094 bash[27803]: cephadm 2023-10-09T08:27:04.021841+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:27:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:05 smithi094 bash[27803]: cephadm 2023-10-09T08:27:04.033094+0000 mgr.y (mgr.14146) 112 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 8122M 2023-10-09T08:27:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:05 smithi094 bash[27803]: cluster 2023-10-09T08:27:04.210424+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:05 smithi016 bash[27454]: cephadm 2023-10-09T08:27:04.021841+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:27:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:05 smithi016 bash[27454]: cephadm 2023-10-09T08:27:04.033094+0000 mgr.y (mgr.14146) 112 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 8122M 2023-10-09T08:27:05.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:05 smithi016 bash[27454]: cluster 2023-10-09T08:27:04.210424+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:05.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:05 smithi016 bash[33851]: cephadm 2023-10-09T08:27:04.021841+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:27:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:05 smithi016 bash[33851]: cephadm 2023-10-09T08:27:04.033094+0000 mgr.y (mgr.14146) 112 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 8122M 2023-10-09T08:27:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:05 smithi016 bash[33851]: cluster 2023-10-09T08:27:04.210424+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:07.544 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:07 smithi016 bash[27454]: cluster 2023-10-09T08:27:06.211036+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:07.544 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:07 smithi016 bash[33851]: cluster 2023-10-09T08:27:06.211036+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:07 smithi094 bash[27803]: cluster 2023-10-09T08:27:06.211036+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:09 smithi094 bash[27803]: cluster 2023-10-09T08:27:08.211652+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:09.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:09 smithi016 bash[33851]: cluster 2023-10-09T08:27:08.211652+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:09.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:09 smithi016 bash[27454]: cluster 2023-10-09T08:27:08.211652+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:11 smithi094 bash[27803]: cluster 2023-10-09T08:27:10.212241+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:11 smithi094 bash[27803]: audit 2023-10-09T08:27:10.899294+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.16:0/2134970409' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]: dispatch 2023-10-09T08:27:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:11 smithi094 bash[27803]: audit 2023-10-09T08:27:10.900124+0000 mon.a (mon.0) 297 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]: dispatch 2023-10-09T08:27:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:11 smithi094 bash[27803]: audit 2023-10-09T08:27:10.905806+0000 mon.a (mon.0) 298 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]': finished 2023-10-09T08:27:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:11 smithi094 bash[27803]: cluster 2023-10-09T08:27:10.905870+0000 mon.a (mon.0) 299 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T08:27:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:11 smithi094 bash[27803]: audit 2023-10-09T08:27:10.906067+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:11.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[27454]: cluster 2023-10-09T08:27:10.212241+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:11.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[27454]: audit 2023-10-09T08:27:10.899294+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.16:0/2134970409' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]: dispatch 2023-10-09T08:27:11.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[27454]: audit 2023-10-09T08:27:10.900124+0000 mon.a (mon.0) 297 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]: dispatch 2023-10-09T08:27:11.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[27454]: audit 2023-10-09T08:27:10.905806+0000 mon.a (mon.0) 298 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]': finished 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[27454]: cluster 2023-10-09T08:27:10.905870+0000 mon.a (mon.0) 299 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[27454]: audit 2023-10-09T08:27:10.906067+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[33851]: cluster 2023-10-09T08:27:10.212241+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[33851]: audit 2023-10-09T08:27:10.899294+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.16:0/2134970409' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]: dispatch 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[33851]: audit 2023-10-09T08:27:10.900124+0000 mon.a (mon.0) 297 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]: dispatch 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[33851]: audit 2023-10-09T08:27:10.905806+0000 mon.a (mon.0) 298 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "19af46cb-b0d8-410d-bf62-d4352bafb831"}]': finished 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[33851]: cluster 2023-10-09T08:27:10.905870+0000 mon.a (mon.0) 299 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T08:27:11.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:11 smithi016 bash[33851]: audit 2023-10-09T08:27:10.906067+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:12.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:12 smithi094 bash[27803]: audit 2023-10-09T08:27:11.632016+0000 mon.a (mon.0) 301 : audit [DBG] from='client.? 172.21.15.16:0/257197911' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:27:12.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:12 smithi016 bash[27454]: audit 2023-10-09T08:27:11.632016+0000 mon.a (mon.0) 301 : audit [DBG] from='client.? 172.21.15.16:0/257197911' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:27:12.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:12 smithi016 bash[33851]: audit 2023-10-09T08:27:11.632016+0000 mon.a (mon.0) 301 : audit [DBG] from='client.? 172.21.15.16:0/257197911' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:27:13.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:13 smithi094 bash[27803]: cluster 2023-10-09T08:27:12.212758+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:13.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:13 smithi016 bash[27454]: cluster 2023-10-09T08:27:12.212758+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:13.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:13 smithi016 bash[33851]: cluster 2023-10-09T08:27:12.212758+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:15 smithi094 bash[27803]: cluster 2023-10-09T08:27:14.213346+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:15.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:15 smithi016 bash[27454]: cluster 2023-10-09T08:27:14.213346+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:15.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:15 smithi016 bash[33851]: cluster 2023-10-09T08:27:14.213346+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:17 smithi094 bash[27803]: cluster 2023-10-09T08:27:16.213878+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:17.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:17 smithi016 bash[27454]: cluster 2023-10-09T08:27:16.213878+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:17.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:17 smithi016 bash[33851]: cluster 2023-10-09T08:27:16.213878+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:19 smithi094 bash[27803]: cluster 2023-10-09T08:27:18.214435+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:19.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:19 smithi016 bash[27454]: cluster 2023-10-09T08:27:18.214435+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:19.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:19 smithi016 bash[33851]: cluster 2023-10-09T08:27:18.214435+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:21.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:21 smithi094 bash[27803]: cluster 2023-10-09T08:27:20.214928+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:21.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:21 smithi016 bash[27454]: cluster 2023-10-09T08:27:20.214928+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:21.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:21 smithi016 bash[33851]: cluster 2023-10-09T08:27:20.214928+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:23.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:23 smithi016 bash[27454]: cluster 2023-10-09T08:27:22.215282+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:23.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:23 smithi016 bash[33851]: cluster 2023-10-09T08:27:22.215282+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:23 smithi094 bash[27803]: cluster 2023-10-09T08:27:22.215282+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:24.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:24 smithi094 bash[27803]: audit 2023-10-09T08:27:23.680557+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:27:24.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:24 smithi094 bash[27803]: audit 2023-10-09T08:27:23.682251+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:24 smithi016 bash[27454]: audit 2023-10-09T08:27:23.680557+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:27:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:24 smithi016 bash[27454]: audit 2023-10-09T08:27:23.682251+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:24.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:24 smithi016 bash[33851]: audit 2023-10-09T08:27:23.680557+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:27:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:24 smithi016 bash[33851]: audit 2023-10-09T08:27:23.682251+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:25.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:25 smithi016 bash[33851]: cephadm 2023-10-09T08:27:23.683798+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Deploying daemon osd.2 on smithi016 2023-10-09T08:27:25.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:25 smithi016 bash[33851]: cluster 2023-10-09T08:27:24.215654+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:25 smithi016 bash[27454]: cephadm 2023-10-09T08:27:23.683798+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Deploying daemon osd.2 on smithi016 2023-10-09T08:27:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:25 smithi016 bash[27454]: cluster 2023-10-09T08:27:24.215654+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:25 smithi094 bash[27803]: cephadm 2023-10-09T08:27:23.683798+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Deploying daemon osd.2 on smithi016 2023-10-09T08:27:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:25 smithi094 bash[27803]: cluster 2023-10-09T08:27:24.215654+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:27.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:27 smithi016 bash[33851]: cluster 2023-10-09T08:27:26.216186+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:27.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:27 smithi016 bash[27454]: cluster 2023-10-09T08:27:26.216186+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:27 smithi094 bash[27803]: cluster 2023-10-09T08:27:26.216186+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:29.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:29 smithi094 bash[27803]: audit 2023-10-09T08:27:28.197432+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:29.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:29 smithi094 bash[27803]: cluster 2023-10-09T08:27:28.216624+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:29.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:29 smithi094 bash[27803]: audit 2023-10-09T08:27:28.227867+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:27:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:29 smithi016 bash[27454]: audit 2023-10-09T08:27:28.197432+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:29 smithi016 bash[27454]: cluster 2023-10-09T08:27:28.216624+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:29 smithi016 bash[27454]: audit 2023-10-09T08:27:28.227867+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:27:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:29 smithi016 bash[33851]: audit 2023-10-09T08:27:28.197432+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:29 smithi016 bash[33851]: cluster 2023-10-09T08:27:28.216624+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:29 smithi016 bash[33851]: audit 2023-10-09T08:27:28.227867+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:27:31.345 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:31 smithi016 bash[27454]: cluster 2023-10-09T08:27:30.217080+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:31.346 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:31 smithi016 bash[33851]: cluster 2023-10-09T08:27:30.217080+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:31.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:31 smithi094 bash[27803]: cluster 2023-10-09T08:27:30.217080+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:33.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:33 smithi016 bash[27454]: cluster 2023-10-09T08:27:32.217555+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:33.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:33 smithi016 bash[33851]: cluster 2023-10-09T08:27:32.217555+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:33.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:33 smithi094 bash[27803]: cluster 2023-10-09T08:27:32.217555+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:34.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[27454]: audit 2023-10-09T08:27:33.560235+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:34.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[27454]: audit 2023-10-09T08:27:33.561421+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:34.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[27454]: audit 2023-10-09T08:27:33.562550+0000 mon.a (mon.0) 308 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:34.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[27454]: audit 2023-10-09T08:27:33.569136+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:34.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[27454]: cluster 2023-10-09T08:27:34.218017+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:34.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[33851]: audit 2023-10-09T08:27:33.560235+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:34.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[33851]: audit 2023-10-09T08:27:33.561421+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:34.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[33851]: audit 2023-10-09T08:27:33.562550+0000 mon.a (mon.0) 308 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:34.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[33851]: audit 2023-10-09T08:27:33.569136+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:34.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:34 smithi016 bash[33851]: cluster 2023-10-09T08:27:34.218017+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:34.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:34 smithi094 bash[27803]: audit 2023-10-09T08:27:33.560235+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:34.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:34 smithi094 bash[27803]: audit 2023-10-09T08:27:33.561421+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:34.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:34 smithi094 bash[27803]: audit 2023-10-09T08:27:33.562550+0000 mon.a (mon.0) 308 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:34.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:34 smithi094 bash[27803]: audit 2023-10-09T08:27:33.569136+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:34.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:34 smithi094 bash[27803]: cluster 2023-10-09T08:27:34.218017+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:35.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:35 smithi016 bash[27454]: audit 2023-10-09T08:27:34.955718+0000 mon.a (mon.0) 310 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:27:35.968 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:35 smithi016 bash[33851]: audit 2023-10-09T08:27:34.955718+0000 mon.a (mon.0) 310 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:27:35.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:35 smithi094 bash[27803]: audit 2023-10-09T08:27:34.955718+0000 mon.a (mon.0) 310 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:27:36.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[33851]: audit 2023-10-09T08:27:35.580664+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:27:36.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[33851]: cluster 2023-10-09T08:27:35.580799+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T08:27:36.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[33851]: audit 2023-10-09T08:27:35.581026+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:36.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[33851]: audit 2023-10-09T08:27:35.581507+0000 mon.a (mon.0) 314 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:27:36.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[33851]: cluster 2023-10-09T08:27:36.218557+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:36.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[27454]: audit 2023-10-09T08:27:35.580664+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:27:36.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[27454]: cluster 2023-10-09T08:27:35.580799+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T08:27:36.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[27454]: audit 2023-10-09T08:27:35.581026+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:36.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[27454]: audit 2023-10-09T08:27:35.581507+0000 mon.a (mon.0) 314 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:27:36.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:36 smithi016 bash[27454]: cluster 2023-10-09T08:27:36.218557+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:36.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:36 smithi094 bash[27803]: audit 2023-10-09T08:27:35.580664+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:27:36.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:36 smithi094 bash[27803]: cluster 2023-10-09T08:27:35.580799+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T08:27:36.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:36 smithi094 bash[27803]: audit 2023-10-09T08:27:35.581026+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:36.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:36 smithi094 bash[27803]: audit 2023-10-09T08:27:35.581507+0000 mon.a (mon.0) 314 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:27:36.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:36 smithi094 bash[27803]: cluster 2023-10-09T08:27:36.218557+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:37.373 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 2 on host 'smithi016' 2023-10-09T08:27:38.053 DEBUG:teuthology.orchestra.run.smithi016:osd.2> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T08:27:38.055 INFO:tasks.cephadm:Deploying osd.3 on smithi016 with /dev/vg_nvme/lv_1... 2023-10-09T08:27:38.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T08:27:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:38 smithi094 bash[27803]: audit 2023-10-09T08:27:36.581802+0000 mon.a (mon.0) 315 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:27:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:38 smithi094 bash[27803]: cluster 2023-10-09T08:27:36.581886+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T08:27:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:38 smithi094 bash[27803]: audit 2023-10-09T08:27:36.582502+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:38 smithi094 bash[27803]: audit 2023-10-09T08:27:36.590809+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:38 smithi094 bash[27803]: audit 2023-10-09T08:27:37.371293+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:38.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:38 smithi094 bash[27803]: audit 2023-10-09T08:27:37.395285+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:27:38.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[27454]: audit 2023-10-09T08:27:36.581802+0000 mon.a (mon.0) 315 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:27:38.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[27454]: cluster 2023-10-09T08:27:36.581886+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T08:27:38.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[27454]: audit 2023-10-09T08:27:36.582502+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:38.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[27454]: audit 2023-10-09T08:27:36.590809+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:38.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[27454]: audit 2023-10-09T08:27:37.371293+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:38.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[27454]: audit 2023-10-09T08:27:37.395285+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:27:38.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[33851]: audit 2023-10-09T08:27:36.581802+0000 mon.a (mon.0) 315 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:27:38.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[33851]: cluster 2023-10-09T08:27:36.581886+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T08:27:38.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[33851]: audit 2023-10-09T08:27:36.582502+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:38.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[33851]: audit 2023-10-09T08:27:36.590809+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:38.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[33851]: audit 2023-10-09T08:27:37.371293+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:38.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:38 smithi016 bash[33851]: audit 2023-10-09T08:27:37.395285+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:27:38.691 INFO:journalctl@ceph.osd.2.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:27:39.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: cluster 2023-10-09T08:27:35.963501+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:27:39.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: cluster 2023-10-09T08:27:35.963580+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:27:39.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: cluster 2023-10-09T08:27:37.591829+0000 mon.a (mon.0) 321 : cluster [INF] osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001] boot 2023-10-09T08:27:39.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: cluster 2023-10-09T08:27:37.591909+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T08:27:39.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: audit 2023-10-09T08:27:37.593043+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:39.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: audit 2023-10-09T08:27:37.593695+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:27:39.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: cluster 2023-10-09T08:27:38.218935+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: audit 2023-10-09T08:27:38.593415+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: cluster 2023-10-09T08:27:38.593565+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[33851]: audit 2023-10-09T08:27:38.595805+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: cluster 2023-10-09T08:27:35.963501+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: cluster 2023-10-09T08:27:35.963580+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: cluster 2023-10-09T08:27:37.591829+0000 mon.a (mon.0) 321 : cluster [INF] osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001] boot 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: cluster 2023-10-09T08:27:37.591909+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: audit 2023-10-09T08:27:37.593043+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:39.342 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: audit 2023-10-09T08:27:37.593695+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:27:39.343 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: cluster 2023-10-09T08:27:38.218935+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:39.343 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: audit 2023-10-09T08:27:38.593415+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T08:27:39.343 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: cluster 2023-10-09T08:27:38.593565+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T08:27:39.343 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:39 smithi016 bash[27454]: audit 2023-10-09T08:27:38.595805+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:27:39.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: cluster 2023-10-09T08:27:35.963501+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:27:39.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: cluster 2023-10-09T08:27:35.963580+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: cluster 2023-10-09T08:27:37.591829+0000 mon.a (mon.0) 321 : cluster [INF] osd.2 [v2:172.21.15.16:6818/3541024001,v1:172.21.15.16:6819/3541024001] boot 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: cluster 2023-10-09T08:27:37.591909+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: audit 2023-10-09T08:27:37.593043+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: audit 2023-10-09T08:27:37.593695+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: cluster 2023-10-09T08:27:38.218935+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: audit 2023-10-09T08:27:38.593415+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: cluster 2023-10-09T08:27:38.593565+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T08:27:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:39 smithi094 bash[27803]: audit 2023-10-09T08:27:38.595805+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:27:40.899 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:40 smithi016 bash[33851]: audit 2023-10-09T08:27:39.593219+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:27:41.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:40 smithi016 bash[33851]: cluster 2023-10-09T08:27:39.593335+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T08:27:41.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:40 smithi016 bash[33851]: cluster 2023-10-09T08:27:40.219336+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v112: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:41.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:40 smithi016 bash[27454]: audit 2023-10-09T08:27:39.593219+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:27:41.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:40 smithi016 bash[27454]: cluster 2023-10-09T08:27:39.593335+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T08:27:41.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:40 smithi016 bash[27454]: cluster 2023-10-09T08:27:40.219336+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v112: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:41.190 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:40 smithi094 bash[27803]: audit 2023-10-09T08:27:39.593219+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:27:41.190 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:40 smithi094 bash[27803]: cluster 2023-10-09T08:27:39.593335+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T08:27:41.190 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:40 smithi094 bash[27803]: cluster 2023-10-09T08:27:40.219336+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v112: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:41.622 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T08:27:41.622 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T08:27:41.622 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-10-09T08:27:41.622 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-10-09T08:27:41.622 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0132071 s, 794 MB/s 2023-10-09T08:27:41.623 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-10-09T08:27:41.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:41 smithi016 bash[27454]: cluster 2023-10-09T08:27:40.595495+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T08:27:41.894 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:41 smithi016 bash[33851]: cluster 2023-10-09T08:27:40.595495+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T08:27:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:41 smithi094 bash[27803]: cluster 2023-10-09T08:27:40.595495+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T08:27:42.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_1 2023-10-09T08:27:42.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:42 smithi016 bash[27454]: cluster 2023-10-09T08:27:42.219817+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:42.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:42 smithi016 bash[33851]: cluster 2023-10-09T08:27:42.219817+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:42.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:42 smithi094 bash[27803]: cluster 2023-10-09T08:27:42.219817+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:45 smithi094 bash[27803]: cluster 2023-10-09T08:27:44.220401+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:45.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:45 smithi016 bash[27454]: cluster 2023-10-09T08:27:44.220401+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:45.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:45 smithi016 bash[33851]: cluster 2023-10-09T08:27:44.220401+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:47.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[27454]: cluster 2023-10-09T08:27:46.220906+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:47.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[27454]: audit 2023-10-09T08:27:47.159336+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:27:47.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[27454]: audit 2023-10-09T08:27:47.164114+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:27:47.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[27454]: audit 2023-10-09T08:27:47.165305+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:47.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[33851]: cluster 2023-10-09T08:27:46.220906+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:47.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[33851]: audit 2023-10-09T08:27:47.159336+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:27:47.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[33851]: audit 2023-10-09T08:27:47.164114+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:27:47.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:47 smithi016 bash[33851]: audit 2023-10-09T08:27:47.165305+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:47.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:47 smithi094 bash[27803]: cluster 2023-10-09T08:27:46.220906+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:47.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:47 smithi094 bash[27803]: audit 2023-10-09T08:27:47.159336+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:27:47.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:47 smithi094 bash[27803]: audit 2023-10-09T08:27:47.164114+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:27:47.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:47 smithi094 bash[27803]: audit 2023-10-09T08:27:47.165305+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:48.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:48 smithi094 bash[27803]: audit 2023-10-09T08:27:47.153801+0000 mgr.y (mgr.14146) 136 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:27:48.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:48 smithi016 bash[27454]: audit 2023-10-09T08:27:47.153801+0000 mgr.y (mgr.14146) 136 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:27:48.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:48 smithi016 bash[33851]: audit 2023-10-09T08:27:47.153801+0000 mgr.y (mgr.14146) 136 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:27:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: cluster 2023-10-09T08:27:48.221409+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.619167+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.620941+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.622156+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.623050+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.628349+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.630756+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.632054+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:49 smithi094 bash[27803]: audit 2023-10-09T08:27:48.638250+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: cluster 2023-10-09T08:27:48.221409+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.619167+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.620941+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.622156+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.623050+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.628349+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.630756+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.632054+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[27454]: audit 2023-10-09T08:27:48.638250+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: cluster 2023-10-09T08:27:48.221409+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.619167+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.620941+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.622156+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.623050+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:27:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.628349+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.630756+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:27:49.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.632054+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:27:49.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:49 smithi016 bash[33851]: audit 2023-10-09T08:27:48.638250+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:27:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:50 smithi094 bash[27803]: cephadm 2023-10-09T08:27:48.610359+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:27:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:50 smithi094 bash[27803]: cephadm 2023-10-09T08:27:48.623735+0000 mgr.y (mgr.14146) 139 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 5415M 2023-10-09T08:27:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:50 smithi016 bash[27454]: cephadm 2023-10-09T08:27:48.610359+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:27:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:50 smithi016 bash[27454]: cephadm 2023-10-09T08:27:48.623735+0000 mgr.y (mgr.14146) 139 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 5415M 2023-10-09T08:27:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:50 smithi016 bash[33851]: cephadm 2023-10-09T08:27:48.610359+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:27:50.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:50 smithi016 bash[33851]: cephadm 2023-10-09T08:27:48.623735+0000 mgr.y (mgr.14146) 139 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 5415M 2023-10-09T08:27:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:51 smithi094 bash[27803]: cluster 2023-10-09T08:27:50.221953+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:51.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:51 smithi016 bash[27454]: cluster 2023-10-09T08:27:50.221953+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:51.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:51 smithi016 bash[33851]: cluster 2023-10-09T08:27:50.221953+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:53.421 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:53 smithi016 bash[27454]: cluster 2023-10-09T08:27:52.222353+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:53 smithi094 bash[27803]: cluster 2023-10-09T08:27:52.222353+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:53.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:53 smithi016 bash[33851]: cluster 2023-10-09T08:27:52.222353+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:55 smithi094 bash[27803]: cluster 2023-10-09T08:27:54.222815+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:55.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:55 smithi016 bash[27454]: cluster 2023-10-09T08:27:54.222815+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:55.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:55 smithi016 bash[33851]: cluster 2023-10-09T08:27:54.222815+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:56.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:56 smithi094 bash[27803]: audit 2023-10-09T08:27:55.514706+0000 mon.a (mon.0) 342 : audit [INF] from='client.? 172.21.15.16:0/1782964710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9ee2e425-a2d1-432d-838c-e246649de096"}]: dispatch 2023-10-09T08:27:56.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:56 smithi094 bash[27803]: audit 2023-10-09T08:27:55.521981+0000 mon.a (mon.0) 343 : audit [INF] from='client.? 172.21.15.16:0/1782964710' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9ee2e425-a2d1-432d-838c-e246649de096"}]': finished 2023-10-09T08:27:56.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:56 smithi094 bash[27803]: cluster 2023-10-09T08:27:55.522151+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T08:27:56.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:56 smithi094 bash[27803]: audit 2023-10-09T08:27:55.522334+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:27:56.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:56 smithi094 bash[27803]: audit 2023-10-09T08:27:56.295842+0000 mon.a (mon.0) 346 : audit [DBG] from='client.? 172.21.15.16:0/2290678810' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:27:56.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[27454]: audit 2023-10-09T08:27:55.514706+0000 mon.a (mon.0) 342 : audit [INF] from='client.? 172.21.15.16:0/1782964710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9ee2e425-a2d1-432d-838c-e246649de096"}]: dispatch 2023-10-09T08:27:56.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[27454]: audit 2023-10-09T08:27:55.521981+0000 mon.a (mon.0) 343 : audit [INF] from='client.? 172.21.15.16:0/1782964710' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9ee2e425-a2d1-432d-838c-e246649de096"}]': finished 2023-10-09T08:27:56.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[27454]: cluster 2023-10-09T08:27:55.522151+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T08:27:56.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[27454]: audit 2023-10-09T08:27:55.522334+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:27:56.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[27454]: audit 2023-10-09T08:27:56.295842+0000 mon.a (mon.0) 346 : audit [DBG] from='client.? 172.21.15.16:0/2290678810' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:27:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[33851]: audit 2023-10-09T08:27:55.514706+0000 mon.a (mon.0) 342 : audit [INF] from='client.? 172.21.15.16:0/1782964710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9ee2e425-a2d1-432d-838c-e246649de096"}]: dispatch 2023-10-09T08:27:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[33851]: audit 2023-10-09T08:27:55.521981+0000 mon.a (mon.0) 343 : audit [INF] from='client.? 172.21.15.16:0/1782964710' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9ee2e425-a2d1-432d-838c-e246649de096"}]': finished 2023-10-09T08:27:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[33851]: cluster 2023-10-09T08:27:55.522151+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T08:27:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[33851]: audit 2023-10-09T08:27:55.522334+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:27:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:56 smithi016 bash[33851]: audit 2023-10-09T08:27:56.295842+0000 mon.a (mon.0) 346 : audit [DBG] from='client.? 172.21.15.16:0/2290678810' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:27:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:57 smithi094 bash[27803]: cluster 2023-10-09T08:27:56.223238+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:57.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:57 smithi016 bash[33851]: cluster 2023-10-09T08:27:56.223238+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:57.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:57 smithi016 bash[27454]: cluster 2023-10-09T08:27:56.223238+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:27:59 smithi094 bash[27803]: cluster 2023-10-09T08:27:58.223810+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:59.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:27:59 smithi016 bash[27454]: cluster 2023-10-09T08:27:58.223810+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:27:59.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:27:59 smithi016 bash[33851]: cluster 2023-10-09T08:27:58.223810+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:01.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:01 smithi094 bash[27803]: cluster 2023-10-09T08:28:00.224368+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:01.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:01 smithi016 bash[33851]: cluster 2023-10-09T08:28:00.224368+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:01.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:01 smithi016 bash[27454]: cluster 2023-10-09T08:28:00.224368+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:03.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:03 smithi094 bash[27803]: cluster 2023-10-09T08:28:02.225008+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:03.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:03 smithi016 bash[33851]: cluster 2023-10-09T08:28:02.225008+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:03.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:03 smithi016 bash[27454]: cluster 2023-10-09T08:28:02.225008+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:05.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:05 smithi016 bash[27454]: cluster 2023-10-09T08:28:04.225683+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:05.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:05 smithi016 bash[33851]: cluster 2023-10-09T08:28:04.225683+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:05 smithi094 bash[27803]: cluster 2023-10-09T08:28:04.225683+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:07.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:07 smithi016 bash[27454]: cluster 2023-10-09T08:28:06.226256+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:07.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:07 smithi016 bash[33851]: cluster 2023-10-09T08:28:06.226256+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:07 smithi094 bash[27803]: cluster 2023-10-09T08:28:06.226256+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:08 smithi094 bash[27803]: audit 2023-10-09T08:28:08.161580+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:28:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:08 smithi094 bash[27803]: audit 2023-10-09T08:28:08.163175+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:08.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:08 smithi016 bash[27454]: audit 2023-10-09T08:28:08.161580+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:28:08.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:08 smithi016 bash[27454]: audit 2023-10-09T08:28:08.163175+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:08 smithi016 bash[33851]: audit 2023-10-09T08:28:08.161580+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:28:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:08 smithi016 bash[33851]: audit 2023-10-09T08:28:08.163175+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:09.564 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:09 smithi016 bash[27454]: cephadm 2023-10-09T08:28:08.164283+0000 mgr.y (mgr.14146) 149 : cephadm [INF] Deploying daemon osd.3 on smithi016 2023-10-09T08:28:09.564 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:09 smithi016 bash[27454]: cluster 2023-10-09T08:28:08.226788+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:09.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:09 smithi016 bash[33851]: cephadm 2023-10-09T08:28:08.164283+0000 mgr.y (mgr.14146) 149 : cephadm [INF] Deploying daemon osd.3 on smithi016 2023-10-09T08:28:09.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:09 smithi016 bash[33851]: cluster 2023-10-09T08:28:08.226788+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:09 smithi094 bash[27803]: cephadm 2023-10-09T08:28:08.164283+0000 mgr.y (mgr.14146) 149 : cephadm [INF] Deploying daemon osd.3 on smithi016 2023-10-09T08:28:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:09 smithi094 bash[27803]: cluster 2023-10-09T08:28:08.226788+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:11.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:11 smithi016 bash[33851]: cluster 2023-10-09T08:28:10.227155+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:11.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:11 smithi016 bash[27454]: cluster 2023-10-09T08:28:10.227155+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:11 smithi094 bash[27803]: cluster 2023-10-09T08:28:10.227155+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:13.607 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:13 smithi016 bash[27454]: cluster 2023-10-09T08:28:12.227622+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:13.607 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:13 smithi016 bash[27454]: audit 2023-10-09T08:28:12.607832+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:13.607 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:13 smithi016 bash[27454]: audit 2023-10-09T08:28:12.620406+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:13.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:13 smithi016 bash[33851]: cluster 2023-10-09T08:28:12.227622+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:13.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:13 smithi016 bash[33851]: audit 2023-10-09T08:28:12.607832+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:13.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:13 smithi016 bash[33851]: audit 2023-10-09T08:28:12.620406+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:13 smithi094 bash[27803]: cluster 2023-10-09T08:28:12.227622+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:13 smithi094 bash[27803]: audit 2023-10-09T08:28:12.607832+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:13 smithi094 bash[27803]: audit 2023-10-09T08:28:12.620406+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:15.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:15 smithi016 bash[33851]: cluster 2023-10-09T08:28:14.228147+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:15 smithi016 bash[27454]: cluster 2023-10-09T08:28:14.228147+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:15 smithi094 bash[27803]: cluster 2023-10-09T08:28:14.228147+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:17.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:17 smithi016 bash[27454]: cluster 2023-10-09T08:28:16.228603+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:17.573 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:17 smithi016 bash[33851]: cluster 2023-10-09T08:28:16.228603+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:17 smithi094 bash[27803]: cluster 2023-10-09T08:28:16.228603+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:19.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[27454]: audit 2023-10-09T08:28:18.043311+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:19.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[27454]: cluster 2023-10-09T08:28:18.229226+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:19.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[27454]: audit 2023-10-09T08:28:18.334355+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:19.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[27454]: audit 2023-10-09T08:28:18.335429+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:28:19.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[27454]: audit 2023-10-09T08:28:18.341374+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:19.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[33851]: audit 2023-10-09T08:28:18.043311+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:19.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[33851]: cluster 2023-10-09T08:28:18.229226+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:19.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[33851]: audit 2023-10-09T08:28:18.334355+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:19.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[33851]: audit 2023-10-09T08:28:18.335429+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:28:19.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:19 smithi016 bash[33851]: audit 2023-10-09T08:28:18.341374+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:19 smithi094 bash[27803]: audit 2023-10-09T08:28:18.043311+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:19 smithi094 bash[27803]: cluster 2023-10-09T08:28:18.229226+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:19.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:19 smithi094 bash[27803]: audit 2023-10-09T08:28:18.334355+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:19.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:19 smithi094 bash[27803]: audit 2023-10-09T08:28:18.335429+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:28:19.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:19 smithi094 bash[27803]: audit 2023-10-09T08:28:18.341374+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:20.310 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:20 smithi016 bash[33851]: audit 2023-10-09T08:28:19.525754+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:28:20.310 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:20 smithi016 bash[33851]: audit 2023-10-09T08:28:19.526235+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:28:20.310 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:20 smithi016 bash[27454]: audit 2023-10-09T08:28:19.525754+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:28:20.310 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:20 smithi016 bash[27454]: audit 2023-10-09T08:28:19.526235+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:28:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:20 smithi094 bash[27803]: audit 2023-10-09T08:28:19.525754+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:28:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:20 smithi094 bash[27803]: audit 2023-10-09T08:28:19.526235+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:28:21.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[33851]: audit 2023-10-09T08:28:20.062192+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[33851]: cluster 2023-10-09T08:28:20.062313+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[33851]: audit 2023-10-09T08:28:20.062519+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[33851]: audit 2023-10-09T08:28:20.063245+0000 mon.c (mon.2) 9 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[33851]: audit 2023-10-09T08:28:20.063703+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[33851]: cluster 2023-10-09T08:28:20.229789+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[27454]: audit 2023-10-09T08:28:20.062192+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[27454]: cluster 2023-10-09T08:28:20.062313+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T08:28:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[27454]: audit 2023-10-09T08:28:20.062519+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[27454]: audit 2023-10-09T08:28:20.063245+0000 mon.c (mon.2) 9 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:28:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[27454]: audit 2023-10-09T08:28:20.063703+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:28:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:21 smithi016 bash[27454]: cluster 2023-10-09T08:28:20.229789+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:21.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:21 smithi094 bash[27803]: audit 2023-10-09T08:28:20.062192+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:28:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:21 smithi094 bash[27803]: cluster 2023-10-09T08:28:20.062313+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T08:28:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:21 smithi094 bash[27803]: audit 2023-10-09T08:28:20.062519+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:21 smithi094 bash[27803]: audit 2023-10-09T08:28:20.063245+0000 mon.c (mon.2) 9 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:28:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:21 smithi094 bash[27803]: audit 2023-10-09T08:28:20.063703+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:28:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:21 smithi094 bash[27803]: cluster 2023-10-09T08:28:20.229789+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:21.641 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 3 on host 'smithi016' 2023-10-09T08:28:22.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[33851]: audit 2023-10-09T08:28:21.067922+0000 mon.a (mon.0) 360 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:28:22.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[33851]: cluster 2023-10-09T08:28:21.068218+0000 mon.a (mon.0) 361 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T08:28:22.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[33851]: audit 2023-10-09T08:28:21.069010+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:22.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[33851]: audit 2023-10-09T08:28:21.074547+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[33851]: audit 2023-10-09T08:28:21.638917+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[33851]: audit 2023-10-09T08:28:21.666685+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:22.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[27454]: audit 2023-10-09T08:28:21.067922+0000 mon.a (mon.0) 360 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:28:22.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[27454]: cluster 2023-10-09T08:28:21.068218+0000 mon.a (mon.0) 361 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T08:28:22.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[27454]: audit 2023-10-09T08:28:21.069010+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[27454]: audit 2023-10-09T08:28:21.074547+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[27454]: audit 2023-10-09T08:28:21.638917+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:22 smithi016 bash[27454]: audit 2023-10-09T08:28:21.666685+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:22.350 DEBUG:teuthology.orchestra.run.smithi016:osd.3> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T08:28:22.353 INFO:tasks.cephadm:Deploying osd.4 on smithi094 with /dev/vg_nvme/lv_4... 2023-10-09T08:28:22.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T08:28:22.367 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:22 smithi094 bash[27803]: audit 2023-10-09T08:28:21.067922+0000 mon.a (mon.0) 360 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-10-09T08:28:22.367 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:22 smithi094 bash[27803]: cluster 2023-10-09T08:28:21.068218+0000 mon.a (mon.0) 361 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T08:28:22.367 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:22 smithi094 bash[27803]: audit 2023-10-09T08:28:21.069010+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:22.367 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:22 smithi094 bash[27803]: audit 2023-10-09T08:28:21.074547+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:22.367 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:22 smithi094 bash[27803]: audit 2023-10-09T08:28:21.638917+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:22.367 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:22 smithi094 bash[27803]: audit 2023-10-09T08:28:21.666685+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:22.399 INFO:journalctl@ceph.osd.3.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:28:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:23 smithi094 bash[27803]: cluster 2023-10-09T08:28:20.549892+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:28:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:23 smithi094 bash[27803]: cluster 2023-10-09T08:28:20.549977+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:28:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:23 smithi094 bash[27803]: audit 2023-10-09T08:28:22.079884+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:23 smithi094 bash[27803]: cluster 2023-10-09T08:28:22.083442+0000 mon.a (mon.0) 367 : cluster [INF] osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952] boot 2023-10-09T08:28:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:23 smithi094 bash[27803]: cluster 2023-10-09T08:28:22.083486+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T08:28:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:23 smithi094 bash[27803]: audit 2023-10-09T08:28:22.083663+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:23 smithi094 bash[27803]: cluster 2023-10-09T08:28:22.230250+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:23.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[27454]: cluster 2023-10-09T08:28:20.549892+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[27454]: cluster 2023-10-09T08:28:20.549977+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[27454]: audit 2023-10-09T08:28:22.079884+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[27454]: cluster 2023-10-09T08:28:22.083442+0000 mon.a (mon.0) 367 : cluster [INF] osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952] boot 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[27454]: cluster 2023-10-09T08:28:22.083486+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[27454]: audit 2023-10-09T08:28:22.083663+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[27454]: cluster 2023-10-09T08:28:22.230250+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[33851]: cluster 2023-10-09T08:28:20.549892+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[33851]: cluster 2023-10-09T08:28:20.549977+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:28:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[33851]: audit 2023-10-09T08:28:22.079884+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:23.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[33851]: cluster 2023-10-09T08:28:22.083442+0000 mon.a (mon.0) 367 : cluster [INF] osd.3 [v2:172.21.15.16:6826/3124750952,v1:172.21.15.16:6827/3124750952] boot 2023-10-09T08:28:23.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[33851]: cluster 2023-10-09T08:28:22.083486+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T08:28:23.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[33851]: audit 2023-10-09T08:28:22.083663+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:28:23.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:23 smithi016 bash[33851]: cluster 2023-10-09T08:28:22.230250+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:28:24.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:24 smithi094 bash[27803]: cluster 2023-10-09T08:28:23.090065+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T08:28:24.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:24 smithi016 bash[27454]: cluster 2023-10-09T08:28:23.090065+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T08:28:24.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:24 smithi016 bash[33851]: cluster 2023-10-09T08:28:23.090065+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T08:28:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:25 smithi094 bash[27803]: cluster 2023-10-09T08:28:24.230740+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:25 smithi016 bash[27454]: cluster 2023-10-09T08:28:24.230740+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:25.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:25 smithi016 bash[33851]: cluster 2023-10-09T08:28:24.230740+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:26.254 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T08:28:26.254 INFO:teuthology.orchestra.run.smithi094.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T08:28:26.254 INFO:teuthology.orchestra.run.smithi094.stderr: stderr: 10+0 records in 2023-10-09T08:28:26.254 INFO:teuthology.orchestra.run.smithi094.stderr:10+0 records out 2023-10-09T08:28:26.254 INFO:teuthology.orchestra.run.smithi094.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0138673 s, 756 MB/s 2023-10-09T08:28:26.255 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping successful for: 2023-10-09T08:28:27.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi094:vg_nvme/lv_4 2023-10-09T08:28:27.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:27 smithi094 bash[27803]: cluster 2023-10-09T08:28:26.231273+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:27.540 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:27 smithi016 bash[33851]: cluster 2023-10-09T08:28:26.231273+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:27.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:27 smithi016 bash[27454]: cluster 2023-10-09T08:28:26.231273+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:29.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:29 smithi094 bash[27803]: cluster 2023-10-09T08:28:28.231831+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:29 smithi016 bash[33851]: cluster 2023-10-09T08:28:28.231831+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:29 smithi016 bash[27454]: cluster 2023-10-09T08:28:28.231831+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: cluster 2023-10-09T08:28:30.232314+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.416011+0000 mgr.y (mgr.14146) 162 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:28:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.422064+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:28:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.427531+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:28:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.429021+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.806422+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.808315+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.809725+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.811038+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.812242+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.819071+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.820794+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.821945+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:28:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:31 smithi094 bash[27803]: audit 2023-10-09T08:28:30.829689+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: cluster 2023-10-09T08:28:30.232314+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:31.507 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.416011+0000 mgr.y (mgr.14146) 162 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:28:31.507 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.422064+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:28:31.507 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.427531+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:28:31.507 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.429021+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:31.507 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.806422+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.808315+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.809725+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.811038+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.812242+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.819071+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.820794+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.821945+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[27454]: audit 2023-10-09T08:28:30.829689+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.508 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: cluster 2023-10-09T08:28:30.232314+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.416011+0000 mgr.y (mgr.14146) 162 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.422064+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.427531+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.429021+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.806422+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.808315+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.809725+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.811038+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.812242+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.819071+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.820794+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:31.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.821945+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:28:31.510 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:31 smithi016 bash[33851]: audit 2023-10-09T08:28:30.829689+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:32.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:32 smithi094 bash[27803]: cephadm 2023-10-09T08:28:30.796148+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:28:32.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:32 smithi094 bash[27803]: cephadm 2023-10-09T08:28:30.813108+0000 mgr.y (mgr.14146) 164 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 4061M 2023-10-09T08:28:32.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:32 smithi016 bash[27454]: cephadm 2023-10-09T08:28:30.796148+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:28:32.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:32 smithi016 bash[27454]: cephadm 2023-10-09T08:28:30.813108+0000 mgr.y (mgr.14146) 164 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 4061M 2023-10-09T08:28:32.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:32 smithi016 bash[33851]: cephadm 2023-10-09T08:28:30.796148+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Detected new or changed devices on smithi016 2023-10-09T08:28:32.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:32 smithi016 bash[33851]: cephadm 2023-10-09T08:28:30.813108+0000 mgr.y (mgr.14146) 164 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 4061M 2023-10-09T08:28:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:33 smithi094 bash[27803]: cluster 2023-10-09T08:28:32.232922+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:33.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:33 smithi016 bash[33851]: cluster 2023-10-09T08:28:32.232922+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:33.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:33 smithi016 bash[27454]: cluster 2023-10-09T08:28:32.232922+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:35.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:35 smithi094 bash[27803]: cluster 2023-10-09T08:28:34.233635+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:35.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:35 smithi016 bash[27454]: cluster 2023-10-09T08:28:34.233635+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:35.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:35 smithi016 bash[33851]: cluster 2023-10-09T08:28:34.233635+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:37 smithi094 bash[27803]: cluster 2023-10-09T08:28:36.234285+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:37.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:37 smithi016 bash[27454]: cluster 2023-10-09T08:28:36.234285+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:37.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:37 smithi016 bash[33851]: cluster 2023-10-09T08:28:36.234285+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:39 smithi094 bash[27803]: cluster 2023-10-09T08:28:38.234849+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:39.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:39 smithi094 bash[27803]: audit 2023-10-09T08:28:38.999449+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.94:0/3122939385' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]: dispatch 2023-10-09T08:28:39.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:39 smithi094 bash[27803]: audit 2023-10-09T08:28:39.000464+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]: dispatch 2023-10-09T08:28:39.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:39 smithi094 bash[27803]: audit 2023-10-09T08:28:39.008793+0000 mon.a (mon.0) 384 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]': finished 2023-10-09T08:28:39.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:39 smithi094 bash[27803]: cluster 2023-10-09T08:28:39.009018+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T08:28:39.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:39 smithi094 bash[27803]: audit 2023-10-09T08:28:39.009295+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:28:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[27454]: cluster 2023-10-09T08:28:38.234849+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[27454]: audit 2023-10-09T08:28:38.999449+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.94:0/3122939385' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]: dispatch 2023-10-09T08:28:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[27454]: audit 2023-10-09T08:28:39.000464+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]: dispatch 2023-10-09T08:28:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[27454]: audit 2023-10-09T08:28:39.008793+0000 mon.a (mon.0) 384 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]': finished 2023-10-09T08:28:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[27454]: cluster 2023-10-09T08:28:39.009018+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T08:28:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[27454]: audit 2023-10-09T08:28:39.009295+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:28:39.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[33851]: cluster 2023-10-09T08:28:38.234849+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[33851]: audit 2023-10-09T08:28:38.999449+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.94:0/3122939385' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]: dispatch 2023-10-09T08:28:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[33851]: audit 2023-10-09T08:28:39.000464+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]: dispatch 2023-10-09T08:28:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[33851]: audit 2023-10-09T08:28:39.008793+0000 mon.a (mon.0) 384 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44c51083-8ea0-4d32-8849-c5524f94afa4"}]': finished 2023-10-09T08:28:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[33851]: cluster 2023-10-09T08:28:39.009018+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T08:28:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:39 smithi016 bash[33851]: audit 2023-10-09T08:28:39.009295+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:28:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:40 smithi094 bash[27803]: audit 2023-10-09T08:28:39.693973+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.94:0/1360903543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:28:40.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:40 smithi016 bash[33851]: audit 2023-10-09T08:28:39.693973+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.94:0/1360903543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:28:40.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:40 smithi016 bash[27454]: audit 2023-10-09T08:28:39.693973+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.94:0/1360903543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:28:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:41 smithi094 bash[27803]: cluster 2023-10-09T08:28:40.235382+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:41.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:41 smithi016 bash[27454]: cluster 2023-10-09T08:28:40.235382+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:41.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:41 smithi016 bash[33851]: cluster 2023-10-09T08:28:40.235382+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:43 smithi094 bash[27803]: cluster 2023-10-09T08:28:42.235864+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:43.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:43 smithi016 bash[27454]: cluster 2023-10-09T08:28:42.235864+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:43.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:43 smithi016 bash[33851]: cluster 2023-10-09T08:28:42.235864+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:45 smithi094 bash[27803]: cluster 2023-10-09T08:28:44.236402+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:45.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:45 smithi016 bash[33851]: cluster 2023-10-09T08:28:44.236402+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:45.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:45 smithi016 bash[27454]: cluster 2023-10-09T08:28:44.236402+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:47.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:47 smithi094 bash[27803]: cluster 2023-10-09T08:28:46.236897+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:47 smithi016 bash[27454]: cluster 2023-10-09T08:28:46.236897+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:47.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:47 smithi016 bash[33851]: cluster 2023-10-09T08:28:46.236897+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:49 smithi094 bash[27803]: cluster 2023-10-09T08:28:48.237472+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:49 smithi016 bash[27454]: cluster 2023-10-09T08:28:48.237472+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:49 smithi016 bash[33851]: cluster 2023-10-09T08:28:48.237472+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:51.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:51 smithi094 bash[27803]: cluster 2023-10-09T08:28:50.238035+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:51 smithi016 bash[27454]: cluster 2023-10-09T08:28:50.238035+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:51.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:51 smithi016 bash[33851]: cluster 2023-10-09T08:28:50.238035+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:52.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:52 smithi094 bash[27803]: audit 2023-10-09T08:28:51.779010+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:28:52.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:52 smithi094 bash[27803]: audit 2023-10-09T08:28:51.780855+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:52 smithi016 bash[27454]: audit 2023-10-09T08:28:51.779010+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:28:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:52 smithi016 bash[27454]: audit 2023-10-09T08:28:51.780855+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:52.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:52 smithi016 bash[33851]: audit 2023-10-09T08:28:51.779010+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:28:52.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:52 smithi016 bash[33851]: audit 2023-10-09T08:28:51.780855+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:28:53.204 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:53 smithi094 bash[27803]: cephadm 2023-10-09T08:28:51.782121+0000 mgr.y (mgr.14146) 175 : cephadm [INF] Deploying daemon osd.4 on smithi094 2023-10-09T08:28:53.204 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:53 smithi094 bash[27803]: cluster 2023-10-09T08:28:52.238543+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:53.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:53 smithi016 bash[27454]: cephadm 2023-10-09T08:28:51.782121+0000 mgr.y (mgr.14146) 175 : cephadm [INF] Deploying daemon osd.4 on smithi094 2023-10-09T08:28:53.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:53 smithi016 bash[27454]: cluster 2023-10-09T08:28:52.238543+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:53.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:53 smithi016 bash[33851]: cephadm 2023-10-09T08:28:51.782121+0000 mgr.y (mgr.14146) 175 : cephadm [INF] Deploying daemon osd.4 on smithi094 2023-10-09T08:28:53.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:53 smithi016 bash[33851]: cluster 2023-10-09T08:28:52.238543+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:55.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:55 smithi094 bash[27803]: cluster 2023-10-09T08:28:54.239196+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:55.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:55 smithi016 bash[27454]: cluster 2023-10-09T08:28:54.239196+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:55.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:55 smithi016 bash[33851]: cluster 2023-10-09T08:28:54.239196+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:57 smithi094 bash[27803]: cluster 2023-10-09T08:28:56.239853+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:57 smithi094 bash[27803]: audit 2023-10-09T08:28:56.484494+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:57 smithi094 bash[27803]: audit 2023-10-09T08:28:56.519362+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:57 smithi016 bash[27454]: cluster 2023-10-09T08:28:56.239853+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:57 smithi016 bash[27454]: audit 2023-10-09T08:28:56.484494+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:57 smithi016 bash[27454]: audit 2023-10-09T08:28:56.519362+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:57 smithi016 bash[33851]: cluster 2023-10-09T08:28:56.239853+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:57 smithi016 bash[33851]: audit 2023-10-09T08:28:56.484494+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:28:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:57 smithi016 bash[33851]: audit 2023-10-09T08:28:56.519362+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:28:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:28:59 smithi094 bash[27803]: cluster 2023-10-09T08:28:58.240413+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:59.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:28:59 smithi016 bash[27454]: cluster 2023-10-09T08:28:58.240413+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:28:59.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:28:59 smithi016 bash[33851]: cluster 2023-10-09T08:28:58.240413+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:01.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:01 smithi094 bash[27803]: cluster 2023-10-09T08:29:00.240950+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:01.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:01 smithi016 bash[27454]: cluster 2023-10-09T08:29:00.240950+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:01.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:01 smithi016 bash[33851]: cluster 2023-10-09T08:29:00.240950+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:03.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:03 smithi094 bash[27803]: cluster 2023-10-09T08:29:02.241463+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:03.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:03 smithi094 bash[27803]: audit 2023-10-09T08:29:02.697889+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:03.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:03 smithi094 bash[27803]: audit 2023-10-09T08:29:02.967662+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:03.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:03 smithi094 bash[27803]: audit 2023-10-09T08:29:02.969789+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:03.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:03 smithi094 bash[27803]: audit 2023-10-09T08:29:02.978234+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[27454]: cluster 2023-10-09T08:29:02.241463+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[27454]: audit 2023-10-09T08:29:02.697889+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[27454]: audit 2023-10-09T08:29:02.967662+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[27454]: audit 2023-10-09T08:29:02.969789+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[27454]: audit 2023-10-09T08:29:02.978234+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:03.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[33851]: cluster 2023-10-09T08:29:02.241463+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:03.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[33851]: audit 2023-10-09T08:29:02.697889+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:03.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[33851]: audit 2023-10-09T08:29:02.967662+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:03.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[33851]: audit 2023-10-09T08:29:02.969789+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:03.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:03 smithi016 bash[33851]: audit 2023-10-09T08:29:02.978234+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:04 smithi094 bash[27803]: audit 2023-10-09T08:29:03.411088+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:29:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:04 smithi016 bash[27454]: audit 2023-10-09T08:29:03.411088+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:29:04.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:04 smithi016 bash[33851]: audit 2023-10-09T08:29:03.411088+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:29:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:05 smithi094 bash[27803]: cluster 2023-10-09T08:29:04.241987+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:05.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:05 smithi094 bash[27803]: audit 2023-10-09T08:29:04.310282+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T08:29:05.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:05 smithi094 bash[27803]: cluster 2023-10-09T08:29:04.310383+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T08:29:05.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:05 smithi094 bash[27803]: audit 2023-10-09T08:29:04.310538+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:05.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:05 smithi094 bash[27803]: audit 2023-10-09T08:29:04.311181+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:05.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[27454]: cluster 2023-10-09T08:29:04.241987+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:05.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[27454]: audit 2023-10-09T08:29:04.310282+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T08:29:05.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[27454]: cluster 2023-10-09T08:29:04.310383+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T08:29:05.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[27454]: audit 2023-10-09T08:29:04.310538+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:05.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[27454]: audit 2023-10-09T08:29:04.311181+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[33851]: cluster 2023-10-09T08:29:04.241987+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:05.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[33851]: audit 2023-10-09T08:29:04.310282+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T08:29:05.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[33851]: cluster 2023-10-09T08:29:04.310383+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T08:29:05.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[33851]: audit 2023-10-09T08:29:04.310538+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:05.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:05 smithi016 bash[33851]: audit 2023-10-09T08:29:04.311181+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:06 smithi094 bash[27803]: audit 2023-10-09T08:29:05.314100+0000 mon.a (mon.0) 400 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:29:06.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:06 smithi094 bash[27803]: cluster 2023-10-09T08:29:05.314242+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T08:29:06.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:06 smithi094 bash[27803]: audit 2023-10-09T08:29:05.315102+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:06.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:06 smithi094 bash[27803]: audit 2023-10-09T08:29:05.321146+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[27454]: audit 2023-10-09T08:29:05.314100+0000 mon.a (mon.0) 400 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:29:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[27454]: cluster 2023-10-09T08:29:05.314242+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T08:29:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[27454]: audit 2023-10-09T08:29:05.315102+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[27454]: audit 2023-10-09T08:29:05.321146+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[33851]: audit 2023-10-09T08:29:05.314100+0000 mon.a (mon.0) 400 : audit [INF] from='osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:29:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[33851]: cluster 2023-10-09T08:29:05.314242+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T08:29:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[33851]: audit 2023-10-09T08:29:05.315102+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:06.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:06 smithi016 bash[33851]: audit 2023-10-09T08:29:05.321146+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:06.856 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 4 on host 'smithi094' 2023-10-09T08:29:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: cluster 2023-10-09T08:29:04.411850+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:29:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: cluster 2023-10-09T08:29:04.411943+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:29:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: cluster 2023-10-09T08:29:06.242485+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: cluster 2023-10-09T08:29:06.317229+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189] boot 2023-10-09T08:29:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: cluster 2023-10-09T08:29:06.317350+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T08:29:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: audit 2023-10-09T08:29:06.318809+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: audit 2023-10-09T08:29:06.853790+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: audit 2023-10-09T08:29:06.857135+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:07 smithi094 bash[27803]: cluster 2023-10-09T08:29:07.317696+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T08:29:07.719 DEBUG:teuthology.orchestra.run.smithi094:osd.4> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T08:29:07.722 INFO:tasks.cephadm:Deploying osd.5 on smithi094 with /dev/vg_nvme/lv_3... 2023-10-09T08:29:07.722 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T08:29:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: cluster 2023-10-09T08:29:04.411850+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:29:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: cluster 2023-10-09T08:29:04.411943+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:29:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: cluster 2023-10-09T08:29:06.242485+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: cluster 2023-10-09T08:29:06.317229+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189] boot 2023-10-09T08:29:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: cluster 2023-10-09T08:29:06.317350+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T08:29:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: audit 2023-10-09T08:29:06.318809+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: audit 2023-10-09T08:29:06.853790+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: audit 2023-10-09T08:29:06.857135+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[27454]: cluster 2023-10-09T08:29:07.317696+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T08:29:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: cluster 2023-10-09T08:29:04.411850+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:29:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: cluster 2023-10-09T08:29:04.411943+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:29:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: cluster 2023-10-09T08:29:06.242485+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:29:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: cluster 2023-10-09T08:29:06.317229+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189] boot 2023-10-09T08:29:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: cluster 2023-10-09T08:29:06.317350+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T08:29:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: audit 2023-10-09T08:29:06.318809+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:29:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: audit 2023-10-09T08:29:06.853790+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: audit 2023-10-09T08:29:06.857135+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:07 smithi016 bash[33851]: cluster 2023-10-09T08:29:07.317696+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T08:29:07.905 INFO:journalctl@ceph.osd.4.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:29:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:09 smithi094 bash[27803]: cluster 2023-10-09T08:29:08.243029+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:09.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:09 smithi094 bash[27803]: cluster 2023-10-09T08:29:08.320582+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T08:29:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:09 smithi016 bash[27454]: cluster 2023-10-09T08:29:08.243029+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:09 smithi016 bash[27454]: cluster 2023-10-09T08:29:08.320582+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T08:29:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:09 smithi016 bash[33851]: cluster 2023-10-09T08:29:08.243029+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:09 smithi016 bash[33851]: cluster 2023-10-09T08:29:08.320582+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T08:29:11.461 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T08:29:11.461 INFO:teuthology.orchestra.run.smithi094.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T08:29:11.461 INFO:teuthology.orchestra.run.smithi094.stderr: stderr: 10+0 records in 2023-10-09T08:29:11.461 INFO:teuthology.orchestra.run.smithi094.stderr:10+0 records out 2023-10-09T08:29:11.461 INFO:teuthology.orchestra.run.smithi094.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124017 s, 846 MB/s 2023-10-09T08:29:11.461 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping successful for: 2023-10-09T08:29:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:11 smithi094 bash[27803]: cluster 2023-10-09T08:29:10.243716+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:11.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:11 smithi016 bash[27454]: cluster 2023-10-09T08:29:10.243716+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:11.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:11 smithi016 bash[33851]: cluster 2023-10-09T08:29:10.243716+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:12.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi094:vg_nvme/lv_3 2023-10-09T08:29:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:13 smithi094 bash[27803]: cluster 2023-10-09T08:29:12.244326+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v170: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:13 smithi016 bash[27454]: cluster 2023-10-09T08:29:12.244326+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v170: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:13 smithi016 bash[33851]: cluster 2023-10-09T08:29:12.244326+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v170: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:15.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:15 smithi094 bash[27803]: cluster 2023-10-09T08:29:14.244895+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:15.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:15 smithi016 bash[27454]: cluster 2023-10-09T08:29:14.244895+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:15 smithi016 bash[33851]: cluster 2023-10-09T08:29:14.244895+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:17 smithi094 bash[27803]: cluster 2023-10-09T08:29:16.245515+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:17 smithi016 bash[27454]: cluster 2023-10-09T08:29:16.245515+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:17 smithi016 bash[33851]: cluster 2023-10-09T08:29:16.245515+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:18.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:18 smithi094 bash[27803]: audit 2023-10-09T08:29:17.760962+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:29:18.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:18 smithi094 bash[27803]: audit 2023-10-09T08:29:17.765086+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:29:18.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:18 smithi094 bash[27803]: audit 2023-10-09T08:29:17.766774+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:18 smithi016 bash[27454]: audit 2023-10-09T08:29:17.760962+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:29:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:18 smithi016 bash[27454]: audit 2023-10-09T08:29:17.765086+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:29:18.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:18 smithi016 bash[27454]: audit 2023-10-09T08:29:17.766774+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:18 smithi016 bash[33851]: audit 2023-10-09T08:29:17.760962+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:29:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:18 smithi016 bash[33851]: audit 2023-10-09T08:29:17.765086+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:29:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:18 smithi016 bash[33851]: audit 2023-10-09T08:29:17.766774+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: audit 2023-10-09T08:29:17.754326+0000 mgr.y (mgr.14146) 189 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: cluster 2023-10-09T08:29:18.246225+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: audit 2023-10-09T08:29:19.156633+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: audit 2023-10-09T08:29:19.158094+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: audit 2023-10-09T08:29:19.164111+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: audit 2023-10-09T08:29:19.165578+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: audit 2023-10-09T08:29:19.166709+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:19 smithi094 bash[27803]: audit 2023-10-09T08:29:19.172588+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: audit 2023-10-09T08:29:17.754326+0000 mgr.y (mgr.14146) 189 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:29:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: cluster 2023-10-09T08:29:18.246225+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: audit 2023-10-09T08:29:19.156633+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: audit 2023-10-09T08:29:19.158094+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: audit 2023-10-09T08:29:19.164111+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: audit 2023-10-09T08:29:19.165578+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: audit 2023-10-09T08:29:19.166709+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[27454]: audit 2023-10-09T08:29:19.172588+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: audit 2023-10-09T08:29:17.754326+0000 mgr.y (mgr.14146) 189 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:29:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: cluster 2023-10-09T08:29:18.246225+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: audit 2023-10-09T08:29:19.156633+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: audit 2023-10-09T08:29:19.158094+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:29:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: audit 2023-10-09T08:29:19.164111+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: audit 2023-10-09T08:29:19.165578+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: audit 2023-10-09T08:29:19.166709+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:19 smithi016 bash[33851]: audit 2023-10-09T08:29:19.172588+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:20 smithi094 bash[27803]: cephadm 2023-10-09T08:29:19.147014+0000 mgr.y (mgr.14146) 191 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:29:20.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:20 smithi094 bash[27803]: cephadm 2023-10-09T08:29:19.159142+0000 mgr.y (mgr.14146) 192 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 17265M 2023-10-09T08:29:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:20 smithi016 bash[27454]: cephadm 2023-10-09T08:29:19.147014+0000 mgr.y (mgr.14146) 191 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:29:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:20 smithi016 bash[27454]: cephadm 2023-10-09T08:29:19.159142+0000 mgr.y (mgr.14146) 192 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 17265M 2023-10-09T08:29:20.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:20 smithi016 bash[33851]: cephadm 2023-10-09T08:29:19.147014+0000 mgr.y (mgr.14146) 191 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:29:20.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:20 smithi016 bash[33851]: cephadm 2023-10-09T08:29:19.159142+0000 mgr.y (mgr.14146) 192 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 17265M 2023-10-09T08:29:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:21 smithi094 bash[27803]: cluster 2023-10-09T08:29:20.246802+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:21.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:21 smithi016 bash[27454]: cluster 2023-10-09T08:29:20.246802+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:21 smithi016 bash[33851]: cluster 2023-10-09T08:29:20.246802+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:23.618 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:23 smithi094 bash[27803]: cluster 2023-10-09T08:29:22.247373+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:23 smithi016 bash[27454]: cluster 2023-10-09T08:29:22.247373+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:23 smithi016 bash[33851]: cluster 2023-10-09T08:29:22.247373+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:25 smithi094 bash[27803]: cluster 2023-10-09T08:29:24.247997+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:25.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:25 smithi016 bash[33851]: cluster 2023-10-09T08:29:24.247997+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:25 smithi016 bash[27454]: cluster 2023-10-09T08:29:24.247997+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:26.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:26 smithi094 bash[27803]: audit 2023-10-09T08:29:26.264499+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.94:0/1563366637' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]: dispatch 2023-10-09T08:29:26.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:26 smithi094 bash[27803]: audit 2023-10-09T08:29:26.265291+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]: dispatch 2023-10-09T08:29:26.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:26 smithi094 bash[27803]: audit 2023-10-09T08:29:26.273951+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]': finished 2023-10-09T08:29:26.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:26 smithi094 bash[27803]: cluster 2023-10-09T08:29:26.274090+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T08:29:26.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:26 smithi094 bash[27803]: audit 2023-10-09T08:29:26.274397+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[27454]: audit 2023-10-09T08:29:26.264499+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.94:0/1563366637' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]: dispatch 2023-10-09T08:29:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[27454]: audit 2023-10-09T08:29:26.265291+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]: dispatch 2023-10-09T08:29:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[27454]: audit 2023-10-09T08:29:26.273951+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]': finished 2023-10-09T08:29:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[27454]: cluster 2023-10-09T08:29:26.274090+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T08:29:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[27454]: audit 2023-10-09T08:29:26.274397+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[33851]: audit 2023-10-09T08:29:26.264499+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.94:0/1563366637' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]: dispatch 2023-10-09T08:29:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[33851]: audit 2023-10-09T08:29:26.265291+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]: dispatch 2023-10-09T08:29:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[33851]: audit 2023-10-09T08:29:26.273951+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc3fc2f1-b5f5-4986-9260-98c23cd9c49e"}]': finished 2023-10-09T08:29:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[33851]: cluster 2023-10-09T08:29:26.274090+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T08:29:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:26 smithi016 bash[33851]: audit 2023-10-09T08:29:26.274397+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:27 smithi094 bash[27803]: cluster 2023-10-09T08:29:26.248454+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:27 smithi094 bash[27803]: audit 2023-10-09T08:29:26.967448+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.94:0/632490528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:29:27.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:27 smithi016 bash[33851]: cluster 2023-10-09T08:29:26.248454+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:27.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:27 smithi016 bash[33851]: audit 2023-10-09T08:29:26.967448+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.94:0/632490528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:29:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:27 smithi016 bash[27454]: cluster 2023-10-09T08:29:26.248454+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:27.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:27 smithi016 bash[27454]: audit 2023-10-09T08:29:26.967448+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.94:0/632490528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:29:29.723 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:29 smithi094 bash[27803]: cluster 2023-10-09T08:29:28.249008+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:29 smithi016 bash[27454]: cluster 2023-10-09T08:29:28.249008+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:29 smithi016 bash[33851]: cluster 2023-10-09T08:29:28.249008+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:31 smithi094 bash[27803]: cluster 2023-10-09T08:29:30.249582+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:31.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:31 smithi016 bash[27454]: cluster 2023-10-09T08:29:30.249582+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:31.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:31 smithi016 bash[33851]: cluster 2023-10-09T08:29:30.249582+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:33 smithi094 bash[27803]: cluster 2023-10-09T08:29:32.250058+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:33 smithi016 bash[27454]: cluster 2023-10-09T08:29:32.250058+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:33.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:33 smithi016 bash[33851]: cluster 2023-10-09T08:29:32.250058+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:35.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:35 smithi094 bash[27803]: cluster 2023-10-09T08:29:34.250648+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:35.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:35 smithi016 bash[33851]: cluster 2023-10-09T08:29:34.250648+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:35.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:35 smithi016 bash[27454]: cluster 2023-10-09T08:29:34.250648+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:37.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:37 smithi094 bash[27803]: cluster 2023-10-09T08:29:36.251251+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:37.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:37 smithi016 bash[27454]: cluster 2023-10-09T08:29:36.251251+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:37.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:37 smithi016 bash[33851]: cluster 2023-10-09T08:29:36.251251+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:39.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:39 smithi094 bash[27803]: cluster 2023-10-09T08:29:38.251841+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:39.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:39 smithi016 bash[33851]: cluster 2023-10-09T08:29:38.251841+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:39 smithi016 bash[27454]: cluster 2023-10-09T08:29:38.251841+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:40.640 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:40 smithi094 bash[27803]: audit 2023-10-09T08:29:39.324103+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:29:40.641 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:40 smithi094 bash[27803]: audit 2023-10-09T08:29:39.325757+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:40.641 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:40 smithi094 bash[27803]: cephadm 2023-10-09T08:29:39.327058+0000 mgr.y (mgr.14146) 203 : cephadm [INF] Deploying daemon osd.5 on smithi094 2023-10-09T08:29:40.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:40 smithi016 bash[27454]: audit 2023-10-09T08:29:39.324103+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:29:40.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:40 smithi016 bash[27454]: audit 2023-10-09T08:29:39.325757+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:40.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:40 smithi016 bash[27454]: cephadm 2023-10-09T08:29:39.327058+0000 mgr.y (mgr.14146) 203 : cephadm [INF] Deploying daemon osd.5 on smithi094 2023-10-09T08:29:40.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:40 smithi016 bash[33851]: audit 2023-10-09T08:29:39.324103+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:29:40.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:40 smithi016 bash[33851]: audit 2023-10-09T08:29:39.325757+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:40.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:40 smithi016 bash[33851]: cephadm 2023-10-09T08:29:39.327058+0000 mgr.y (mgr.14146) 203 : cephadm [INF] Deploying daemon osd.5 on smithi094 2023-10-09T08:29:41.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:41 smithi094 bash[27803]: cluster 2023-10-09T08:29:40.252378+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:41.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:41 smithi016 bash[33851]: cluster 2023-10-09T08:29:40.252378+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:41.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:41 smithi016 bash[27454]: cluster 2023-10-09T08:29:40.252378+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:43 smithi094 bash[27803]: cluster 2023-10-09T08:29:42.252892+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:43.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:43 smithi016 bash[27454]: cluster 2023-10-09T08:29:42.252892+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:43.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:43 smithi016 bash[33851]: cluster 2023-10-09T08:29:42.252892+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:45 smithi094 bash[27803]: cluster 2023-10-09T08:29:44.253451+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:45 smithi094 bash[27803]: audit 2023-10-09T08:29:44.310647+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:45 smithi094 bash[27803]: audit 2023-10-09T08:29:44.313937+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:45.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:45 smithi016 bash[27454]: cluster 2023-10-09T08:29:44.253451+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:45.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:45 smithi016 bash[27454]: audit 2023-10-09T08:29:44.310647+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:45.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:45 smithi016 bash[27454]: audit 2023-10-09T08:29:44.313937+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:45.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:45 smithi016 bash[33851]: cluster 2023-10-09T08:29:44.253451+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:45.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:45 smithi016 bash[33851]: audit 2023-10-09T08:29:44.310647+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:45.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:45 smithi016 bash[33851]: audit 2023-10-09T08:29:44.313937+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:47 smithi094 bash[27803]: cluster 2023-10-09T08:29:46.253953+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:47 smithi016 bash[27454]: cluster 2023-10-09T08:29:46.253953+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:47.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:47 smithi016 bash[33851]: cluster 2023-10-09T08:29:46.253953+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:49 smithi094 bash[27803]: cluster 2023-10-09T08:29:48.254517+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:49 smithi016 bash[27454]: cluster 2023-10-09T08:29:48.254517+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:49 smithi016 bash[33851]: cluster 2023-10-09T08:29:48.254517+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:51.370 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:51 smithi094 bash[27803]: cluster 2023-10-09T08:29:50.255096+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:51.371 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:51 smithi094 bash[27803]: audit 2023-10-09T08:29:50.735741+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:51.371 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:51 smithi094 bash[27803]: audit 2023-10-09T08:29:50.737234+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:51.371 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:51 smithi094 bash[27803]: audit 2023-10-09T08:29:50.738934+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:51.371 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:51 smithi094 bash[27803]: audit 2023-10-09T08:29:50.746256+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[27454]: cluster 2023-10-09T08:29:50.255096+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[27454]: audit 2023-10-09T08:29:50.735741+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[27454]: audit 2023-10-09T08:29:50.737234+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[27454]: audit 2023-10-09T08:29:50.738934+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[27454]: audit 2023-10-09T08:29:50.746256+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[33851]: cluster 2023-10-09T08:29:50.255096+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[33851]: audit 2023-10-09T08:29:50.735741+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[33851]: audit 2023-10-09T08:29:50.737234+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:29:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[33851]: audit 2023-10-09T08:29:50.738934+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:29:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:51 smithi016 bash[33851]: audit 2023-10-09T08:29:50.746256+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:52.398 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:52 smithi094 bash[27803]: audit 2023-10-09T08:29:51.475441+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:29:52.398 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:52 smithi094 bash[27803]: audit 2023-10-09T08:29:51.476028+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:29:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:52 smithi016 bash[27454]: audit 2023-10-09T08:29:51.475441+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:29:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:52 smithi016 bash[27454]: audit 2023-10-09T08:29:51.476028+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:29:52.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:52 smithi016 bash[33851]: audit 2023-10-09T08:29:51.475441+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:29:52.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:52 smithi016 bash[33851]: audit 2023-10-09T08:29:51.476028+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:29:53.446 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:53 smithi094 bash[27803]: audit 2023-10-09T08:29:52.071306+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T08:29:53.446 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:53 smithi094 bash[27803]: cluster 2023-10-09T08:29:52.071392+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T08:29:53.447 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:53 smithi094 bash[27803]: audit 2023-10-09T08:29:52.071569+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:53.447 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:53 smithi094 bash[27803]: audit 2023-10-09T08:29:52.072480+0000 mon.b (mon.1) 12 : audit [INF] from='osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:53.447 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:53 smithi094 bash[27803]: audit 2023-10-09T08:29:52.072755+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:53.447 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:53 smithi094 bash[27803]: cluster 2023-10-09T08:29:52.255629+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:53.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[33851]: audit 2023-10-09T08:29:52.071306+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T08:29:53.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[33851]: cluster 2023-10-09T08:29:52.071392+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[33851]: audit 2023-10-09T08:29:52.071569+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[33851]: audit 2023-10-09T08:29:52.072480+0000 mon.b (mon.1) 12 : audit [INF] from='osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[33851]: audit 2023-10-09T08:29:52.072755+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[33851]: cluster 2023-10-09T08:29:52.255629+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[27454]: audit 2023-10-09T08:29:52.071306+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[27454]: cluster 2023-10-09T08:29:52.071392+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[27454]: audit 2023-10-09T08:29:52.071569+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[27454]: audit 2023-10-09T08:29:52.072480+0000 mon.b (mon.1) 12 : audit [INF] from='osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:53.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[27454]: audit 2023-10-09T08:29:52.072755+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:29:53.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:53 smithi016 bash[27454]: cluster 2023-10-09T08:29:52.255629+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:54.320 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:54 smithi094 bash[27803]: audit 2023-10-09T08:29:53.075238+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:29:54.321 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:54 smithi094 bash[27803]: cluster 2023-10-09T08:29:53.075415+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T08:29:54.321 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:54 smithi094 bash[27803]: audit 2023-10-09T08:29:53.076468+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:54.321 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:54 smithi094 bash[27803]: audit 2023-10-09T08:29:53.082461+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:54.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[27454]: audit 2023-10-09T08:29:53.075238+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:29:54.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[27454]: cluster 2023-10-09T08:29:53.075415+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T08:29:54.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[27454]: audit 2023-10-09T08:29:53.076468+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:54.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[27454]: audit 2023-10-09T08:29:53.082461+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:54.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[33851]: audit 2023-10-09T08:29:53.075238+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:29:54.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[33851]: cluster 2023-10-09T08:29:53.075415+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T08:29:54.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[33851]: audit 2023-10-09T08:29:53.076468+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:54.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:54 smithi016 bash[33851]: audit 2023-10-09T08:29:53.082461+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:54.907 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 5 on host 'smithi094' 2023-10-09T08:29:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: cluster 2023-10-09T08:29:52.489127+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:29:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: cluster 2023-10-09T08:29:52.489240+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:29:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: audit 2023-10-09T08:29:54.086683+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: cluster 2023-10-09T08:29:54.090871+0000 mon.a (mon.0) 442 : cluster [INF] osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646] boot 2023-10-09T08:29:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: cluster 2023-10-09T08:29:54.090939+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T08:29:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: audit 2023-10-09T08:29:54.091618+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: cluster 2023-10-09T08:29:54.256217+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:55.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: audit 2023-10-09T08:29:54.890063+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:55.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: audit 2023-10-09T08:29:54.927060+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:55.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:55 smithi094 bash[27803]: cluster 2023-10-09T08:29:55.058950+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T08:29:55.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: cluster 2023-10-09T08:29:52.489127+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:29:55.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: cluster 2023-10-09T08:29:52.489240+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:29:55.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: audit 2023-10-09T08:29:54.086683+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: cluster 2023-10-09T08:29:54.090871+0000 mon.a (mon.0) 442 : cluster [INF] osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646] boot 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: cluster 2023-10-09T08:29:54.090939+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: audit 2023-10-09T08:29:54.091618+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: cluster 2023-10-09T08:29:54.256217+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: audit 2023-10-09T08:29:54.890063+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: audit 2023-10-09T08:29:54.927060+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[33851]: cluster 2023-10-09T08:29:55.058950+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T08:29:55.522 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: cluster 2023-10-09T08:29:52.489127+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: cluster 2023-10-09T08:29:52.489240+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: audit 2023-10-09T08:29:54.086683+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: cluster 2023-10-09T08:29:54.090871+0000 mon.a (mon.0) 442 : cluster [INF] osd.5 [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646] boot 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: cluster 2023-10-09T08:29:54.090939+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: audit 2023-10-09T08:29:54.091618+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: cluster 2023-10-09T08:29:54.256217+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: audit 2023-10-09T08:29:54.890063+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: audit 2023-10-09T08:29:54.927060+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:29:55.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:55 smithi016 bash[27454]: cluster 2023-10-09T08:29:55.058950+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T08:29:55.719 DEBUG:teuthology.orchestra.run.smithi094:osd.5> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T08:29:55.721 INFO:tasks.cephadm:Deploying osd.6 on smithi094 with /dev/vg_nvme/lv_2... 2023-10-09T08:29:55.722 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T08:29:55.996 INFO:journalctl@ceph.osd.5.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:29:57.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:57 smithi094 bash[27803]: cluster 2023-10-09T08:29:56.058980+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T08:29:57.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:57 smithi094 bash[27803]: cluster 2023-10-09T08:29:56.256723+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:29:57.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:57 smithi016 bash[33851]: cluster 2023-10-09T08:29:56.058980+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T08:29:57.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:57 smithi016 bash[33851]: cluster 2023-10-09T08:29:56.256723+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:29:57.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:57 smithi016 bash[27454]: cluster 2023-10-09T08:29:56.058980+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T08:29:57.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:57 smithi016 bash[27454]: cluster 2023-10-09T08:29:56.256723+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:29:59.519 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T08:29:59.519 INFO:teuthology.orchestra.run.smithi094.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T08:29:59.519 INFO:teuthology.orchestra.run.smithi094.stderr: stderr: 10+0 records in 2023-10-09T08:29:59.519 INFO:teuthology.orchestra.run.smithi094.stderr:10+0 records out 2023-10-09T08:29:59.519 INFO:teuthology.orchestra.run.smithi094.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0150626 s, 696 MB/s 2023-10-09T08:29:59.519 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping successful for: 2023-10-09T08:29:59.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:29:59 smithi094 bash[27803]: cluster 2023-10-09T08:29:58.257267+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:29:59.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:29:59 smithi016 bash[33851]: cluster 2023-10-09T08:29:58.257267+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:29:59.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:29:59 smithi016 bash[27454]: cluster 2023-10-09T08:29:58.257267+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:00.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi094:vg_nvme/lv_2 2023-10-09T08:30:00.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:00 smithi094 bash[27803]: cluster 2023-10-09T08:30:00.000128+0000 mon.a (mon.0) 449 : cluster [INF] overall HEALTH_OK 2023-10-09T08:30:00.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:00 smithi016 bash[27454]: cluster 2023-10-09T08:30:00.000128+0000 mon.a (mon.0) 449 : cluster [INF] overall HEALTH_OK 2023-10-09T08:30:00.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:00 smithi016 bash[33851]: cluster 2023-10-09T08:30:00.000128+0000 mon.a (mon.0) 449 : cluster [INF] overall HEALTH_OK 2023-10-09T08:30:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:01 smithi094 bash[27803]: cluster 2023-10-09T08:30:00.257848+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:01.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:01 smithi016 bash[27454]: cluster 2023-10-09T08:30:00.257848+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:01.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:01 smithi016 bash[33851]: cluster 2023-10-09T08:30:00.257848+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:03.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:03 smithi094 bash[27803]: cluster 2023-10-09T08:30:02.258526+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:03 smithi016 bash[27454]: cluster 2023-10-09T08:30:02.258526+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:03.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:03 smithi016 bash[33851]: cluster 2023-10-09T08:30:02.258526+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:05.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:05 smithi094 bash[27803]: cluster 2023-10-09T08:30:04.259109+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:05.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:05 smithi016 bash[27454]: cluster 2023-10-09T08:30:04.259109+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:05.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:05 smithi016 bash[33851]: cluster 2023-10-09T08:30:04.259109+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:06 smithi094 bash[27803]: audit 2023-10-09T08:30:05.407554+0000 mgr.y (mgr.14146) 217 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:30:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:06 smithi094 bash[27803]: audit 2023-10-09T08:30:05.413599+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:30:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:06 smithi094 bash[27803]: audit 2023-10-09T08:30:05.419362+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:30:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:06 smithi094 bash[27803]: audit 2023-10-09T08:30:05.420922+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[27454]: audit 2023-10-09T08:30:05.407554+0000 mgr.y (mgr.14146) 217 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:30:06.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[27454]: audit 2023-10-09T08:30:05.413599+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:30:06.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[27454]: audit 2023-10-09T08:30:05.419362+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:30:06.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[27454]: audit 2023-10-09T08:30:05.420922+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:06.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[33851]: audit 2023-10-09T08:30:05.407554+0000 mgr.y (mgr.14146) 217 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:30:06.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[33851]: audit 2023-10-09T08:30:05.413599+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:30:06.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[33851]: audit 2023-10-09T08:30:05.419362+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:30:06.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:06 smithi016 bash[33851]: audit 2023-10-09T08:30:05.420922+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: cluster 2023-10-09T08:30:06.259678+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: audit 2023-10-09T08:30:06.897115+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: audit 2023-10-09T08:30:07.178622+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: audit 2023-10-09T08:30:07.180724+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: audit 2023-10-09T08:30:07.189770+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: audit 2023-10-09T08:30:07.191338+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:07.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: audit 2023-10-09T08:30:07.192536+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:07.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:07 smithi094 bash[27803]: audit 2023-10-09T08:30:07.199217+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: cluster 2023-10-09T08:30:06.259678+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: audit 2023-10-09T08:30:06.897115+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: audit 2023-10-09T08:30:07.178622+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: audit 2023-10-09T08:30:07.180724+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: audit 2023-10-09T08:30:07.189770+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: audit 2023-10-09T08:30:07.191338+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: audit 2023-10-09T08:30:07.192536+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[27454]: audit 2023-10-09T08:30:07.199217+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: cluster 2023-10-09T08:30:06.259678+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: audit 2023-10-09T08:30:06.897115+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: audit 2023-10-09T08:30:07.178622+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: audit 2023-10-09T08:30:07.180724+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: audit 2023-10-09T08:30:07.189770+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: audit 2023-10-09T08:30:07.191338+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: audit 2023-10-09T08:30:07.192536+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:07.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:07 smithi016 bash[33851]: audit 2023-10-09T08:30:07.199217+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:08.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:08 smithi094 bash[27803]: cephadm 2023-10-09T08:30:06.887860+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:30:08.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:08 smithi094 bash[27803]: cephadm 2023-10-09T08:30:07.182129+0000 mgr.y (mgr.14146) 220 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 8632M 2023-10-09T08:30:08.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:08 smithi016 bash[27454]: cephadm 2023-10-09T08:30:06.887860+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:30:08.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:08 smithi016 bash[27454]: cephadm 2023-10-09T08:30:07.182129+0000 mgr.y (mgr.14146) 220 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 8632M 2023-10-09T08:30:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:08 smithi016 bash[33851]: cephadm 2023-10-09T08:30:06.887860+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:30:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:08 smithi016 bash[33851]: cephadm 2023-10-09T08:30:07.182129+0000 mgr.y (mgr.14146) 220 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 8632M 2023-10-09T08:30:09.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:09 smithi094 bash[27803]: cluster 2023-10-09T08:30:08.260273+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:09 smithi016 bash[27454]: cluster 2023-10-09T08:30:08.260273+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:09 smithi016 bash[33851]: cluster 2023-10-09T08:30:08.260273+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:11.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:11 smithi094 bash[27803]: cluster 2023-10-09T08:30:10.260837+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:11.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:11 smithi016 bash[27454]: cluster 2023-10-09T08:30:10.260837+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:11.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:11 smithi016 bash[33851]: cluster 2023-10-09T08:30:10.260837+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:13.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:13 smithi094 bash[27803]: cluster 2023-10-09T08:30:12.261401+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:13 smithi016 bash[33851]: cluster 2023-10-09T08:30:12.261401+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:13 smithi016 bash[27454]: cluster 2023-10-09T08:30:12.261401+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:14 smithi094 bash[27803]: audit 2023-10-09T08:30:14.067957+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.94:0/497700503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]: dispatch 2023-10-09T08:30:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:14 smithi094 bash[27803]: audit 2023-10-09T08:30:14.068776+0000 mon.a (mon.0) 460 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]: dispatch 2023-10-09T08:30:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:14 smithi094 bash[27803]: audit 2023-10-09T08:30:14.078338+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]': finished 2023-10-09T08:30:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:14 smithi094 bash[27803]: cluster 2023-10-09T08:30:14.078474+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T08:30:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:14 smithi094 bash[27803]: audit 2023-10-09T08:30:14.078672+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:14.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[27454]: audit 2023-10-09T08:30:14.067957+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.94:0/497700503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]: dispatch 2023-10-09T08:30:14.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[27454]: audit 2023-10-09T08:30:14.068776+0000 mon.a (mon.0) 460 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]: dispatch 2023-10-09T08:30:14.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[27454]: audit 2023-10-09T08:30:14.078338+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]': finished 2023-10-09T08:30:14.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[27454]: cluster 2023-10-09T08:30:14.078474+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T08:30:14.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[27454]: audit 2023-10-09T08:30:14.078672+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:14.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[33851]: audit 2023-10-09T08:30:14.067957+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.94:0/497700503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]: dispatch 2023-10-09T08:30:14.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[33851]: audit 2023-10-09T08:30:14.068776+0000 mon.a (mon.0) 460 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]: dispatch 2023-10-09T08:30:14.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[33851]: audit 2023-10-09T08:30:14.078338+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "deb8f0b9-0b80-4dea-b180-353b84fb5daf"}]': finished 2023-10-09T08:30:14.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[33851]: cluster 2023-10-09T08:30:14.078474+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T08:30:14.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:14 smithi016 bash[33851]: audit 2023-10-09T08:30:14.078672+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:15 smithi094 bash[27803]: cluster 2023-10-09T08:30:14.261933+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:15 smithi094 bash[27803]: audit 2023-10-09T08:30:14.798127+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.94:0/4148458412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:30:15.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:15 smithi016 bash[27454]: cluster 2023-10-09T08:30:14.261933+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:15.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:15 smithi016 bash[27454]: audit 2023-10-09T08:30:14.798127+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.94:0/4148458412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:30:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:15 smithi016 bash[33851]: cluster 2023-10-09T08:30:14.261933+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:15.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:15 smithi016 bash[33851]: audit 2023-10-09T08:30:14.798127+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.94:0/4148458412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:30:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:17 smithi094 bash[27803]: cluster 2023-10-09T08:30:16.262461+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:17 smithi016 bash[27454]: cluster 2023-10-09T08:30:16.262461+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:17 smithi016 bash[33851]: cluster 2023-10-09T08:30:16.262461+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:19 smithi094 bash[27803]: cluster 2023-10-09T08:30:18.263028+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:19 smithi016 bash[27454]: cluster 2023-10-09T08:30:18.263028+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:19 smithi016 bash[33851]: cluster 2023-10-09T08:30:18.263028+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:21 smithi094 bash[27803]: cluster 2023-10-09T08:30:20.263594+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:21.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:21 smithi016 bash[27454]: cluster 2023-10-09T08:30:20.263594+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:21.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:21 smithi016 bash[33851]: cluster 2023-10-09T08:30:20.263594+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:23.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:23 smithi094 bash[27803]: cluster 2023-10-09T08:30:22.264186+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:23.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:23 smithi016 bash[27454]: cluster 2023-10-09T08:30:22.264186+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:23 smithi016 bash[33851]: cluster 2023-10-09T08:30:22.264186+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:25.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:25 smithi094 bash[27803]: cluster 2023-10-09T08:30:24.264707+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:25 smithi016 bash[27454]: cluster 2023-10-09T08:30:24.264707+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:25.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:25 smithi016 bash[33851]: cluster 2023-10-09T08:30:24.264707+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:27 smithi094 bash[27803]: cluster 2023-10-09T08:30:26.265238+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:27 smithi094 bash[27803]: audit 2023-10-09T08:30:27.169794+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:30:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:27 smithi094 bash[27803]: audit 2023-10-09T08:30:27.171640+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:27 smithi016 bash[27454]: cluster 2023-10-09T08:30:26.265238+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:27 smithi016 bash[27454]: audit 2023-10-09T08:30:27.169794+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:30:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:27 smithi016 bash[27454]: audit 2023-10-09T08:30:27.171640+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:27 smithi016 bash[33851]: cluster 2023-10-09T08:30:26.265238+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:27 smithi016 bash[33851]: audit 2023-10-09T08:30:27.169794+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:30:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:27 smithi016 bash[33851]: audit 2023-10-09T08:30:27.171640+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:28.700 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:28 smithi094 bash[27803]: cephadm 2023-10-09T08:30:27.173142+0000 mgr.y (mgr.14146) 231 : cephadm [INF] Deploying daemon osd.6 on smithi094 2023-10-09T08:30:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:28 smithi016 bash[27454]: cephadm 2023-10-09T08:30:27.173142+0000 mgr.y (mgr.14146) 231 : cephadm [INF] Deploying daemon osd.6 on smithi094 2023-10-09T08:30:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:28 smithi016 bash[33851]: cephadm 2023-10-09T08:30:27.173142+0000 mgr.y (mgr.14146) 231 : cephadm [INF] Deploying daemon osd.6 on smithi094 2023-10-09T08:30:29.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:29 smithi094 bash[27803]: cluster 2023-10-09T08:30:28.265896+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:29 smithi016 bash[27454]: cluster 2023-10-09T08:30:28.265896+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:29 smithi016 bash[33851]: cluster 2023-10-09T08:30:28.265896+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:31 smithi094 bash[27803]: cluster 2023-10-09T08:30:30.266493+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:31.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:31 smithi016 bash[27454]: cluster 2023-10-09T08:30:30.266493+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:31.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:31 smithi016 bash[33851]: cluster 2023-10-09T08:30:30.266493+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:33 smithi094 bash[27803]: audit 2023-10-09T08:30:32.089709+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:33 smithi094 bash[27803]: audit 2023-10-09T08:30:32.092039+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:30:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:33 smithi094 bash[27803]: cluster 2023-10-09T08:30:32.267050+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:33.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:33 smithi016 bash[27454]: audit 2023-10-09T08:30:32.089709+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:33.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:33 smithi016 bash[27454]: audit 2023-10-09T08:30:32.092039+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:30:33.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:33 smithi016 bash[27454]: cluster 2023-10-09T08:30:32.267050+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:33.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:33 smithi016 bash[33851]: audit 2023-10-09T08:30:32.089709+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:33.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:33 smithi016 bash[33851]: audit 2023-10-09T08:30:32.092039+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:30:33.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:33 smithi016 bash[33851]: cluster 2023-10-09T08:30:32.267050+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:35.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:35 smithi094 bash[27803]: cluster 2023-10-09T08:30:34.267703+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:35.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:35 smithi016 bash[33851]: cluster 2023-10-09T08:30:34.267703+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:35.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:35 smithi016 bash[27454]: cluster 2023-10-09T08:30:34.267703+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:37.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:37 smithi094 bash[27803]: cluster 2023-10-09T08:30:36.268418+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:37.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:37 smithi016 bash[27454]: cluster 2023-10-09T08:30:36.268418+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:37.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:37 smithi016 bash[33851]: cluster 2023-10-09T08:30:36.268418+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:39 smithi094 bash[27803]: cluster 2023-10-09T08:30:38.269122+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:39 smithi094 bash[27803]: audit 2023-10-09T08:30:38.556277+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:39 smithi094 bash[27803]: audit 2023-10-09T08:30:38.557622+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:39 smithi094 bash[27803]: audit 2023-10-09T08:30:38.559040+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:39.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:39 smithi094 bash[27803]: audit 2023-10-09T08:30:38.566243+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:39.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:39 smithi094 bash[27803]: audit 2023-10-09T08:30:39.242325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:30:39.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:39 smithi094 bash[27803]: audit 2023-10-09T08:30:39.242840+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[27454]: cluster 2023-10-09T08:30:38.269122+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[27454]: audit 2023-10-09T08:30:38.556277+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[27454]: audit 2023-10-09T08:30:38.557622+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[27454]: audit 2023-10-09T08:30:38.559040+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[27454]: audit 2023-10-09T08:30:38.566243+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[27454]: audit 2023-10-09T08:30:39.242325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[27454]: audit 2023-10-09T08:30:39.242840+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[33851]: cluster 2023-10-09T08:30:38.269122+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[33851]: audit 2023-10-09T08:30:38.556277+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[33851]: audit 2023-10-09T08:30:38.557622+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[33851]: audit 2023-10-09T08:30:38.559040+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[33851]: audit 2023-10-09T08:30:38.566243+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[33851]: audit 2023-10-09T08:30:39.242325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:30:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:39 smithi016 bash[33851]: audit 2023-10-09T08:30:39.242840+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:30:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:40 smithi094 bash[27803]: audit 2023-10-09T08:30:39.569732+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T08:30:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:40 smithi094 bash[27803]: cluster 2023-10-09T08:30:39.569883+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T08:30:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:40 smithi094 bash[27803]: audit 2023-10-09T08:30:39.570219+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:40 smithi094 bash[27803]: audit 2023-10-09T08:30:39.572104+0000 mon.b (mon.1) 16 : audit [INF] from='osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:30:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:40 smithi094 bash[27803]: audit 2023-10-09T08:30:39.572612+0000 mon.a (mon.0) 476 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:30:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:40 smithi094 bash[27803]: cluster 2023-10-09T08:30:40.269828+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:41.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[27454]: audit 2023-10-09T08:30:39.569732+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[27454]: cluster 2023-10-09T08:30:39.569883+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[27454]: audit 2023-10-09T08:30:39.570219+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[27454]: audit 2023-10-09T08:30:39.572104+0000 mon.b (mon.1) 16 : audit [INF] from='osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[27454]: audit 2023-10-09T08:30:39.572612+0000 mon.a (mon.0) 476 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[27454]: cluster 2023-10-09T08:30:40.269828+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[33851]: audit 2023-10-09T08:30:39.569732+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[33851]: cluster 2023-10-09T08:30:39.569883+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T08:30:41.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[33851]: audit 2023-10-09T08:30:39.570219+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:41.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[33851]: audit 2023-10-09T08:30:39.572104+0000 mon.b (mon.1) 16 : audit [INF] from='osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:30:41.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[33851]: audit 2023-10-09T08:30:39.572612+0000 mon.a (mon.0) 476 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:30:41.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:40 smithi016 bash[33851]: cluster 2023-10-09T08:30:40.269828+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:30:41.834 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:41 smithi094 bash[27803]: audit 2023-10-09T08:30:40.573963+0000 mon.a (mon.0) 477 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:30:41.835 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:41 smithi094 bash[27803]: cluster 2023-10-09T08:30:40.574228+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T08:30:41.835 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:41 smithi094 bash[27803]: audit 2023-10-09T08:30:40.575098+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:41.835 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:41 smithi094 bash[27803]: audit 2023-10-09T08:30:40.581748+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:42.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[33851]: audit 2023-10-09T08:30:40.573963+0000 mon.a (mon.0) 477 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:30:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[33851]: cluster 2023-10-09T08:30:40.574228+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T08:30:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[33851]: audit 2023-10-09T08:30:40.575098+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[33851]: audit 2023-10-09T08:30:40.581748+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:42.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[27454]: audit 2023-10-09T08:30:40.573963+0000 mon.a (mon.0) 477 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:30:42.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[27454]: cluster 2023-10-09T08:30:40.574228+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T08:30:42.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[27454]: audit 2023-10-09T08:30:40.575098+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:42.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:41 smithi016 bash[27454]: audit 2023-10-09T08:30:40.581748+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:42.655 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 6 on host 'smithi094' 2023-10-09T08:30:42.948 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:42 smithi094 bash[27803]: cluster 2023-10-09T08:30:40.257979+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:30:42.948 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:42 smithi094 bash[27803]: cluster 2023-10-09T08:30:40.258068+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:30:42.948 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:42 smithi094 bash[27803]: cluster 2023-10-09T08:30:41.578271+0000 mon.a (mon.0) 481 : cluster [INF] osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278] boot 2023-10-09T08:30:42.948 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:42 smithi094 bash[27803]: cluster 2023-10-09T08:30:41.578412+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T08:30:42.949 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:42 smithi094 bash[27803]: audit 2023-10-09T08:30:41.578723+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:42.949 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:42 smithi094 bash[27803]: cluster 2023-10-09T08:30:42.270410+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:42.949 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:42 smithi094 bash[27803]: cluster 2023-10-09T08:30:42.578810+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T08:30:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[33851]: cluster 2023-10-09T08:30:40.257979+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:30:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[33851]: cluster 2023-10-09T08:30:40.258068+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:30:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[33851]: cluster 2023-10-09T08:30:41.578271+0000 mon.a (mon.0) 481 : cluster [INF] osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278] boot 2023-10-09T08:30:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[33851]: cluster 2023-10-09T08:30:41.578412+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T08:30:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[33851]: audit 2023-10-09T08:30:41.578723+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[33851]: cluster 2023-10-09T08:30:42.270410+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[33851]: cluster 2023-10-09T08:30:42.578810+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T08:30:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[27454]: cluster 2023-10-09T08:30:40.257979+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:30:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[27454]: cluster 2023-10-09T08:30:40.258068+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:30:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[27454]: cluster 2023-10-09T08:30:41.578271+0000 mon.a (mon.0) 481 : cluster [INF] osd.6 [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278] boot 2023-10-09T08:30:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[27454]: cluster 2023-10-09T08:30:41.578412+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T08:30:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[27454]: audit 2023-10-09T08:30:41.578723+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:30:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[27454]: cluster 2023-10-09T08:30:42.270410+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:42 smithi016 bash[27454]: cluster 2023-10-09T08:30:42.578810+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T08:30:43.517 DEBUG:teuthology.orchestra.run.smithi094:osd.6> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T08:30:43.520 INFO:tasks.cephadm:Deploying osd.7 on smithi094 with /dev/vg_nvme/lv_1... 2023-10-09T08:30:43.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T08:30:43.666 INFO:journalctl@ceph.osd.6.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:30:43.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:43 smithi094 bash[27803]: audit 2023-10-09T08:30:42.652055+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:43.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:43 smithi094 bash[27803]: audit 2023-10-09T08:30:42.652663+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:30:43.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:43 smithi094 bash[27803]: cluster 2023-10-09T08:30:43.589627+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T08:30:44.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:43 smithi016 bash[27454]: audit 2023-10-09T08:30:42.652055+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:44.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:43 smithi016 bash[27454]: audit 2023-10-09T08:30:42.652663+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:30:44.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:43 smithi016 bash[27454]: cluster 2023-10-09T08:30:43.589627+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T08:30:44.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:43 smithi016 bash[33851]: audit 2023-10-09T08:30:42.652055+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:44.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:43 smithi016 bash[33851]: audit 2023-10-09T08:30:42.652663+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:30:44.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:43 smithi016 bash[33851]: cluster 2023-10-09T08:30:43.589627+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T08:30:44.921 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:44 smithi094 bash[27803]: cluster 2023-10-09T08:30:44.270987+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:45.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:44 smithi016 bash[33851]: cluster 2023-10-09T08:30:44.270987+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:45.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:44 smithi016 bash[27454]: cluster 2023-10-09T08:30:44.270987+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:47.210 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T08:30:47.210 INFO:teuthology.orchestra.run.smithi094.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T08:30:47.210 INFO:teuthology.orchestra.run.smithi094.stderr: stderr: 10+0 records in 2023-10-09T08:30:47.210 INFO:teuthology.orchestra.run.smithi094.stderr:10+0 records out 2023-10-09T08:30:47.210 INFO:teuthology.orchestra.run.smithi094.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135696 s, 773 MB/s 2023-10-09T08:30:47.210 INFO:teuthology.orchestra.run.smithi094.stderr:--> Zapping successful for: 2023-10-09T08:30:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:47 smithi094 bash[27803]: cluster 2023-10-09T08:30:46.271553+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:47 smithi016 bash[27454]: cluster 2023-10-09T08:30:46.271553+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:47.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:47 smithi016 bash[33851]: cluster 2023-10-09T08:30:46.271553+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:48.065 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi094:vg_nvme/lv_1 2023-10-09T08:30:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:49 smithi094 bash[27803]: cluster 2023-10-09T08:30:48.272208+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:49 smithi016 bash[27454]: cluster 2023-10-09T08:30:48.272208+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:49 smithi016 bash[33851]: cluster 2023-10-09T08:30:48.272208+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:51.392 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:51 smithi094 bash[27803]: cluster 2023-10-09T08:30:50.272728+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:51 smithi016 bash[27454]: cluster 2023-10-09T08:30:50.272728+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:51 smithi016 bash[33851]: cluster 2023-10-09T08:30:50.272728+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:53 smithi094 bash[27803]: cluster 2023-10-09T08:30:52.273279+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:53.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:53 smithi016 bash[33851]: cluster 2023-10-09T08:30:52.273279+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:53.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:53 smithi016 bash[27454]: cluster 2023-10-09T08:30:52.273279+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:54.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:54 smithi094 bash[27803]: audit 2023-10-09T08:30:53.409360+0000 mgr.y (mgr.14146) 245 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:30:54.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:54 smithi094 bash[27803]: audit 2023-10-09T08:30:53.416029+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:30:54.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:54 smithi094 bash[27803]: audit 2023-10-09T08:30:53.421606+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:30:54.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:54 smithi094 bash[27803]: audit 2023-10-09T08:30:53.423135+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:54.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[33851]: audit 2023-10-09T08:30:53.409360+0000 mgr.y (mgr.14146) 245 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:30:54.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[33851]: audit 2023-10-09T08:30:53.416029+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:30:54.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[33851]: audit 2023-10-09T08:30:53.421606+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:30:54.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[33851]: audit 2023-10-09T08:30:53.423135+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:54.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[27454]: audit 2023-10-09T08:30:53.409360+0000 mgr.y (mgr.14146) 245 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:30:54.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[27454]: audit 2023-10-09T08:30:53.416029+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:30:54.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[27454]: audit 2023-10-09T08:30:53.421606+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:30:54.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:54 smithi016 bash[27454]: audit 2023-10-09T08:30:53.423135+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: cluster 2023-10-09T08:30:54.273808+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.866285+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.868134+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.869570+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.871011+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.879600+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.881397+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.882786+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:55 smithi094 bash[27803]: audit 2023-10-09T08:30:54.890206+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: cluster 2023-10-09T08:30:54.273808+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.866285+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.868134+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.869570+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.871011+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.879600+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.881397+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.882786+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[27454]: audit 2023-10-09T08:30:54.890206+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: cluster 2023-10-09T08:30:54.273808+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.866285+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.868134+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.869570+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.871011+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.879600+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.881397+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.882786+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:30:55.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:55 smithi016 bash[33851]: audit 2023-10-09T08:30:54.890206+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:30:56.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:56 smithi094 bash[27803]: cephadm 2023-10-09T08:30:54.856670+0000 mgr.y (mgr.14146) 247 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:30:56.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:56 smithi094 bash[27803]: cephadm 2023-10-09T08:30:54.872149+0000 mgr.y (mgr.14146) 248 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 5755M 2023-10-09T08:30:56.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:56 smithi016 bash[27454]: cephadm 2023-10-09T08:30:54.856670+0000 mgr.y (mgr.14146) 247 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:30:56.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:56 smithi016 bash[27454]: cephadm 2023-10-09T08:30:54.872149+0000 mgr.y (mgr.14146) 248 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 5755M 2023-10-09T08:30:56.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:56 smithi016 bash[33851]: cephadm 2023-10-09T08:30:54.856670+0000 mgr.y (mgr.14146) 247 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:30:56.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:56 smithi016 bash[33851]: cephadm 2023-10-09T08:30:54.872149+0000 mgr.y (mgr.14146) 248 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 5755M 2023-10-09T08:30:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:57 smithi094 bash[27803]: cluster 2023-10-09T08:30:56.274401+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:57 smithi016 bash[27454]: cluster 2023-10-09T08:30:56.274401+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:57.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:57 smithi016 bash[33851]: cluster 2023-10-09T08:30:56.274401+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:59.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:30:59 smithi094 bash[27803]: cluster 2023-10-09T08:30:58.275064+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:59.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:30:59 smithi016 bash[27454]: cluster 2023-10-09T08:30:58.275064+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:30:59.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:30:59 smithi016 bash[33851]: cluster 2023-10-09T08:30:58.275064+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:01.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:01 smithi094 bash[27803]: cluster 2023-10-09T08:31:00.275766+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:01.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:01 smithi016 bash[27454]: cluster 2023-10-09T08:31:00.275766+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:01.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:01 smithi016 bash[33851]: cluster 2023-10-09T08:31:00.275766+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:02.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:02 smithi094 bash[27803]: audit 2023-10-09T08:31:01.962622+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.94:0/1557058524' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]: dispatch 2023-10-09T08:31:02.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:02 smithi094 bash[27803]: audit 2023-10-09T08:31:01.963134+0000 mon.a (mon.0) 499 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]: dispatch 2023-10-09T08:31:02.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:02 smithi094 bash[27803]: audit 2023-10-09T08:31:01.971644+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]': finished 2023-10-09T08:31:02.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:02 smithi094 bash[27803]: cluster 2023-10-09T08:31:01.971790+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T08:31:02.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:02 smithi094 bash[27803]: audit 2023-10-09T08:31:01.972109+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:02.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[33851]: audit 2023-10-09T08:31:01.962622+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.94:0/1557058524' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]: dispatch 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[33851]: audit 2023-10-09T08:31:01.963134+0000 mon.a (mon.0) 499 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]: dispatch 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[33851]: audit 2023-10-09T08:31:01.971644+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]': finished 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[33851]: cluster 2023-10-09T08:31:01.971790+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[33851]: audit 2023-10-09T08:31:01.972109+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[27454]: audit 2023-10-09T08:31:01.962622+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.94:0/1557058524' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]: dispatch 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[27454]: audit 2023-10-09T08:31:01.963134+0000 mon.a (mon.0) 499 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]: dispatch 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[27454]: audit 2023-10-09T08:31:01.971644+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84cd66c8-c28e-4d58-ac4d-3836315c04f2"}]': finished 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[27454]: cluster 2023-10-09T08:31:01.971790+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T08:31:02.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:02 smithi016 bash[27454]: audit 2023-10-09T08:31:01.972109+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:03.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:03 smithi094 bash[27803]: cluster 2023-10-09T08:31:02.276372+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:03.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:03 smithi094 bash[27803]: audit 2023-10-09T08:31:02.684147+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.94:0/4259665402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:31:03.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:03 smithi016 bash[33851]: cluster 2023-10-09T08:31:02.276372+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:03.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:03 smithi016 bash[33851]: audit 2023-10-09T08:31:02.684147+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.94:0/4259665402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:31:03.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:03 smithi016 bash[27454]: cluster 2023-10-09T08:31:02.276372+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:03.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:03 smithi016 bash[27454]: audit 2023-10-09T08:31:02.684147+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.94:0/4259665402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:31:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:05 smithi094 bash[27803]: cluster 2023-10-09T08:31:04.277004+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:05.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:05 smithi016 bash[33851]: cluster 2023-10-09T08:31:04.277004+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:05.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:05 smithi016 bash[27454]: cluster 2023-10-09T08:31:04.277004+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:07 smithi094 bash[27803]: cluster 2023-10-09T08:31:06.277687+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:07 smithi016 bash[33851]: cluster 2023-10-09T08:31:06.277687+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:07 smithi016 bash[27454]: cluster 2023-10-09T08:31:06.277687+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:09.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:09 smithi094 bash[27803]: cluster 2023-10-09T08:31:08.278344+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:09 smithi016 bash[27454]: cluster 2023-10-09T08:31:08.278344+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:09 smithi016 bash[33851]: cluster 2023-10-09T08:31:08.278344+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:11.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:11 smithi094 bash[27803]: cluster 2023-10-09T08:31:10.278941+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:11.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:11 smithi016 bash[27454]: cluster 2023-10-09T08:31:10.278941+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:11.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:11 smithi016 bash[33851]: cluster 2023-10-09T08:31:10.278941+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:13.713 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:13 smithi094 bash[27803]: cluster 2023-10-09T08:31:12.279498+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:13.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:13 smithi016 bash[27454]: cluster 2023-10-09T08:31:12.279498+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:13.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:13 smithi016 bash[33851]: cluster 2023-10-09T08:31:12.279498+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:15 smithi094 bash[27803]: cluster 2023-10-09T08:31:14.280080+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:15 smithi094 bash[27803]: audit 2023-10-09T08:31:15.179865+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T08:31:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:15 smithi094 bash[27803]: audit 2023-10-09T08:31:15.181540+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:15.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:15 smithi016 bash[27454]: cluster 2023-10-09T08:31:14.280080+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:15.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:15 smithi016 bash[27454]: audit 2023-10-09T08:31:15.179865+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T08:31:15.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:15 smithi016 bash[27454]: audit 2023-10-09T08:31:15.181540+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:15.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:15 smithi016 bash[33851]: cluster 2023-10-09T08:31:14.280080+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:15.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:15 smithi016 bash[33851]: audit 2023-10-09T08:31:15.179865+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T08:31:15.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:15 smithi016 bash[33851]: audit 2023-10-09T08:31:15.181540+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:16.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:16 smithi094 bash[27803]: cephadm 2023-10-09T08:31:15.182954+0000 mgr.y (mgr.14146) 259 : cephadm [INF] Deploying daemon osd.7 on smithi094 2023-10-09T08:31:16.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:16 smithi016 bash[27454]: cephadm 2023-10-09T08:31:15.182954+0000 mgr.y (mgr.14146) 259 : cephadm [INF] Deploying daemon osd.7 on smithi094 2023-10-09T08:31:16.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:16 smithi016 bash[33851]: cephadm 2023-10-09T08:31:15.182954+0000 mgr.y (mgr.14146) 259 : cephadm [INF] Deploying daemon osd.7 on smithi094 2023-10-09T08:31:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:17 smithi094 bash[27803]: cluster 2023-10-09T08:31:16.280597+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:17 smithi016 bash[27454]: cluster 2023-10-09T08:31:16.280597+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:17 smithi016 bash[33851]: cluster 2023-10-09T08:31:16.280597+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:19 smithi094 bash[27803]: cluster 2023-10-09T08:31:18.281158+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:19 smithi016 bash[27454]: cluster 2023-10-09T08:31:18.281158+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:19 smithi016 bash[33851]: cluster 2023-10-09T08:31:18.281158+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:21 smithi094 bash[27803]: audit 2023-10-09T08:31:20.078025+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:21 smithi094 bash[27803]: audit 2023-10-09T08:31:20.083327+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:21 smithi094 bash[27803]: cluster 2023-10-09T08:31:20.281669+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:21.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:21 smithi016 bash[27454]: audit 2023-10-09T08:31:20.078025+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:21.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:21 smithi016 bash[27454]: audit 2023-10-09T08:31:20.083327+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:21.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:21 smithi016 bash[27454]: cluster 2023-10-09T08:31:20.281669+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:21.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:21 smithi016 bash[33851]: audit 2023-10-09T08:31:20.078025+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:21.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:21 smithi016 bash[33851]: audit 2023-10-09T08:31:20.083327+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:21.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:21 smithi016 bash[33851]: cluster 2023-10-09T08:31:20.281669+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:23.596 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:23 smithi094 bash[27803]: cluster 2023-10-09T08:31:22.282220+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:23.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:23 smithi016 bash[27454]: cluster 2023-10-09T08:31:22.282220+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:23 smithi016 bash[33851]: cluster 2023-10-09T08:31:22.282220+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:25.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:25 smithi094 bash[27803]: cluster 2023-10-09T08:31:24.282774+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:25.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:25 smithi016 bash[27454]: cluster 2023-10-09T08:31:24.282774+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:25 smithi016 bash[33851]: cluster 2023-10-09T08:31:24.282774+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:27.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:27 smithi094 bash[27803]: cluster 2023-10-09T08:31:26.283338+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:27.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:27 smithi094 bash[27803]: audit 2023-10-09T08:31:26.571527+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:27.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:27 smithi094 bash[27803]: audit 2023-10-09T08:31:26.862299+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:27.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:27 smithi094 bash[27803]: audit 2023-10-09T08:31:26.864370+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:27.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:27 smithi094 bash[27803]: audit 2023-10-09T08:31:26.873628+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:27.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:27 smithi094 bash[27803]: audit 2023-10-09T08:31:27.219976+0000 mon.b (mon.1) 19 : audit [INF] from='osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T08:31:27.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:27 smithi094 bash[27803]: audit 2023-10-09T08:31:27.220489+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T08:31:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[33851]: cluster 2023-10-09T08:31:26.283338+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[33851]: audit 2023-10-09T08:31:26.571527+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[33851]: audit 2023-10-09T08:31:26.862299+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[33851]: audit 2023-10-09T08:31:26.864370+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[33851]: audit 2023-10-09T08:31:26.873628+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:27.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[33851]: audit 2023-10-09T08:31:27.219976+0000 mon.b (mon.1) 19 : audit [INF] from='osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[33851]: audit 2023-10-09T08:31:27.220489+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[27454]: cluster 2023-10-09T08:31:26.283338+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[27454]: audit 2023-10-09T08:31:26.571527+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[27454]: audit 2023-10-09T08:31:26.862299+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[27454]: audit 2023-10-09T08:31:26.864370+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[27454]: audit 2023-10-09T08:31:26.873628+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[27454]: audit 2023-10-09T08:31:27.219976+0000 mon.b (mon.1) 19 : audit [INF] from='osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T08:31:27.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:27 smithi016 bash[27454]: audit 2023-10-09T08:31:27.220489+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T08:31:29.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:28 smithi094 bash[27803]: audit 2023-10-09T08:31:27.876757+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T08:31:29.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:28 smithi094 bash[27803]: cluster 2023-10-09T08:31:27.876868+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T08:31:29.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:28 smithi094 bash[27803]: audit 2023-10-09T08:31:27.877114+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:29.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:28 smithi094 bash[27803]: audit 2023-10-09T08:31:27.878370+0000 mon.b (mon.1) 20 : audit [INF] from='osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:31:29.164 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:28 smithi094 bash[27803]: audit 2023-10-09T08:31:27.878834+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:31:29.164 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:28 smithi094 bash[27803]: cluster 2023-10-09T08:31:28.283888+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:29.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[33851]: audit 2023-10-09T08:31:27.876757+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T08:31:29.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[33851]: cluster 2023-10-09T08:31:27.876868+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T08:31:29.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[33851]: audit 2023-10-09T08:31:27.877114+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[33851]: audit 2023-10-09T08:31:27.878370+0000 mon.b (mon.1) 20 : audit [INF] from='osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[33851]: audit 2023-10-09T08:31:27.878834+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[33851]: cluster 2023-10-09T08:31:28.283888+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[27454]: audit 2023-10-09T08:31:27.876757+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[27454]: cluster 2023-10-09T08:31:27.876868+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[27454]: audit 2023-10-09T08:31:27.877114+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[27454]: audit 2023-10-09T08:31:27.878370+0000 mon.b (mon.1) 20 : audit [INF] from='osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[27454]: audit 2023-10-09T08:31:27.878834+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2023-10-09T08:31:29.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:28 smithi016 bash[27454]: cluster 2023-10-09T08:31:28.283888+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:29.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:29 smithi094 bash[27803]: audit 2023-10-09T08:31:28.879814+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:31:29.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:29 smithi094 bash[27803]: cluster 2023-10-09T08:31:28.880046+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T08:31:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:29 smithi094 bash[27803]: audit 2023-10-09T08:31:28.880658+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:29 smithi094 bash[27803]: audit 2023-10-09T08:31:28.887029+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:30.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[27454]: audit 2023-10-09T08:31:28.879814+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:31:30.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[27454]: cluster 2023-10-09T08:31:28.880046+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T08:31:30.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[27454]: audit 2023-10-09T08:31:28.880658+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:30.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[27454]: audit 2023-10-09T08:31:28.887029+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[33851]: audit 2023-10-09T08:31:28.879814+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2023-10-09T08:31:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[33851]: cluster 2023-10-09T08:31:28.880046+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T08:31:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[33851]: audit 2023-10-09T08:31:28.880658+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:29 smithi016 bash[33851]: audit 2023-10-09T08:31:28.887029+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:30.563 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 7 on host 'smithi094' 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: cluster 2023-10-09T08:31:28.263689+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: cluster 2023-10-09T08:31:28.263776+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: audit 2023-10-09T08:31:29.890697+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: cluster 2023-10-09T08:31:29.893938+0000 mon.a (mon.0) 521 : cluster [INF] osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234] boot 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: cluster 2023-10-09T08:31:29.894003+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: audit 2023-10-09T08:31:29.894165+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: cluster 2023-10-09T08:31:30.092553+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: cluster 2023-10-09T08:31:30.284457+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: audit 2023-10-09T08:31:30.552794+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:30 smithi094 bash[27803]: audit 2023-10-09T08:31:30.558764+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: cluster 2023-10-09T08:31:28.263689+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:31:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: cluster 2023-10-09T08:31:28.263776+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:31:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: audit 2023-10-09T08:31:29.890697+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: cluster 2023-10-09T08:31:29.893938+0000 mon.a (mon.0) 521 : cluster [INF] osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234] boot 2023-10-09T08:31:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: cluster 2023-10-09T08:31:29.894003+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: audit 2023-10-09T08:31:29.894165+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: cluster 2023-10-09T08:31:30.092553+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: cluster 2023-10-09T08:31:30.284457+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: audit 2023-10-09T08:31:30.552794+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[33851]: audit 2023-10-09T08:31:30.558764+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: cluster 2023-10-09T08:31:28.263689+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: cluster 2023-10-09T08:31:28.263776+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: audit 2023-10-09T08:31:29.890697+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:31.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: cluster 2023-10-09T08:31:29.893938+0000 mon.a (mon.0) 521 : cluster [INF] osd.7 [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234] boot 2023-10-09T08:31:31.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: cluster 2023-10-09T08:31:29.894003+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T08:31:31.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: audit 2023-10-09T08:31:29.894165+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:31:31.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: cluster 2023-10-09T08:31:30.092553+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T08:31:31.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: cluster 2023-10-09T08:31:30.284457+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:31:31.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: audit 2023-10-09T08:31:30.552794+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:31.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:30 smithi016 bash[27454]: audit 2023-10-09T08:31:30.558764+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:31.335 DEBUG:teuthology.orchestra.run.smithi094:osd.7> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T08:31:31.338 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T08:31:31.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T08:31:31.528 INFO:journalctl@ceph.osd.7.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:31:32.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:32 smithi094 bash[27803]: cluster 2023-10-09T08:31:31.105272+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T08:31:32.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:32 smithi016 bash[33851]: cluster 2023-10-09T08:31:31.105272+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T08:31:32.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:32 smithi016 bash[27454]: cluster 2023-10-09T08:31:31.105272+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T08:31:33.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:33 smithi016 bash[27454]: cluster 2023-10-09T08:31:32.285064+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v258: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:33 smithi094 bash[27803]: cluster 2023-10-09T08:31:32.285064+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v258: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:33.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:33 smithi016 bash[33851]: cluster 2023-10-09T08:31:32.285064+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v258: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:35.088 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:31:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:35 smithi016 bash[27454]: cluster 2023-10-09T08:31:34.285657+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v259: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:35 smithi016 bash[27454]: audit 2023-10-09T08:31:35.085424+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.16:0/1995744753' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T08:31:35.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:35 smithi016 bash[33851]: cluster 2023-10-09T08:31:34.285657+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v259: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:35.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:35 smithi016 bash[33851]: audit 2023-10-09T08:31:35.085424+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.16:0/1995744753' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T08:31:35.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:35 smithi094 bash[27803]: cluster 2023-10-09T08:31:34.285657+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v259: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:35.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:35 smithi094 bash[27803]: audit 2023-10-09T08:31:35.085424+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.16:0/1995744753' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T08:31:35.772 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1696840289,"num_in_osds":8,"osd_in_since":1696840261,"num_remapped_pgs":0} 2023-10-09T08:31:35.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T08:31:37.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:37 smithi094 bash[27803]: cluster 2023-10-09T08:31:36.286261+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:37.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:37 smithi016 bash[27454]: cluster 2023-10-09T08:31:36.286261+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:37.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:37 smithi016 bash[33851]: cluster 2023-10-09T08:31:36.286261+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:39.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:39 smithi094 bash[27803]: cluster 2023-10-09T08:31:38.286828+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:39 smithi016 bash[27454]: cluster 2023-10-09T08:31:38.286828+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:39 smithi016 bash[33851]: cluster 2023-10-09T08:31:38.286828+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:40.535 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:31:41.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: cephadm 2023-10-09T08:31:40.027091+0000 mgr.y (mgr.14146) 272 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:31:41.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.034368+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.036097+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.037316+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.038549+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.039756+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: cephadm 2023-10-09T08:31:40.040780+0000 mgr.y (mgr.14146) 273 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 4316M 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.047246+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.048981+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.050319+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: audit 2023-10-09T08:31:40.056315+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[27454]: cluster 2023-10-09T08:31:40.287416+0000 mgr.y (mgr.14146) 274 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:41.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: cephadm 2023-10-09T08:31:40.027091+0000 mgr.y (mgr.14146) 272 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.034368+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.036097+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.037316+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.038549+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.039756+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: cephadm 2023-10-09T08:31:40.040780+0000 mgr.y (mgr.14146) 273 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 4316M 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.047246+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.048981+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.050319+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: audit 2023-10-09T08:31:40.056315+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:41 smithi016 bash[33851]: cluster 2023-10-09T08:31:40.287416+0000 mgr.y (mgr.14146) 274 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:41.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: cephadm 2023-10-09T08:31:40.027091+0000 mgr.y (mgr.14146) 272 : cephadm [INF] Detected new or changed devices on smithi094 2023-10-09T08:31:41.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.034368+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.036097+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.037316+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.038549+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.039756+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: cephadm 2023-10-09T08:31:40.040780+0000 mgr.y (mgr.14146) 273 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 4316M 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.047246+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.048981+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.050319+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: audit 2023-10-09T08:31:40.056315+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:41 smithi094 bash[27803]: cluster 2023-10-09T08:31:40.287416+0000 mgr.y (mgr.14146) 274 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:43.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:43 smithi016 bash[27454]: cluster 2023-10-09T08:31:42.288078+0000 mgr.y (mgr.14146) 275 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:43.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:43 smithi016 bash[33851]: cluster 2023-10-09T08:31:42.288078+0000 mgr.y (mgr.14146) 275 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:43.633 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:31:43.633 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":50,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:31:31.095252+0000","last_up_change":"2023-10-09T08:31:29.882559+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6803","nonce":818359725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6805","nonce":818359725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6809","nonce":818359725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6807","nonce":818359725}]},"public_addr":"172.21.15.16:6803/818359725","cluster_addr":"172.21.15.16:6805/818359725","heartbeat_back_addr":"172.21.15.16:6809/818359725","heartbeat_front_addr":"172.21.15.16:6807/818359725","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6811","nonce":664142092}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6813","nonce":664142092}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6817","nonce":664142092}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6815","nonce":664142092}]},"public_addr":"172.21.15.16:6811/664142092","cluster_addr":"172.21.15.16:6813/664142092","heartbeat_back_addr":"172.21.15.16:6817/664142092","heartbeat_front_addr":"172.21.15.16:6815/664142092","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6819","nonce":3541024001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6821","nonce":3541024001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6825","nonce":3541024001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6823","nonce":3541024001}]},"public_addr":"172.21.15.16:6819/3541024001","cluster_addr":"172.21.15.16:6821/3541024001","heartbeat_back_addr":"172.21.15.16:6825/3541024001","heartbeat_front_addr":"172.21.15.16:6823/3541024001","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6827","nonce":3124750952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6829","nonce":3124750952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6833","nonce":3124750952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6831","nonce":3124750952}]},"public_addr":"172.21.15.16:6827/3124750952","cluster_addr":"172.21.15.16:6829/3124750952","heartbeat_back_addr":"172.21.15.16:6833/3124750952","heartbeat_front_addr":"172.21.15.16:6831/3124750952","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:31:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:43 smithi094 bash[27803]: cluster 2023-10-09T08:31:42.288078+0000 mgr.y (mgr.14146) 275 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:44.425 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-09T08:27:38.062560+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-09T08:31:44.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-09T08:31:44.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:44 smithi094 bash[27803]: audit 2023-10-09T08:31:43.628913+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.16:0/525095818' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:31:44.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:44 smithi016 bash[27454]: audit 2023-10-09T08:31:43.628913+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.16:0/525095818' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:31:44.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:44 smithi016 bash[33851]: audit 2023-10-09T08:31:43.628913+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.16:0/525095818' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:31:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:45 smithi094 bash[27803]: cluster 2023-10-09T08:31:44.288595+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:45.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:45 smithi016 bash[33851]: cluster 2023-10-09T08:31:44.288595+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:45.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:45 smithi016 bash[27454]: cluster 2023-10-09T08:31:44.288595+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:47 smithi094 bash[27803]: cluster 2023-10-09T08:31:46.289221+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:47 smithi016 bash[27454]: cluster 2023-10-09T08:31:46.289221+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:47.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:47 smithi016 bash[33851]: cluster 2023-10-09T08:31:46.289221+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:49.195 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:31:49.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:49 smithi016 bash[27454]: cluster 2023-10-09T08:31:48.289794+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:49.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:49 smithi016 bash[33851]: cluster 2023-10-09T08:31:48.289794+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:49 smithi094 bash[27803]: cluster 2023-10-09T08:31:48.289794+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:51 smithi094 bash[27803]: cluster 2023-10-09T08:31:50.290377+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:51 smithi016 bash[27454]: cluster 2023-10-09T08:31:50.290377+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:51 smithi016 bash[33851]: cluster 2023-10-09T08:31:50.290377+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:52.351 INFO:teuthology.orchestra.run.smithi016.stdout:pg_num: 1 2023-10-09T08:31:53.050 INFO:tasks.cephadm:Adding prometheus.a on smithi094 2023-10-09T08:31:53.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi094=a' 2023-10-09T08:31:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:53 smithi094 bash[27803]: cluster 2023-10-09T08:31:52.290889+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:53 smithi094 bash[27803]: audit 2023-10-09T08:31:52.349153+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.16:0/2519926450' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T08:31:53.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:53 smithi016 bash[27454]: cluster 2023-10-09T08:31:52.290889+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:53.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:53 smithi016 bash[27454]: audit 2023-10-09T08:31:52.349153+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.16:0/2519926450' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T08:31:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:53 smithi016 bash[33851]: cluster 2023-10-09T08:31:52.290889+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:53 smithi016 bash[33851]: audit 2023-10-09T08:31:52.349153+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.16:0/2519926450' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T08:31:55.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:55 smithi094 bash[27803]: cluster 2023-10-09T08:31:54.291443+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:55.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:55 smithi016 bash[27454]: cluster 2023-10-09T08:31:54.291443+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:55.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:55 smithi016 bash[33851]: cluster 2023-10-09T08:31:54.291443+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:56.706 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled prometheus update... 2023-10-09T08:31:57.575 DEBUG:teuthology.orchestra.run.smithi094:prometheus.a> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T08:31:57.584 INFO:tasks.cephadm:Adding node-exporter.a on smithi016 2023-10-09T08:31:57.584 INFO:tasks.cephadm:Adding node-exporter.b on smithi094 2023-10-09T08:31:57.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi016=a;smithi094=b' 2023-10-09T08:31:57.642 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[27803]: cluster 2023-10-09T08:31:56.292041+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:57.642 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[27803]: audit 2023-10-09T08:31:56.702070+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:57.643 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[27803]: audit 2023-10-09T08:31:56.716902+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:57.643 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[27803]: audit 2023-10-09T08:31:56.720256+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:57.643 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[27803]: audit 2023-10-09T08:31:56.722054+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:57.643 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[27803]: audit 2023-10-09T08:31:56.729160+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:57.643 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[27803]: audit 2023-10-09T08:31:56.740860+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T08:31:57.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27454]: cluster 2023-10-09T08:31:56.292041+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:57.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27454]: audit 2023-10-09T08:31:56.702070+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27454]: audit 2023-10-09T08:31:56.716902+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27454]: audit 2023-10-09T08:31:56.720256+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27454]: audit 2023-10-09T08:31:56.722054+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27454]: audit 2023-10-09T08:31:56.729160+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27454]: audit 2023-10-09T08:31:56.740860+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[33851]: cluster 2023-10-09T08:31:56.292041+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[33851]: audit 2023-10-09T08:31:56.702070+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[33851]: audit 2023-10-09T08:31:56.716902+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:31:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[33851]: audit 2023-10-09T08:31:56.720256+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:31:57.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[33851]: audit 2023-10-09T08:31:56.722054+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:31:57.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[33851]: audit 2023-10-09T08:31:56.729160+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' 2023-10-09T08:31:57.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[33851]: audit 2023-10-09T08:31:56.740860+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T08:31:57.834 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:31:57.938 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[28656]: ignoring --setuser ceph since I am not root 2023-10-09T08:31:57.938 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[28656]: ignoring --setgroup ceph since I am not root 2023-10-09T08:31:57.938 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[28656]: debug 2023-10-09T08:31:57.773+0000 7f638a038700 1 -- 172.21.15.94:0/2034874282 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d7976684e0 con 0x55d79775cc00 2023-10-09T08:31:58.065 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27723]: ignoring --setuser ceph since I am not root 2023-10-09T08:31:58.065 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27723]: ignoring --setgroup ceph since I am not root 2023-10-09T08:31:58.065 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27723]: debug 2023-10-09T08:31:57.785+0000 7f4b4f3f7700 1 -- 172.21.15.16:0/2308000718 <== mon.0 v2:172.21.15.16:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558b1d03a4e0 con 0x558b1d12e800 2023-10-09T08:31:58.066 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:57 smithi016 bash[27723]: debug 2023-10-09T08:31:57.945+0000 7f4b5f5bf680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:31:58.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:57 smithi094 bash[28656]: debug 2023-10-09T08:31:57.929+0000 7f639a200680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:31:58.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:58 smithi094 bash[28656]: debug 2023-10-09T08:31:58.037+0000 7f639a200680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:31:58.361 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[27723]: debug 2023-10-09T08:31:58.057+0000 7f4b5f5bf680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:31:58.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:58 smithi094 bash[28656]: debug 2023-10-09T08:31:58.517+0000 7f639a200680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:31:58.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:58 smithi094 bash[27803]: audit 2023-10-09T08:31:56.691430+0000 mgr.y (mgr.14146) 283 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:31:58.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:58 smithi094 bash[27803]: cephadm 2023-10-09T08:31:56.694930+0000 mgr.y (mgr.14146) 284 : cephadm [INF] Saving service prometheus spec with placement smithi094=a;count:1 2023-10-09T08:31:58.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:58 smithi094 bash[27803]: audit 2023-10-09T08:31:57.739993+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T08:31:58.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:31:58 smithi094 bash[27803]: cluster 2023-10-09T08:31:57.740109+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T08:31:58.751 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[27723]: debug 2023-10-09T08:31:58.544+0000 7f4b5f5bf680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:31:58.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[27454]: audit 2023-10-09T08:31:56.691430+0000 mgr.y (mgr.14146) 283 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:31:58.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[27454]: cephadm 2023-10-09T08:31:56.694930+0000 mgr.y (mgr.14146) 284 : cephadm [INF] Saving service prometheus spec with placement smithi094=a;count:1 2023-10-09T08:31:58.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[27454]: audit 2023-10-09T08:31:57.739993+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T08:31:58.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[27454]: cluster 2023-10-09T08:31:57.740109+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T08:31:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[33851]: audit 2023-10-09T08:31:56.691430+0000 mgr.y (mgr.14146) 283 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:31:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[33851]: cephadm 2023-10-09T08:31:56.694930+0000 mgr.y (mgr.14146) 284 : cephadm [INF] Saving service prometheus spec with placement smithi094=a;count:1 2023-10-09T08:31:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[33851]: audit 2023-10-09T08:31:57.739993+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.16:0/3448871627' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T08:31:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:31:58 smithi016 bash[33851]: cluster 2023-10-09T08:31:57.740109+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T08:31:59.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:59 smithi094 bash[28656]: debug 2023-10-09T08:31:59.473+0000 7f639a200680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:31:59.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:59 smithi094 bash[28656]: debug 2023-10-09T08:31:59.569+0000 7f639a200680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:31:59.871 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:59 smithi016 bash[27723]: debug 2023-10-09T08:31:59.496+0000 7f4b5f5bf680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:31:59.871 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:59 smithi016 bash[27723]: debug 2023-10-09T08:31:59.600+0000 7f4b5f5bf680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:32:00.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:31:59 smithi094 bash[28656]: debug 2023-10-09T08:31:59.817+0000 7f639a200680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:32:00.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:31:59 smithi016 bash[27723]: debug 2023-10-09T08:31:59.864+0000 7f4b5f5bf680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:32:00.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:00 smithi094 bash[28656]: debug 2023-10-09T08:32:00.957+0000 7f639a200680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:32:01.331 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:01 smithi016 bash[27723]: debug 2023-10-09T08:32:01.048+0000 7f4b5f5bf680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:32:01.331 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:01 smithi016 bash[27723]: debug 2023-10-09T08:32:01.216+0000 7f4b5f5bf680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:32:01.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:01 smithi094 bash[28656]: debug 2023-10-09T08:32:01.121+0000 7f639a200680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:32:01.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:01 smithi094 bash[28656]: debug 2023-10-09T08:32:01.217+0000 7f639a200680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:32:01.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:01 smithi094 bash[28656]: debug 2023-10-09T08:32:01.333+0000 7f639a200680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:32:01.678 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:01 smithi016 bash[27723]: debug 2023-10-09T08:32:01.324+0000 7f4b5f5bf680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:32:01.678 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:01 smithi016 bash[27723]: debug 2023-10-09T08:32:01.444+0000 7f4b5f5bf680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:32:01.931 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:01 smithi094 bash[28656]: debug 2023-10-09T08:32:01.553+0000 7f639a200680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:32:02.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:01 smithi016 bash[27723]: debug 2023-10-09T08:32:01.668+0000 7f4b5f5bf680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:32:02.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:01 smithi094 bash[28656]: debug 2023-10-09T08:32:01.925+0000 7f639a200680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:32:02.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:02 smithi094 bash[28656]: debug 2023-10-09T08:32:02.045+0000 7f639a200680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:32:02.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:02 smithi016 bash[27723]: debug 2023-10-09T08:32:02.056+0000 7f4b5f5bf680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:32:02.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:02 smithi016 bash[27723]: debug 2023-10-09T08:32:02.184+0000 7f4b5f5bf680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:32:03.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:02 smithi094 bash[28656]: debug 2023-10-09T08:32:02.909+0000 7f639a200680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:32:03.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:03 smithi094 bash[28656]: debug 2023-10-09T08:32:03.013+0000 7f639a200680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:32:03.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:03 smithi094 bash[28656]: debug 2023-10-09T08:32:03.125+0000 7f639a200680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:32:03.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:03 smithi016 bash[27723]: debug 2023-10-09T08:32:03.088+0000 7f4b5f5bf680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:32:03.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:03 smithi016 bash[27723]: debug 2023-10-09T08:32:03.200+0000 7f4b5f5bf680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:32:03.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:03 smithi016 bash[27723]: debug 2023-10-09T08:32:03.320+0000 7f4b5f5bf680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:32:03.643 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:03 smithi094 bash[28656]: debug 2023-10-09T08:32:03.349+0000 7f639a200680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:32:03.855 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:03 smithi016 bash[27723]: debug 2023-10-09T08:32:03.552+0000 7f4b5f5bf680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:32:03.981 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:03 smithi094 bash[28656]: debug 2023-10-09T08:32:03.637+0000 7f639a200680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:32:03.981 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:03 smithi094 bash[28656]: debug 2023-10-09T08:32:03.801+0000 7f639a200680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:32:04.212 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:03 smithi016 bash[27723]: debug 2023-10-09T08:32:03.848+0000 7f4b5f5bf680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:32:04.212 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: debug 2023-10-09T08:32:04.024+0000 7f4b5f5bf680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:32:04.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:03 smithi094 bash[28656]: debug 2023-10-09T08:32:03.973+0000 7f639a200680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:32:04.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[28656]: debug 2023-10-09T08:32:04.189+0000 7f639a200680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:32:04.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: debug 2023-10-09T08:32:04.204+0000 7f4b5f5bf680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:32:04.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: debug 2023-10-09T08:32:04.440+0000 7f4b5f5bf680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:32:04.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.323602+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:32:04.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.323742+0000 mon.a (mon.0) 547 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:32:04.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27454]: audit 2023-10-09T08:32:04.327818+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27454]: audit 2023-10-09T08:32:04.329499+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27454]: audit 2023-10-09T08:32:04.333860+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27454]: audit 2023-10-09T08:32:04.334381+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.323602+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.323742+0000 mon.a (mon.0) 547 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[33851]: audit 2023-10-09T08:32:04.327818+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[33851]: audit 2023-10-09T08:32:04.329499+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[33851]: audit 2023-10-09T08:32:04.333860+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:32:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[33851]: audit 2023-10-09T08:32:04.334381+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.323602+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.323742+0000 mon.a (mon.0) 547 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[27803]: audit 2023-10-09T08:32:04.327818+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[27803]: audit 2023-10-09T08:32:04.329499+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[27803]: audit 2023-10-09T08:32:04.333860+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[27803]: audit 2023-10-09T08:32:04.334381+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.94:0/2454395719' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[28656]: debug 2023-10-09T08:32:04.297+0000 7f639a200680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:32:04.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[28656]: [09/Oct/2023:08:32:04] ENGINE Bus STARTING 2023-10-09T08:32:04.744 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[28656]: CherryPy Checker: 2023-10-09T08:32:04.744 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[28656]: The Application mounted at '' has an empty config. 2023-10-09T08:32:04.744 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[28656]: [09/Oct/2023:08:32:04] ENGINE Serving on http://:::9283 2023-10-09T08:32:04.744 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:04 smithi094 bash[28656]: [09/Oct/2023:08:32:04] ENGINE Bus STARTED 2023-10-09T08:32:04.818 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: debug 2023-10-09T08:32:04.552+0000 7f4b5f5bf680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:32:05.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: [09/Oct/2023:08:32:04] ENGINE Bus STARTING 2023-10-09T08:32:05.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: Warning: Permanently added '172.21.15.94' (ECDSA) to the list of known hosts. 2023-10-09T08:32:05.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: Warning: Permanently added '172.21.15.16' (ECDSA) to the list of known hosts. 2023-10-09T08:32:05.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: CherryPy Checker: 2023-10-09T08:32:05.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: The Application mounted at '' has an empty config. 2023-10-09T08:32:05.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: [09/Oct/2023:08:32:04] ENGINE Serving on http://:::9283 2023-10-09T08:32:05.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:04 smithi016 bash[27723]: [09/Oct/2023:08:32:04] ENGINE Bus STARTED 2023-10-09T08:32:05.615 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled node-exporter update... 2023-10-09T08:32:05.666 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.384660+0000 mon.a (mon.0) 548 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-10-09T08:32:05.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.579422+0000 mon.a (mon.0) 549 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:32:05.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.579926+0000 mon.a (mon.0) 550 : cluster [INF] Activating manager daemon y 2023-10-09T08:32:05.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.590229+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T08:32:05.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.590640+0000 mon.a (mon.0) 552 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0108866s), standbys: x 2023-10-09T08:32:05.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.594428+0000 mon.c (mon.2) 12 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:32:05.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.594752+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:32:05.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.595066+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:32:05.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.596366+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:32:05.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.596624+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:32:05.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.596994+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:32:05.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.597347+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:32:05.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.597640+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:32:05.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.598026+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.598374+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.598672+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.599006+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.599280+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.599547+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.599792+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.600445+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: cluster 2023-10-09T08:32:04.606030+0000 mon.a (mon.0) 553 : cluster [INF] Manager daemon y is now available 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.748081+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.754595+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:05.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.789004+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:05.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.824650+0000 mon.c (mon.2) 30 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:32:05.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.825181+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:32:05.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.833422+0000 mon.c (mon.2) 31 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:32:05.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:05 smithi094 bash[27803]: audit 2023-10-09T08:32:04.834096+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:32:05.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.384660+0000 mon.a (mon.0) 548 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-10-09T08:32:05.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.579422+0000 mon.a (mon.0) 549 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:32:05.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.579926+0000 mon.a (mon.0) 550 : cluster [INF] Activating manager daemon y 2023-10-09T08:32:05.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.590229+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T08:32:05.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.590640+0000 mon.a (mon.0) 552 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0108866s), standbys: x 2023-10-09T08:32:05.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.594428+0000 mon.c (mon.2) 12 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.594752+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.595066+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.596366+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.596624+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.596994+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.597347+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.597640+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.598026+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.598374+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.598672+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.599006+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:32:05.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.599280+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:32:05.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.599547+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:32:05.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.599792+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:32:05.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.384660+0000 mon.a (mon.0) 548 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-10-09T08:32:05.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.579422+0000 mon.a (mon.0) 549 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:32:05.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.579926+0000 mon.a (mon.0) 550 : cluster [INF] Activating manager daemon y 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.590229+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.590640+0000 mon.a (mon.0) 552 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0108866s), standbys: x 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.594428+0000 mon.c (mon.2) 12 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.594752+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.595066+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.596366+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.596624+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.596994+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:32:05.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.597347+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.597640+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.598026+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.598374+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.598672+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.599006+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.599280+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.599547+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:32:05.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.599792+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.600445+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: cluster 2023-10-09T08:32:04.606030+0000 mon.a (mon.0) 553 : cluster [INF] Manager daemon y is now available 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.748081+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.754595+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.789004+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.824650+0000 mon.c (mon.2) 30 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.825181+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.833422+0000 mon.c (mon.2) 31 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:32:05.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[33851]: audit 2023-10-09T08:32:04.834096+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:32:05.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.600445+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:32:05.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: cluster 2023-10-09T08:32:04.606030+0000 mon.a (mon.0) 553 : cluster [INF] Manager daemon y is now available 2023-10-09T08:32:05.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.748081+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:05.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.754595+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:05.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.789004+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:05.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.824650+0000 mon.c (mon.2) 30 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:32:05.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.825181+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:32:05.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.833422+0000 mon.c (mon.2) 31 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:32:05.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:05 smithi016 bash[27454]: audit 2023-10-09T08:32:04.834096+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:32:06.591 DEBUG:teuthology.orchestra.run.smithi016:node-exporter.a> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T08:32:06.594 DEBUG:teuthology.orchestra.run.smithi094:node-exporter.b> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T08:32:06.636 INFO:tasks.cephadm:Adding alertmanager.a on smithi016 2023-10-09T08:32:06.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi016=a' 2023-10-09T08:32:06.713 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:32:06.787 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:32:06.868 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:06 smithi094 bash[27803]: cluster 2023-10-09T08:32:05.601002+0000 mon.a (mon.0) 557 : cluster [DBG] mgrmap e18: y(active, since 1.02124s), standbys: x 2023-10-09T08:32:06.869 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:06 smithi094 bash[27803]: cephadm 2023-10-09T08:32:05.607187+0000 mgr.y (mgr.24355) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi016=a;smithi094=b;count:2 2023-10-09T08:32:06.869 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:06 smithi094 bash[27803]: audit 2023-10-09T08:32:05.612307+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:07.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:06 smithi016 bash[27454]: cluster 2023-10-09T08:32:05.601002+0000 mon.a (mon.0) 557 : cluster [DBG] mgrmap e18: y(active, since 1.02124s), standbys: x 2023-10-09T08:32:07.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:06 smithi016 bash[27454]: cephadm 2023-10-09T08:32:05.607187+0000 mgr.y (mgr.24355) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi016=a;smithi094=b;count:2 2023-10-09T08:32:07.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:06 smithi016 bash[27454]: audit 2023-10-09T08:32:05.612307+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:07.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:06 smithi016 bash[33851]: cluster 2023-10-09T08:32:05.601002+0000 mon.a (mon.0) 557 : cluster [DBG] mgrmap e18: y(active, since 1.02124s), standbys: x 2023-10-09T08:32:07.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:06 smithi016 bash[33851]: cephadm 2023-10-09T08:32:05.607187+0000 mgr.y (mgr.24355) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi016=a;smithi094=b;count:2 2023-10-09T08:32:07.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:06 smithi016 bash[33851]: audit 2023-10-09T08:32:05.612307+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:07.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:07 smithi094 bash[27803]: cluster 2023-10-09T08:32:06.598414+0000 mgr.y (mgr.24355) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:08.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:07 smithi016 bash[27454]: cluster 2023-10-09T08:32:06.598414+0000 mgr.y (mgr.24355) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:08.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:07 smithi016 bash[33851]: cluster 2023-10-09T08:32:06.598414+0000 mgr.y (mgr.24355) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:08 smithi094 bash[27803]: cluster 2023-10-09T08:32:07.614968+0000 mon.a (mon.0) 559 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T08:32:09.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:08 smithi016 bash[27454]: cluster 2023-10-09T08:32:07.614968+0000 mon.a (mon.0) 559 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T08:32:09.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:08 smithi016 bash[33851]: cluster 2023-10-09T08:32:07.614968+0000 mon.a (mon.0) 559 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T08:32:09.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:09 smithi094 bash[27803]: cluster 2023-10-09T08:32:08.599092+0000 mgr.y (mgr.24355) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:10.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:09 smithi016 bash[27454]: cluster 2023-10-09T08:32:08.599092+0000 mgr.y (mgr.24355) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:10.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:09 smithi016 bash[33851]: cluster 2023-10-09T08:32:08.599092+0000 mgr.y (mgr.24355) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:10.167 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled alertmanager update... 2023-10-09T08:32:11.124 DEBUG:teuthology.orchestra.run.smithi016:alertmanager.a> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T08:32:11.127 INFO:tasks.cephadm:Adding grafana.a on smithi094 2023-10-09T08:32:11.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi094=a' 2023-10-09T08:32:11.315 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:32:11.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:10.154077+0000 mgr.y (mgr.24355) 6 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi016=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:32:11.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: cephadm 2023-10-09T08:32:10.156828+0000 mgr.y (mgr.24355) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi016=a;count:1 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:10.163561+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:10.447169+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:10.565883+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: cluster 2023-10-09T08:32:10.599710+0000 mgr.y (mgr.24355) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:10.822390+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:10.824625+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:10.886870+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:11.050316+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:11 smithi094 bash[27803]: audit 2023-10-09T08:32:11.058814+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:10.154077+0000 mgr.y (mgr.24355) 6 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi016=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:32:11.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: cephadm 2023-10-09T08:32:10.156828+0000 mgr.y (mgr.24355) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi016=a;count:1 2023-10-09T08:32:11.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:10.163561+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:10.447169+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:10.565883+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: cluster 2023-10-09T08:32:10.599710+0000 mgr.y (mgr.24355) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:10.822390+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:10.824625+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:10.886870+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:11.050316+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[27454]: audit 2023-10-09T08:32:11.058814+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:10.154077+0000 mgr.y (mgr.24355) 6 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi016=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:32:11.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: cephadm 2023-10-09T08:32:10.156828+0000 mgr.y (mgr.24355) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi016=a;count:1 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:10.163561+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:10.447169+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:10.565883+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: cluster 2023-10-09T08:32:10.599710+0000 mgr.y (mgr.24355) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:10.822390+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:10.824625+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:10.886870+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:11.050316+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:11.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:11 smithi016 bash[33851]: audit 2023-10-09T08:32:11.058814+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:12.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:12 smithi094 bash[27803]: cephadm 2023-10-09T08:32:10.827276+0000 mgr.y (mgr.24355) 9 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:32:12.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:12 smithi094 bash[27803]: cephadm 2023-10-09T08:32:10.827894+0000 mgr.y (mgr.24355) 10 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:32:12.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:12 smithi094 bash[27803]: cephadm 2023-10-09T08:32:10.850923+0000 mgr.y (mgr.24355) 11 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:32:12.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:12 smithi094 bash[27803]: cephadm 2023-10-09T08:32:10.989541+0000 mgr.y (mgr.24355) 12 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:32:12.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:12 smithi094 bash[27803]: cephadm 2023-10-09T08:32:11.082025+0000 mgr.y (mgr.24355) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi094 2023-10-09T08:32:13.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[27454]: cephadm 2023-10-09T08:32:10.827276+0000 mgr.y (mgr.24355) 9 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:32:13.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[27454]: cephadm 2023-10-09T08:32:10.827894+0000 mgr.y (mgr.24355) 10 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:32:13.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[27454]: cephadm 2023-10-09T08:32:10.850923+0000 mgr.y (mgr.24355) 11 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:32:13.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[27454]: cephadm 2023-10-09T08:32:10.989541+0000 mgr.y (mgr.24355) 12 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:32:13.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[27454]: cephadm 2023-10-09T08:32:11.082025+0000 mgr.y (mgr.24355) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi094 2023-10-09T08:32:13.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[33851]: cephadm 2023-10-09T08:32:10.827276+0000 mgr.y (mgr.24355) 9 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:32:13.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[33851]: cephadm 2023-10-09T08:32:10.827894+0000 mgr.y (mgr.24355) 10 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:32:13.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[33851]: cephadm 2023-10-09T08:32:10.850923+0000 mgr.y (mgr.24355) 11 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:32:13.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[33851]: cephadm 2023-10-09T08:32:10.989541+0000 mgr.y (mgr.24355) 12 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:32:13.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:12 smithi016 bash[33851]: cephadm 2023-10-09T08:32:11.082025+0000 mgr.y (mgr.24355) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi094 2023-10-09T08:32:13.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:13 smithi094 bash[27803]: cluster 2023-10-09T08:32:12.600319+0000 mgr.y (mgr.24355) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:14.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:13 smithi016 bash[27454]: cluster 2023-10-09T08:32:12.600319+0000 mgr.y (mgr.24355) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:13 smithi016 bash[33851]: cluster 2023-10-09T08:32:12.600319+0000 mgr.y (mgr.24355) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:15.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:15 smithi094 bash[27803]: cluster 2023-10-09T08:32:14.600891+0000 mgr.y (mgr.24355) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:16.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:15 smithi016 bash[27454]: cluster 2023-10-09T08:32:14.600891+0000 mgr.y (mgr.24355) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:16.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:15 smithi016 bash[33851]: cluster 2023-10-09T08:32:14.600891+0000 mgr.y (mgr.24355) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:17.089 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled grafana update... 2023-10-09T08:32:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:17 smithi094 bash[27803]: cluster 2023-10-09T08:32:16.601563+0000 mgr.y (mgr.24355) 16 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:17 smithi094 bash[27803]: audit 2023-10-09T08:32:17.085763+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:18.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:17 smithi016 bash[27454]: cluster 2023-10-09T08:32:16.601563+0000 mgr.y (mgr.24355) 16 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:18.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:17 smithi016 bash[27454]: audit 2023-10-09T08:32:17.085763+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:18.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:17 smithi016 bash[33851]: cluster 2023-10-09T08:32:16.601563+0000 mgr.y (mgr.24355) 16 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:18.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:17 smithi016 bash[33851]: audit 2023-10-09T08:32:17.085763+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:18.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:18 smithi094 bash[27803]: audit 2023-10-09T08:32:17.074702+0000 mgr.y (mgr.24355) 17 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:32:18.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:18 smithi094 bash[27803]: cephadm 2023-10-09T08:32:17.078185+0000 mgr.y (mgr.24355) 18 : cephadm [INF] Saving service grafana spec with placement smithi094=a;count:1 2023-10-09T08:32:19.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:18 smithi016 bash[27454]: audit 2023-10-09T08:32:17.074702+0000 mgr.y (mgr.24355) 17 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:32:19.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:18 smithi016 bash[27454]: cephadm 2023-10-09T08:32:17.078185+0000 mgr.y (mgr.24355) 18 : cephadm [INF] Saving service grafana spec with placement smithi094=a;count:1 2023-10-09T08:32:19.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:18 smithi016 bash[33851]: audit 2023-10-09T08:32:17.074702+0000 mgr.y (mgr.24355) 17 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:32:19.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:18 smithi016 bash[33851]: cephadm 2023-10-09T08:32:17.078185+0000 mgr.y (mgr.24355) 18 : cephadm [INF] Saving service grafana spec with placement smithi094=a;count:1 2023-10-09T08:32:19.257 DEBUG:teuthology.orchestra.run.smithi094:grafana.a> sudo journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T08:32:19.262 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T08:32:19.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T08:32:19.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:32:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:19 smithi016 bash[33851]: cluster 2023-10-09T08:32:18.602221+0000 mgr.y (mgr.24355) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:19 smithi016 bash[27454]: cluster 2023-10-09T08:32:18.602221+0000 mgr.y (mgr.24355) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:19.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:19 smithi094 bash[27803]: cluster 2023-10-09T08:32:18.602221+0000 mgr.y (mgr.24355) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:21 smithi094 bash[27803]: cluster 2023-10-09T08:32:20.602738+0000 mgr.y (mgr.24355) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:21.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:21 smithi016 bash[27454]: cluster 2023-10-09T08:32:20.602738+0000 mgr.y (mgr.24355) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:21.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:21 smithi016 bash[33851]: cluster 2023-10-09T08:32:20.602738+0000 mgr.y (mgr.24355) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:22.751 INFO:teuthology.orchestra.run.smithi016.stdout:[client.0] 2023-10-09T08:32:22.752 INFO:teuthology.orchestra.run.smithi016.stdout: key = AQCWuiNlOpxDLBAAMyI67wNp9b+SmtAFn1jeOQ== 2023-10-09T08:32:23.494 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:32:23.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T08:32:23.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T08:32:23.569 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T08:32:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[33851]: cluster 2023-10-09T08:32:22.603308+0000 mgr.y (mgr.24355) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[33851]: audit 2023-10-09T08:32:22.741911+0000 mon.c (mon.2) 34 : audit [INF] from='client.? 172.21.15.16:0/3945186234' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[33851]: audit 2023-10-09T08:32:22.742476+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[33851]: audit 2023-10-09T08:32:22.748963+0000 mon.a (mon.0) 568 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T08:32:23.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[27454]: cluster 2023-10-09T08:32:22.603308+0000 mgr.y (mgr.24355) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:23.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[27454]: audit 2023-10-09T08:32:22.741911+0000 mon.c (mon.2) 34 : audit [INF] from='client.? 172.21.15.16:0/3945186234' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:23.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[27454]: audit 2023-10-09T08:32:22.742476+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:23.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:23 smithi016 bash[27454]: audit 2023-10-09T08:32:22.748963+0000 mon.a (mon.0) 568 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T08:32:23.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:23 smithi094 bash[27803]: cluster 2023-10-09T08:32:22.603308+0000 mgr.y (mgr.24355) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:23.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:23 smithi094 bash[27803]: audit 2023-10-09T08:32:22.741911+0000 mon.c (mon.2) 34 : audit [INF] from='client.? 172.21.15.16:0/3945186234' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:23.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:23 smithi094 bash[27803]: audit 2023-10-09T08:32:22.742476+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:23.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:23 smithi094 bash[27803]: audit 2023-10-09T08:32:22.748963+0000 mon.a (mon.0) 568 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T08:32:25.842 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:25 smithi094 bash[27803]: cluster 2023-10-09T08:32:24.603896+0000 mgr.y (mgr.24355) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:25 smithi016 bash[27454]: cluster 2023-10-09T08:32:24.603896+0000 mgr.y (mgr.24355) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:26.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:25 smithi016 bash[33851]: cluster 2023-10-09T08:32:24.603896+0000 mgr.y (mgr.24355) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:27.325 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:27 smithi016 systemd[1]: Started Ceph node-exporter.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:32:27.325 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[52639]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T08:32:27.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[33851]: audit 2023-10-09T08:32:26.317428+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:27.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[33851]: cephadm 2023-10-09T08:32:26.322017+0000 mgr.y (mgr.24355) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi016 2023-10-09T08:32:27.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[33851]: cluster 2023-10-09T08:32:26.604582+0000 mgr.y (mgr.24355) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:27.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[33851]: audit 2023-10-09T08:32:27.083344+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:27.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[27454]: audit 2023-10-09T08:32:26.317428+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:27.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[27454]: cephadm 2023-10-09T08:32:26.322017+0000 mgr.y (mgr.24355) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi016 2023-10-09T08:32:27.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[27454]: cluster 2023-10-09T08:32:26.604582+0000 mgr.y (mgr.24355) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:27.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[27454]: audit 2023-10-09T08:32:27.083344+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:27 smithi094 bash[27803]: audit 2023-10-09T08:32:26.317428+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:27 smithi094 bash[27803]: cephadm 2023-10-09T08:32:26.322017+0000 mgr.y (mgr.24355) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi016 2023-10-09T08:32:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:27 smithi094 bash[27803]: cluster 2023-10-09T08:32:26.604582+0000 mgr.y (mgr.24355) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:27 smithi094 bash[27803]: audit 2023-10-09T08:32:27.083344+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:27.925 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[52639]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T08:32:27.925 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[52639]: aa2a8d90b84c: Pulling fs layer 2023-10-09T08:32:27.925 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[52639]: b45d31ee2d7f: Pulling fs layer 2023-10-09T08:32:27.925 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:27 smithi016 bash[52639]: b5db1e299295: Pulling fs layer 2023-10-09T08:32:28.099 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:27 smithi094 systemd[1]: Started Ceph node-exporter.b for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:32:28.100 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:27 smithi094 bash[47070]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T08:32:28.189 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[52639]: b45d31ee2d7f: Verifying Checksum 2023-10-09T08:32:28.189 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[52639]: b45d31ee2d7f: Download complete 2023-10-09T08:32:28.189 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[52639]: aa2a8d90b84c: Download complete 2023-10-09T08:32:28.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:28 smithi094 bash[27803]: cephadm 2023-10-09T08:32:27.085715+0000 mgr.y (mgr.24355) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi094 2023-10-09T08:32:28.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:28 smithi094 bash[27803]: audit 2023-10-09T08:32:27.817763+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:28.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:32:28 smithi094 bash[46955]: ts=2023-10-09T08:32:28.297Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T08:32:28.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[27454]: cephadm 2023-10-09T08:32:27.085715+0000 mgr.y (mgr.24355) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi094 2023-10-09T08:32:28.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[27454]: audit 2023-10-09T08:32:27.817763+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:28.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[33851]: cephadm 2023-10-09T08:32:27.085715+0000 mgr.y (mgr.24355) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi094 2023-10-09T08:32:28.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[33851]: audit 2023-10-09T08:32:27.817763+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:28.501 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[52639]: b5db1e299295: Verifying Checksum 2023-10-09T08:32:28.502 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[52639]: b5db1e299295: Download complete 2023-10-09T08:32:28.502 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[52639]: aa2a8d90b84c: Pull complete 2023-10-09T08:32:28.680 INFO:teuthology.orchestra.run.smithi094.stdout:[client.1] 2023-10-09T08:32:28.680 INFO:teuthology.orchestra.run.smithi094.stdout: key = AQCcuiNlUQoDKBAAfv1/XIW9IGhEnJGv7bO7eA== 2023-10-09T08:32:28.953 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:28 smithi094 bash[47070]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T08:32:29.238 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:28 smithi016 bash[52639]: b45d31ee2d7f: Pull complete 2023-10-09T08:32:29.242 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:28 smithi094 bash[47070]: aa2a8d90b84c: Pulling fs layer 2023-10-09T08:32:29.242 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:28 smithi094 bash[47070]: b45d31ee2d7f: Pulling fs layer 2023-10-09T08:32:29.242 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:28 smithi094 bash[47070]: b5db1e299295: Pulling fs layer 2023-10-09T08:32:29.242 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[47070]: b45d31ee2d7f: Download complete 2023-10-09T08:32:29.242 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[47070]: aa2a8d90b84c: Download complete 2023-10-09T08:32:29.335 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[47070]: b5db1e299295: Verifying Checksum 2023-10-09T08:32:29.335 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[47070]: b5db1e299295: Download complete 2023-10-09T08:32:29.498 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[52639]: b5db1e299295: Pull complete 2023-10-09T08:32:29.498 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[52639]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T08:32:29.498 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[52639]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T08:32:29.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[27454]: cephadm 2023-10-09T08:32:27.831776+0000 mgr.y (mgr.24355) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi016 2023-10-09T08:32:29.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[27454]: cluster 2023-10-09T08:32:28.605214+0000 mgr.y (mgr.24355) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:29.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[27454]: audit 2023-10-09T08:32:28.670895+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.94:0/1307971748' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[27454]: audit 2023-10-09T08:32:28.671101+0000 mon.a (mon.0) 572 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[27454]: audit 2023-10-09T08:32:28.677042+0000 mon.a (mon.0) 573 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T08:32:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[33851]: cephadm 2023-10-09T08:32:27.831776+0000 mgr.y (mgr.24355) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi016 2023-10-09T08:32:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[33851]: cluster 2023-10-09T08:32:28.605214+0000 mgr.y (mgr.24355) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[33851]: audit 2023-10-09T08:32:28.670895+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.94:0/1307971748' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[33851]: audit 2023-10-09T08:32:28.671101+0000 mon.a (mon.0) 572 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:29 smithi016 bash[33851]: audit 2023-10-09T08:32:28.677042+0000 mon.a (mon.0) 573 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T08:32:29.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[27803]: cephadm 2023-10-09T08:32:27.831776+0000 mgr.y (mgr.24355) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi016 2023-10-09T08:32:29.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[27803]: cluster 2023-10-09T08:32:28.605214+0000 mgr.y (mgr.24355) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:29.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[27803]: audit 2023-10-09T08:32:28.670895+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.94:0/1307971748' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:29.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[27803]: audit 2023-10-09T08:32:28.671101+0000 mon.a (mon.0) 572 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T08:32:29.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[27803]: audit 2023-10-09T08:32:28.677042+0000 mon.a (mon.0) 573 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T08:32:29.837 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2023-10-09T08:32:29.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T08:32:29.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T08:32:29.855 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T08:32:29.856 DEBUG:teuthology.orchestra.run.smithi016:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T08:32:29.992 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:29 smithi094 bash[47070]: aa2a8d90b84c: Pull complete 2023-10-09T08:32:30.561 INFO:teuthology.misc.health.smithi016.stdout: 2023-10-09T08:32:30.562 INFO:teuthology.misc.health.smithi016.stdout:{"epoch":51,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:32:04.579518+0000","last_up_change":"2023-10-09T08:31:29.882559+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6803","nonce":818359725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6805","nonce":818359725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6809","nonce":818359725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6807","nonce":818359725}]},"public_addr":"172.21.15.16:6803/818359725","cluster_addr":"172.21.15.16:6805/818359725","heartbeat_back_addr":"172.21.15.16:6809/818359725","heartbeat_front_addr":"172.21.15.16:6807/818359725","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6811","nonce":664142092}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6813","nonce":664142092}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6817","nonce":664142092}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6815","nonce":664142092}]},"public_addr":"172.21.15.16:6811/664142092","cluster_addr":"172.21.15.16:6813/664142092","heartbeat_back_addr":"172.21.15.16:6817/664142092","heartbeat_front_addr":"172.21.15.16:6815/664142092","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6819","nonce":3541024001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6821","nonce":3541024001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6825","nonce":3541024001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6823","nonce":3541024001}]},"public_addr":"172.21.15.16:6819/3541024001","cluster_addr":"172.21.15.16:6821/3541024001","heartbeat_back_addr":"172.21.15.16:6825/3541024001","heartbeat_front_addr":"172.21.15.16:6823/3541024001","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6827","nonce":3124750952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6829","nonce":3124750952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6833","nonce":3124750952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6831","nonce":3124750952}]},"public_addr":"172.21.15.16:6827/3124750952","cluster_addr":"172.21.15.16:6829/3124750952","heartbeat_back_addr":"172.21.15.16:6833/3124750952","heartbeat_front_addr":"172.21.15.16:6831/3124750952","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:32:30.575 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T08:32:30.575 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T08:32:30.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T08:32:30.583 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:30 smithi094 bash[47070]: b45d31ee2d7f: Pull complete 2023-10-09T08:32:30.848 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:30 smithi094 bash[27803]: audit 2023-10-09T08:32:29.799106+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:30.848 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:30 smithi094 bash[27803]: audit 2023-10-09T08:32:30.557666+0000 mon.a (mon.0) 575 : audit [DBG] from='client.? 172.21.15.16:0/3716802999' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:32:30.848 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:30 smithi094 bash[47070]: b5db1e299295: Pull complete 2023-10-09T08:32:30.848 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:30 smithi094 bash[47070]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T08:32:30.848 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:30 smithi094 bash[47070]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T08:32:30.882 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:30 smithi016 bash[27454]: audit 2023-10-09T08:32:29.799106+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:30.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:30 smithi016 bash[27454]: audit 2023-10-09T08:32:30.557666+0000 mon.a (mon.0) 575 : audit [DBG] from='client.? 172.21.15.16:0/3716802999' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:32:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:30 smithi016 bash[33851]: audit 2023-10-09T08:32:29.799106+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:30 smithi016 bash[33851]: audit 2023-10-09T08:32:30.557666+0000 mon.a (mon.0) 575 : audit [DBG] from='client.? 172.21.15.16:0/3716802999' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:32:32.159 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:31 smithi094 bash[27803]: cluster 2023-10-09T08:32:30.605623+0000 mgr.y (mgr.24355) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:32.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:31 smithi016 bash[33851]: cluster 2023-10-09T08:32:30.605623+0000 mgr.y (mgr.24355) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:32.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:31 smithi016 bash[27454]: cluster 2023-10-09T08:32:30.605623+0000 mgr.y (mgr.24355) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:32.493 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.287Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T08:32:32.493 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.287Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T08:32:32.493 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T08:32:32.493 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T08:32:32.494 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T08:32:32.495 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T08:32:32.496 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.289Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.290Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.290Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.290Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T08:32:32.497 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[47070]: ts=2023-10-09T08:32:32.290Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T08:32:32.646 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.451Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.451Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T08:32:32.647 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T08:32:32.648 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T08:32:32.649 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T08:32:32.650 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T08:32:32.651 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T08:32:32.651 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T08:32:32.651 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T08:32:32.651 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T08:32:32.651 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.453Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T08:32:32.651 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[52639]: ts=2023-10-09T08:32:32.454Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T08:32:33.086 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[27454]: cluster 2023-10-09T08:32:32.606143+0000 mgr.y (mgr.24355) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:33.087 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:32 smithi016 bash[33851]: cluster 2023-10-09T08:32:32.606143+0000 mgr.y (mgr.24355) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:32 smithi094 bash[27803]: cluster 2023-10-09T08:32:32.606143+0000 mgr.y (mgr.24355) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:34.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:34 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:32:34] "GET /metrics HTTP/1.1" 200 153247 "" "Prometheus/2.33.4" 2023-10-09T08:32:35.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:35 smithi016 bash[27454]: cluster 2023-10-09T08:32:34.606730+0000 mgr.y (mgr.24355) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:35.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:35 smithi016 bash[33851]: cluster 2023-10-09T08:32:34.606730+0000 mgr.y (mgr.24355) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:35.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:35 smithi094 bash[27803]: cluster 2023-10-09T08:32:34.606730+0000 mgr.y (mgr.24355) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:36.679 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'rbd' created 2023-10-09T08:32:36.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:36 smithi016 bash[27454]: audit 2023-10-09T08:32:36.107427+0000 mon.a (mon.0) 576 : audit [INF] from='client.? 172.21.15.16:0/2568433882' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T08:32:36.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:36 smithi016 bash[33851]: audit 2023-10-09T08:32:36.107427+0000 mon.a (mon.0) 576 : audit [INF] from='client.? 172.21.15.16:0/2568433882' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T08:32:36.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:36 smithi094 bash[27803]: audit 2023-10-09T08:32:36.107427+0000 mon.a (mon.0) 576 : audit [INF] from='client.? 172.21.15.16:0/2568433882' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T08:32:37.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T08:32:37.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[27454]: cluster 2023-10-09T08:32:36.607310+0000 mgr.y (mgr.24355) 31 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:37.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[27454]: audit 2023-10-09T08:32:36.671652+0000 mon.a (mon.0) 577 : audit [INF] from='client.? 172.21.15.16:0/2568433882' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T08:32:37.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[27454]: cluster 2023-10-09T08:32:36.671720+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T08:32:37.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[27454]: audit 2023-10-09T08:32:36.672895+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:32:37.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[27454]: audit 2023-10-09T08:32:36.673305+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:32:37.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[27454]: audit 2023-10-09T08:32:36.674806+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:32:37.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[27454]: audit 2023-10-09T08:32:36.675414+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:32:37.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[33851]: cluster 2023-10-09T08:32:36.607310+0000 mgr.y (mgr.24355) 31 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:37.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[33851]: audit 2023-10-09T08:32:36.671652+0000 mon.a (mon.0) 577 : audit [INF] from='client.? 172.21.15.16:0/2568433882' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T08:32:37.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[33851]: cluster 2023-10-09T08:32:36.671720+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T08:32:37.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[33851]: audit 2023-10-09T08:32:36.672895+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:32:37.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[33851]: audit 2023-10-09T08:32:36.673305+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:32:37.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[33851]: audit 2023-10-09T08:32:36.674806+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:32:37.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:37 smithi016 bash[33851]: audit 2023-10-09T08:32:36.675414+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:32:37.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:37 smithi094 bash[27803]: cluster 2023-10-09T08:32:36.607310+0000 mgr.y (mgr.24355) 31 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:37.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:37 smithi094 bash[27803]: audit 2023-10-09T08:32:36.671652+0000 mon.a (mon.0) 577 : audit [INF] from='client.? 172.21.15.16:0/2568433882' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T08:32:37.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:37 smithi094 bash[27803]: cluster 2023-10-09T08:32:36.671720+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T08:32:37.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:37 smithi094 bash[27803]: audit 2023-10-09T08:32:36.672895+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:32:37.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:37 smithi094 bash[27803]: audit 2023-10-09T08:32:36.673305+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:32:37.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:37 smithi094 bash[27803]: audit 2023-10-09T08:32:36.674806+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:32:37.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:37 smithi094 bash[27803]: audit 2023-10-09T08:32:36.675414+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:32:38.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:38 smithi094 bash[27803]: audit 2023-10-09T08:32:37.676787+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:32:38.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:38 smithi094 bash[27803]: cluster 2023-10-09T08:32:37.677018+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T08:32:39.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:38 smithi016 bash[27454]: audit 2023-10-09T08:32:37.676787+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:32:39.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:38 smithi016 bash[27454]: cluster 2023-10-09T08:32:37.677018+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T08:32:39.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:38 smithi016 bash[33851]: audit 2023-10-09T08:32:37.676787+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:32:39.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:38 smithi016 bash[33851]: cluster 2023-10-09T08:32:37.677018+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T08:32:39.953 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:39 smithi016 bash[27454]: cluster 2023-10-09T08:32:38.607814+0000 mgr.y (mgr.24355) 32 : cluster [DBG] pgmap v22: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:39.954 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:39 smithi016 bash[27454]: cluster 2023-10-09T08:32:38.690215+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T08:32:39.954 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:39 smithi016 bash[33851]: cluster 2023-10-09T08:32:38.607814+0000 mgr.y (mgr.24355) 32 : cluster [DBG] pgmap v22: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:39.954 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:39 smithi016 bash[33851]: cluster 2023-10-09T08:32:38.690215+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T08:32:39.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:39 smithi094 bash[27803]: cluster 2023-10-09T08:32:38.607814+0000 mgr.y (mgr.24355) 32 : cluster [DBG] pgmap v22: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:39.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:39 smithi094 bash[27803]: cluster 2023-10-09T08:32:38.690215+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T08:32:40.501 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:40 smithi016 systemd[1]: Started Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:32:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:41 smithi016 bash[27454]: audit 2023-10-09T08:32:40.250474+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:41 smithi016 bash[27454]: audit 2023-10-09T08:32:40.254469+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:41 smithi016 bash[27454]: cluster 2023-10-09T08:32:40.608448+0000 mgr.y (mgr.24355) 33 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:41.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:41 smithi016 bash[33851]: audit 2023-10-09T08:32:40.250474+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:41.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:41 smithi016 bash[33851]: audit 2023-10-09T08:32:40.254469+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:41.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:41 smithi016 bash[33851]: cluster 2023-10-09T08:32:40.608448+0000 mgr.y (mgr.24355) 33 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:41.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:41 smithi094 bash[27803]: audit 2023-10-09T08:32:40.250474+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:41.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:41 smithi094 bash[27803]: audit 2023-10-09T08:32:40.254469+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:32:41.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:41 smithi094 bash[27803]: cluster 2023-10-09T08:32:40.608448+0000 mgr.y (mgr.24355) 33 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:42.501 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.230Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T08:32:42.501 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.230Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T08:32:42.501 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.232Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.16 port=9094 2023-10-09T08:32:42.501 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.233Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T08:32:42.502 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.283Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T08:32:42.502 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.283Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T08:32:42.502 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.289Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T08:32:42.502 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:42.289Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T08:32:42.723 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T08:32:42.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:42 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:32:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:32:42.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:42 smithi094 bash[27803]: audit 2023-10-09T08:32:42.656474+0000 mon.b (mon.1) 26 : audit [INF] from='client.? 172.21.15.16:0/543853464' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:32:42.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:42 smithi094 bash[27803]: audit 2023-10-09T08:32:42.656819+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:32:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[33851]: audit 2023-10-09T08:32:42.656474+0000 mon.b (mon.1) 26 : audit [INF] from='client.? 172.21.15.16:0/543853464' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:32:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[33851]: audit 2023-10-09T08:32:42.656819+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:32:43.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[27454]: audit 2023-10-09T08:32:42.656474+0000 mon.b (mon.1) 26 : audit [INF] from='client.? 172.21.15.16:0/543853464' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:32:43.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:42 smithi016 bash[27454]: audit 2023-10-09T08:32:42.656819+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:32:43.433 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T08:32:43.434 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T08:32:43.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T08:32:43.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:43 smithi094 bash[27803]: cluster 2023-10-09T08:32:42.608987+0000 mgr.y (mgr.24355) 34 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:43.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:43 smithi094 bash[27803]: audit 2023-10-09T08:32:42.714704+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:32:43.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:43 smithi094 bash[27803]: cluster 2023-10-09T08:32:42.714914+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T08:32:44.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:43 smithi016 bash[27454]: cluster 2023-10-09T08:32:42.608987+0000 mgr.y (mgr.24355) 34 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:44.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:43 smithi016 bash[27454]: audit 2023-10-09T08:32:42.714704+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:32:44.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:43 smithi016 bash[27454]: cluster 2023-10-09T08:32:42.714914+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T08:32:44.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:43 smithi016 bash[33851]: cluster 2023-10-09T08:32:42.608987+0000 mgr.y (mgr.24355) 34 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:44.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:43 smithi016 bash[33851]: audit 2023-10-09T08:32:42.714704+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:32:44.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:43 smithi016 bash[33851]: cluster 2023-10-09T08:32:42.714914+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T08:32:44.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:44 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:32:44] "GET /metrics HTTP/1.1" 200 153244 "" "Prometheus/2.33.4" 2023-10-09T08:32:44.501 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:44 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:44.234Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000160801s 2023-10-09T08:32:45.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:45 smithi094 bash[27803]: cluster 2023-10-09T08:32:44.609576+0000 mgr.y (mgr.24355) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:45.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:45 smithi094 bash[27803]: audit 2023-10-09T08:32:44.815513+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:46.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:45 smithi016 bash[27454]: cluster 2023-10-09T08:32:44.609576+0000 mgr.y (mgr.24355) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:46.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:45 smithi016 bash[27454]: audit 2023-10-09T08:32:44.815513+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:46.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:45 smithi016 bash[33851]: cluster 2023-10-09T08:32:44.609576+0000 mgr.y (mgr.24355) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:46.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:45 smithi016 bash[33851]: audit 2023-10-09T08:32:44.815513+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:47.232 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:46 smithi016 bash[27454]: audit 2023-10-09T08:32:45.942585+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:47.232 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:46 smithi016 bash[27454]: cluster 2023-10-09T08:32:46.610214+0000 mgr.y (mgr.24355) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:47.233 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27723]: [09/Oct/2023:08:32:47] ENGINE Bus STOPPING 2023-10-09T08:32:47.233 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27723]: [09/Oct/2023:08:32:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:32:47.233 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27723]: [09/Oct/2023:08:32:47] ENGINE Bus STOPPED 2023-10-09T08:32:47.233 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27723]: [09/Oct/2023:08:32:47] ENGINE Bus STARTING 2023-10-09T08:32:47.233 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27723]: [09/Oct/2023:08:32:47] ENGINE Serving on http://:::9283 2023-10-09T08:32:47.233 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:46 smithi016 bash[33851]: audit 2023-10-09T08:32:45.942585+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:47.233 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:46 smithi016 bash[33851]: cluster 2023-10-09T08:32:46.610214+0000 mgr.y (mgr.24355) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:47.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:46 smithi094 bash[27803]: audit 2023-10-09T08:32:45.942585+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:47.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:46 smithi094 bash[27803]: cluster 2023-10-09T08:32:46.610214+0000 mgr.y (mgr.24355) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:47.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27723]: [09/Oct/2023:08:32:47] ENGINE Bus STARTED 2023-10-09T08:32:48.044 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:32:48.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:46.966277+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:46.969222+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:32:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:46.970416+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:32:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:46.976968+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:47.079543+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:47.084862+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:47.089794+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T08:32:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:47.090556+0000 mgr.y (mgr.24355) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T08:32:48.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: audit 2023-10-09T08:32:47.096771+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:47 smithi094 bash[27803]: cephadm 2023-10-09T08:32:47.106256+0000 mgr.y (mgr.24355) 38 : cephadm [INF] Deploying daemon grafana.a on smithi094 2023-10-09T08:32:48.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:46.966277+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:46.969222+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:32:48.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:46.970416+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:32:48.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:46.976968+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:47.079543+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:47.084862+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:47.089794+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:47.090556+0000 mgr.y (mgr.24355) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: audit 2023-10-09T08:32:47.096771+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[27454]: cephadm 2023-10-09T08:32:47.106256+0000 mgr.y (mgr.24355) 38 : cephadm [INF] Deploying daemon grafana.a on smithi094 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:46.966277+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:46.969222+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:32:48.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:46.970416+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:32:48.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:46.976968+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:47.079543+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:47.084862+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:47.089794+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T08:32:48.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:47.090556+0000 mgr.y (mgr.24355) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T08:32:48.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: audit 2023-10-09T08:32:47.096771+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:48.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:47 smithi016 bash[33851]: cephadm 2023-10-09T08:32:47.106256+0000 mgr.y (mgr.24355) 38 : cephadm [INF] Deploying daemon grafana.a on smithi094 2023-10-09T08:32:49.229 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:48 smithi016 bash[27454]: cluster 2023-10-09T08:32:48.610968+0000 mgr.y (mgr.24355) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:49.229 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:48 smithi016 bash[33851]: cluster 2023-10-09T08:32:48.610968+0000 mgr.y (mgr.24355) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:49.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:48 smithi094 bash[27803]: cluster 2023-10-09T08:32:48.610968+0000 mgr.y (mgr.24355) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:51.015 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:32:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:51 smithi094 bash[27803]: cluster 2023-10-09T08:32:50.611712+0000 mgr.y (mgr.24355) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:51 smithi094 bash[27803]: audit 2023-10-09T08:32:51.003497+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.16:0/226103990' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:32:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:51 smithi016 bash[27454]: cluster 2023-10-09T08:32:50.611712+0000 mgr.y (mgr.24355) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:51 smithi016 bash[27454]: audit 2023-10-09T08:32:51.003497+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.16:0/226103990' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:32:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:51 smithi016 bash[33851]: cluster 2023-10-09T08:32:50.611712+0000 mgr.y (mgr.24355) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:51 smithi016 bash[33851]: audit 2023-10-09T08:32:51.003497+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.16:0/226103990' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:32:51.754 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":19,"active_gid":24355,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6800","nonce":1710200957},{"type":"v1","addr":"172.21.15.16:6801","nonce":1710200957}]},"active_addr":"172.21.15.16:6801/1710200957","active_change":"2023-10-09T08:32:04.579738+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":1478513102}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":1021312597}]}]} 2023-10-09T08:32:51.759 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T08:32:51.759 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T08:32:51.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T08:32:52.501 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:32:52 smithi016 bash[53269]: level=info ts=2023-10-09T08:32:52.238Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004106163s 2023-10-09T08:32:52.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:32:52 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:32:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:32:53.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:53 smithi094 bash[27803]: cluster 2023-10-09T08:32:52.612360+0000 mgr.y (mgr.24355) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:54.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:53 smithi016 bash[27454]: cluster 2023-10-09T08:32:52.612360+0000 mgr.y (mgr.24355) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:54.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:53 smithi016 bash[33851]: cluster 2023-10-09T08:32:52.612360+0000 mgr.y (mgr.24355) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:54.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:32:54 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:32:54] "GET /metrics HTTP/1.1" 200 155639 "" "Prometheus/2.33.4" 2023-10-09T08:32:55.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:55 smithi094 bash[27803]: cluster 2023-10-09T08:32:54.612950+0000 mgr.y (mgr.24355) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:55.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:55 smithi094 bash[27803]: audit 2023-10-09T08:32:54.829980+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:56.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:55 smithi016 bash[27454]: cluster 2023-10-09T08:32:54.612950+0000 mgr.y (mgr.24355) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:56.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:55 smithi016 bash[27454]: audit 2023-10-09T08:32:54.829980+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:56.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:55 smithi016 bash[33851]: cluster 2023-10-09T08:32:54.612950+0000 mgr.y (mgr.24355) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:56.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:55 smithi016 bash[33851]: audit 2023-10-09T08:32:54.829980+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:32:56.539 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:32:57.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:57 smithi016 bash[27454]: cluster 2023-10-09T08:32:56.613690+0000 mgr.y (mgr.24355) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:57.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:57 smithi016 bash[33851]: cluster 2023-10-09T08:32:56.613690+0000 mgr.y (mgr.24355) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:57 smithi094 bash[27803]: cluster 2023-10-09T08:32:56.613690+0000 mgr.y (mgr.24355) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:59.608 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:32:59.608 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":55,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:32:42.707202+0000","last_up_change":"2023-10-09T08:31:29.882559+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6803","nonce":818359725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6805","nonce":818359725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6809","nonce":818359725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6807","nonce":818359725}]},"public_addr":"172.21.15.16:6803/818359725","cluster_addr":"172.21.15.16:6805/818359725","heartbeat_back_addr":"172.21.15.16:6809/818359725","heartbeat_front_addr":"172.21.15.16:6807/818359725","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6811","nonce":664142092}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6813","nonce":664142092}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6817","nonce":664142092}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6815","nonce":664142092}]},"public_addr":"172.21.15.16:6811/664142092","cluster_addr":"172.21.15.16:6813/664142092","heartbeat_back_addr":"172.21.15.16:6817/664142092","heartbeat_front_addr":"172.21.15.16:6815/664142092","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6819","nonce":3541024001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6821","nonce":3541024001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6825","nonce":3541024001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6823","nonce":3541024001}]},"public_addr":"172.21.15.16:6819/3541024001","cluster_addr":"172.21.15.16:6821/3541024001","heartbeat_back_addr":"172.21.15.16:6825/3541024001","heartbeat_front_addr":"172.21.15.16:6823/3541024001","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6827","nonce":3124750952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6829","nonce":3124750952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6833","nonce":3124750952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6831","nonce":3124750952}]},"public_addr":"172.21.15.16:6827/3124750952","cluster_addr":"172.21.15.16:6829/3124750952","heartbeat_back_addr":"172.21.15.16:6833/3124750952","heartbeat_front_addr":"172.21.15.16:6831/3124750952","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:32:59.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:59 smithi094 bash[27803]: cluster 2023-10-09T08:32:58.614207+0000 mgr.y (mgr.24355) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:32:59.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:32:59 smithi094 bash[27803]: audit 2023-10-09T08:32:59.603882+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.16:0/2896607627' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:33:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:59 smithi016 bash[27454]: cluster 2023-10-09T08:32:58.614207+0000 mgr.y (mgr.24355) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:32:59 smithi016 bash[27454]: audit 2023-10-09T08:32:59.603882+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.16:0/2896607627' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:33:00.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:59 smithi016 bash[33851]: cluster 2023-10-09T08:32:58.614207+0000 mgr.y (mgr.24355) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:00.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:32:59 smithi016 bash[33851]: audit 2023-10-09T08:32:59.603882+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.16:0/2896607627' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:33:00.406 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T08:33:00.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T08:33:01.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:01 smithi094 bash[27803]: cluster 2023-10-09T08:33:00.614815+0000 mgr.y (mgr.24355) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:01.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:01 smithi016 bash[27454]: cluster 2023-10-09T08:33:00.614815+0000 mgr.y (mgr.24355) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:01.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:01 smithi016 bash[33851]: cluster 2023-10-09T08:33:00.614815+0000 mgr.y (mgr.24355) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:02.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:33:02 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:33:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:03 smithi094 bash[27803]: cluster 2023-10-09T08:33:02.615424+0000 mgr.y (mgr.24355) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:04.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:03 smithi016 bash[27454]: cluster 2023-10-09T08:33:02.615424+0000 mgr.y (mgr.24355) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:04.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:03 smithi016 bash[33851]: cluster 2023-10-09T08:33:02.615424+0000 mgr.y (mgr.24355) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:04.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:04 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:04] "GET /metrics HTTP/1.1" 200 155639 "" "Prometheus/2.33.4" 2023-10-09T08:33:05.181 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:05.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:05 smithi094 bash[27803]: cluster 2023-10-09T08:33:04.616019+0000 mgr.y (mgr.24355) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:05 smithi016 bash[27454]: cluster 2023-10-09T08:33:04.616019+0000 mgr.y (mgr.24355) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:06.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:05 smithi016 bash[33851]: cluster 2023-10-09T08:33:04.616019+0000 mgr.y (mgr.24355) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:07.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:07 smithi094 bash[27803]: cluster 2023-10-09T08:33:06.616814+0000 mgr.y (mgr.24355) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:08.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:07 smithi016 bash[27454]: cluster 2023-10-09T08:33:06.616814+0000 mgr.y (mgr.24355) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:08.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:07 smithi016 bash[33851]: cluster 2023-10-09T08:33:06.616814+0000 mgr.y (mgr.24355) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:08.222 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:33:08.222 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":55,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:32:42.707202+0000","last_up_change":"2023-10-09T08:31:29.882559+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6803","nonce":818359725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6805","nonce":818359725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6809","nonce":818359725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6807","nonce":818359725}]},"public_addr":"172.21.15.16:6803/818359725","cluster_addr":"172.21.15.16:6805/818359725","heartbeat_back_addr":"172.21.15.16:6809/818359725","heartbeat_front_addr":"172.21.15.16:6807/818359725","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6811","nonce":664142092}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6813","nonce":664142092}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6817","nonce":664142092}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6815","nonce":664142092}]},"public_addr":"172.21.15.16:6811/664142092","cluster_addr":"172.21.15.16:6813/664142092","heartbeat_back_addr":"172.21.15.16:6817/664142092","heartbeat_front_addr":"172.21.15.16:6815/664142092","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6819","nonce":3541024001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6821","nonce":3541024001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6825","nonce":3541024001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6823","nonce":3541024001}]},"public_addr":"172.21.15.16:6819/3541024001","cluster_addr":"172.21.15.16:6821/3541024001","heartbeat_back_addr":"172.21.15.16:6825/3541024001","heartbeat_front_addr":"172.21.15.16:6823/3541024001","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6827","nonce":3124750952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6829","nonce":3124750952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6833","nonce":3124750952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6831","nonce":3124750952}]},"public_addr":"172.21.15.16:6827/3124750952","cluster_addr":"172.21.15.16:6829/3124750952","heartbeat_back_addr":"172.21.15.16:6833/3124750952","heartbeat_front_addr":"172.21.15.16:6831/3124750952","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:33:08.405 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:08 smithi094 systemd[1]: Started Ceph grafana.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:33:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:08 smithi016 bash[33851]: audit 2023-10-09T08:33:08.217581+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.16:0/3266001982' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:33:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:08 smithi016 bash[33851]: audit 2023-10-09T08:33:08.388799+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:08 smithi016 bash[33851]: audit 2023-10-09T08:33:08.392197+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:08.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:08 smithi016 bash[27454]: audit 2023-10-09T08:33:08.217581+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.16:0/3266001982' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:33:08.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:08 smithi016 bash[27454]: audit 2023-10-09T08:33:08.388799+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:08.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:08 smithi016 bash[27454]: audit 2023-10-09T08:33:08.392197+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:08 smithi094 bash[27803]: audit 2023-10-09T08:33:08.217581+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.16:0/3266001982' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:33:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:08 smithi094 bash[27803]: audit 2023-10-09T08:33:08.388799+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:08.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:08 smithi094 bash[27803]: audit 2023-10-09T08:33:08.392197+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:09.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T08:33:09.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T08:33:09.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T08:33:09.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T08:33:09.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T08:33:09.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T08:33:09.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T08:33:09.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T08:33:09.689 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T08:33:09.689 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T08:33:09.690 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T08:33:09.691 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T08:33:09.691 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T08:33:09.691 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T08:33:09.691 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T08:33:09.691 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-10-09T08:33:09.691 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-10-09T08:33:09.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[27803]: cluster 2023-10-09T08:33:08.617205+0000 mgr.y (mgr.24355) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:09.964 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-10-09T08:33:09.964 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-09T08:33:09.964 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-09T08:33:09.964 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-09T08:33:09.964 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-09T08:33:09.964 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-09T08:33:09.965 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-09T08:33:09.965 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-09T08:33:09.965 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T08:33:10.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:09 smithi016 bash[27454]: cluster 2023-10-09T08:33:08.617205+0000 mgr.y (mgr.24355) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:10.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:09 smithi016 bash[33851]: cluster 2023-10-09T08:33:08.617205+0000 mgr.y (mgr.24355) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:10.237 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:09 smithi094 bash[47834]: t=2023-10-09T08:33:09+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-09T08:33:10.238 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-09T08:33:10.238 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-09T08:33:10.238 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-09T08:33:10.238 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-09T08:33:10.238 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-09T08:33:10.238 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-09T08:33:10.239 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-09T08:33:10.239 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-09T08:33:10.487 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-09T08:33:10.487 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T08:33:10.487 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-09T08:33:10.487 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-09T08:33:10.487 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-09T08:33:10.487 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-09T08:33:10.487 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-09T08:33:10.742 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-09T08:33:10.742 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-09T08:33:10.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T08:33:10.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-09T08:33:10.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-09T08:33:10.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-09T08:33:10.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-09T08:33:11.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-09T08:33:11.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-09T08:33:11.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-09T08:33:11.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T08:33:11.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-09T08:33:11.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[47834]: t=2023-10-09T08:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-09T08:33:11.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-09T08:33:11.035 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[27803]: audit 2023-10-09T08:33:09.851346+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:11.035 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:10 smithi094 bash[27803]: cluster 2023-10-09T08:33:10.617670+0000 mgr.y (mgr.24355) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:11.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:10 smithi016 bash[27454]: audit 2023-10-09T08:33:09.851346+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:11.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:10 smithi016 bash[27454]: cluster 2023-10-09T08:33:10.617670+0000 mgr.y (mgr.24355) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:11.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:10 smithi016 bash[33851]: audit 2023-10-09T08:33:09.851346+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:11.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:10 smithi016 bash[33851]: cluster 2023-10-09T08:33:10.617670+0000 mgr.y (mgr.24355) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:11.286 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-09T08:33:11.287 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-09T08:33:11.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-09T08:33:11.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-09T08:33:11.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-09T08:33:11.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T08:33:11.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T08:33:11.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T08:33:11.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-09T08:33:11.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-09T08:33:11.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T08:33:11.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-09T08:33:11.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-09T08:33:11.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-09T08:33:11.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-09T08:33:11.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-09T08:33:11.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-09T08:33:11.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-09T08:33:11.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-09T08:33:11.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-09T08:33:12.103 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-09T08:33:12.103 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-09T08:33:12.103 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-09T08:33:12.103 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-09T08:33:12.104 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-09T08:33:12.104 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-09T08:33:12.104 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:11 smithi094 bash[47834]: t=2023-10-09T08:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-09T08:33:12.104 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-09T08:33:12.104 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-09T08:33:12.104 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-09T08:33:12.364 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T08:33:12.616 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:33:12.617 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-09T08:33:12.617 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-09T08:33:12.617 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-09T08:33:12.617 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-09T08:33:12.617 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-09T08:33:12.617 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-09T08:33:12.617 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-09T08:33:12.886 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T08:33:13.155 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-09T08:33:13.155 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-09T08:33:13.155 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:12 smithi094 bash[47834]: t=2023-10-09T08:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-09T08:33:13.155 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-09T08:33:13.155 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-09T08:33:13.155 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-09T08:33:13.156 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-09T08:33:13.156 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-09T08:33:13.156 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-09T08:33:13.179 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.180 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.181 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.183 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.184 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.184 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.186 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.194 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:13.424 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-09T08:33:13.424 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-09T08:33:13.424 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-09T08:33:13.424 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-09T08:33:13.424 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-09T08:33:13.424 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-09T08:33:13.425 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-09T08:33:13.676 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-09T08:33:13.676 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-09T08:33:13.676 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T08:33:13.677 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-09T08:33:13.677 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-09T08:33:13.677 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-09T08:33:13.677 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-09T08:33:13.937 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[27803]: cluster 2023-10-09T08:33:12.618340+0000 mgr.y (mgr.24355) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-09T08:33:13.938 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-09T08:33:14.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:13 smithi016 bash[27454]: cluster 2023-10-09T08:33:12.618340+0000 mgr.y (mgr.24355) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:13 smithi016 bash[33851]: cluster 2023-10-09T08:33:12.618340+0000 mgr.y (mgr.24355) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:14.206 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:13 smithi094 bash[47834]: t=2023-10-09T08:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T08:33:14.206 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T08:33:14.207 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T08:33:14.207 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T08:33:14.207 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-09T08:33:14.207 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T08:33:14.207 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-09T08:33:14.467 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T08:33:14.468 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-09T08:33:14.468 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T08:33:14.468 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-09T08:33:14.468 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-09T08:33:14.468 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T08:33:14.468 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-09T08:33:14.468 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-09T08:33:14.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:14 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:14] "GET /metrics HTTP/1.1" 200 155635 "" "Prometheus/2.33.4" 2023-10-09T08:33:14.737 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-09T08:33:14.737 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-09T08:33:14.737 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-09T08:33:14.737 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-09T08:33:14.737 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-09T08:33:14.990 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-09T08:33:14.990 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-09T08:33:14.990 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-09T08:33:14.990 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-09T08:33:14.990 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-09T08:33:14.990 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-09T08:33:14.991 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[47834]: t=2023-10-09T08:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-09T08:33:15.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[27803]: audit 2023-10-09T08:33:13.986318+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:15.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:14 smithi094 bash[27803]: cluster 2023-10-09T08:33:14.619073+0000 mgr.y (mgr.24355) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:15.242 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-09T08:33:15.242 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-09T08:33:15.242 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-09T08:33:15.243 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-09T08:33:15.243 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-09T08:33:15.243 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-09T08:33:15.243 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-09T08:33:15.243 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T08:33:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:14 smithi016 bash[33851]: audit 2023-10-09T08:33:13.986318+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:14 smithi016 bash[33851]: cluster 2023-10-09T08:33:14.619073+0000 mgr.y (mgr.24355) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:15.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:14 smithi016 bash[27454]: audit 2023-10-09T08:33:13.986318+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:15.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:14 smithi016 bash[27454]: cluster 2023-10-09T08:33:14.619073+0000 mgr.y (mgr.24355) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:15.492 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T08:33:15.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T08:33:15.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-09T08:33:15.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T08:33:15.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-09T08:33:15.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-09T08:33:15.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-09T08:33:15.790 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-09T08:33:15.790 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-09T08:33:15.790 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-09T08:33:15.790 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-09T08:33:15.791 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-09T08:33:15.791 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-09T08:33:15.791 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-09T08:33:15.791 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-09T08:33:15.791 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T08:33:16.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-09T08:33:16.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-09T08:33:16.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T08:33:16.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-09T08:33:16.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-09T08:33:16.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-09T08:33:16.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:15 smithi094 bash[47834]: t=2023-10-09T08:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-09T08:33:16.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-09T08:33:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[27454]: audit 2023-10-09T08:33:15.158465+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[27454]: audit 2023-10-09T08:33:15.428536+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:33:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[27454]: audit 2023-10-09T08:33:15.430444+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:33:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[27454]: audit 2023-10-09T08:33:15.438251+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:16.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[27454]: cephadm 2023-10-09T08:33:15.446749+0000 mgr.y (mgr.24355) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T08:33:16.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[27454]: cephadm 2023-10-09T08:33:15.451412+0000 mgr.y (mgr.24355) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-10-09T08:33:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[33851]: audit 2023-10-09T08:33:15.158465+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[33851]: audit 2023-10-09T08:33:15.428536+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:33:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[33851]: audit 2023-10-09T08:33:15.430444+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:33:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[33851]: audit 2023-10-09T08:33:15.438251+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:16.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[33851]: cephadm 2023-10-09T08:33:15.446749+0000 mgr.y (mgr.24355) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T08:33:16.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:16 smithi016 bash[33851]: cephadm 2023-10-09T08:33:15.451412+0000 mgr.y (mgr.24355) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-10-09T08:33:16.303 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[27803]: audit 2023-10-09T08:33:15.158465+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:16.303 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[27803]: audit 2023-10-09T08:33:15.428536+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:33:16.303 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[27803]: audit 2023-10-09T08:33:15.430444+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:33:16.304 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[27803]: audit 2023-10-09T08:33:15.438251+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:16.304 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[27803]: cephadm 2023-10-09T08:33:15.446749+0000 mgr.y (mgr.24355) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T08:33:16.304 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[27803]: cephadm 2023-10-09T08:33:15.451412+0000 mgr.y (mgr.24355) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-10-09T08:33:16.305 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-09T08:33:16.305 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-09T08:33:16.305 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T08:33:16.305 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T08:33:16.305 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-09T08:33:16.306 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-09T08:33:16.306 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-09T08:33:16.306 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-09T08:33:16.572 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-09T08:33:16.572 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-09T08:33:16.572 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-09T08:33:16.573 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-09T08:33:16.573 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-09T08:33:16.573 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-09T08:33:16.573 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-09T08:33:16.574 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-09T08:33:16.574 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T08:33:16.833 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T08:33:16.848 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T08:33:16.848 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-09T08:33:16.848 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T08:33:16.849 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-09T08:33:16.849 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-09T08:33:16.849 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-09T08:33:16.849 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-09T08:33:16.850 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-09T08:33:17.085 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-09T08:33:17.086 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-09T08:33:17.086 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-09T08:33:17.086 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-09T08:33:17.086 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:16 smithi094 bash[47834]: t=2023-10-09T08:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-09T08:33:17.086 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-09T08:33:17.087 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-09T08:33:17.354 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[27803]: cluster 2023-10-09T08:33:16.619908+0000 mgr.y (mgr.24355) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-09T08:33:17.355 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-09T08:33:17.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:17 smithi016 bash[27454]: cluster 2023-10-09T08:33:16.619908+0000 mgr.y (mgr.24355) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:17.439 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:17 smithi016 bash[33851]: cluster 2023-10-09T08:33:16.619908+0000 mgr.y (mgr.24355) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-09T08:33:17.607 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-09T08:33:17.876 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-09T08:33:17.877 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-09T08:33:18.137 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T08:33:18.137 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T08:33:18.137 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-09T08:33:18.138 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:17 smithi094 bash[47834]: t=2023-10-09T08:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-09T08:33:18.138 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-09T08:33:18.138 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-09T08:33:18.138 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-09T08:33:18.138 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-09T08:33:18.138 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-09T08:33:18.398 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-09T08:33:18.398 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-09T08:33:18.398 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-09T08:33:18.398 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-09T08:33:18.398 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T08:33:18.398 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-09T08:33:18.398 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-09T08:33:18.399 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-09T08:33:18.683 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-09T08:33:18.684 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-09T08:33:18.684 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T08:33:18.684 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-09T08:33:18.684 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-09T08:33:18.684 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-09T08:33:18.684 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-09T08:33:18.944 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-09T08:33:18.945 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-09T08:33:18.945 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-09T08:33:18.945 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-09T08:33:18.945 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-09T08:33:18.945 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-09T08:33:18.945 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-09T08:33:18.945 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-09T08:33:19.230 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:18 smithi094 bash[47834]: t=2023-10-09T08:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-09T08:33:19.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T08:33:19.483 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-09T08:33:19.483 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-09T08:33:19.483 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-09T08:33:19.483 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T08:33:19.483 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-09T08:33:19.483 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-09T08:33:19.483 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-09T08:33:19.484 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-09T08:33:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[27803]: cluster 2023-10-09T08:33:18.620384+0000 mgr.y (mgr.24355) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T08:33:19.743 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-09T08:33:19.891 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:19 smithi016 bash[27454]: cluster 2023-10-09T08:33:18.620384+0000 mgr.y (mgr.24355) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:19.891 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:19 smithi016 bash[33851]: cluster 2023-10-09T08:33:18.620384+0000 mgr.y (mgr.24355) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:20.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:19 smithi094 bash[47834]: t=2023-10-09T08:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-09T08:33:20.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-09T08:33:20.307 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T08:33:20.307 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T08:33:20.308 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-09T08:33:20.308 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-09T08:33:20.308 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-09T08:33:20.308 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T08:33:20.308 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T08:33:20.308 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-09T08:33:20.308 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-09T08:33:20.594 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T08:33:20.594 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-09T08:33:20.594 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T08:33:20.594 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T08:33:20.594 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T08:33:20.595 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-09T08:33:20.595 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-09T08:33:20.595 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-09T08:33:20.595 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-09T08:33:20.855 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-09T08:33:20.855 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T08:33:20.855 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-09T08:33:20.855 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-09T08:33:20.855 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-09T08:33:20.855 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-09T08:33:20.855 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-09T08:33:20.856 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-09T08:33:20.856 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-09T08:33:21.133 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-09T08:33:21.133 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-09T08:33:21.133 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T08:33:21.133 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-09T08:33:21.134 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-09T08:33:21.134 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:20 smithi094 bash[47834]: t=2023-10-09T08:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-09T08:33:21.134 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-09T08:33:21.134 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-09T08:33:21.134 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T08:33:21.135 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-09T08:33:21.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[27803]: cluster 2023-10-09T08:33:20.620948+0000 mgr.y (mgr.24355) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:21.385 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-09T08:33:21.386 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-09T08:33:21.386 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-09T08:33:21.386 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-09T08:33:21.386 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-09T08:33:21.386 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-09T08:33:21.386 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-09T08:33:21.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:21 smithi016 bash[27454]: cluster 2023-10-09T08:33:20.620948+0000 mgr.y (mgr.24355) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:21.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:21 smithi016 bash[33851]: cluster 2023-10-09T08:33:20.620948+0000 mgr.y (mgr.24355) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:21.637 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-09T08:33:21.638 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-09T08:33:21.638 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-09T08:33:21.638 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T08:33:21.638 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-09T08:33:21.638 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-09T08:33:21.638 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-09T08:33:21.638 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T08:33:21.923 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-09T08:33:21.923 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-09T08:33:21.924 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-09T08:33:21.924 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-09T08:33:21.924 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-09T08:33:21.924 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-09T08:33:21.924 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T08:33:21.924 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-09T08:33:21.924 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.396723028s 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:21 smithi094 bash[47834]: t=2023-10-09T08:33:21+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T08:33:22.229 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T08:33:22.492 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T08:33:22.492 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T08:33:22.492 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T08:33:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[47834]: t=2023-10-09T08:33:22+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T08:33:22.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:33:22 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:33:23.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:23 smithi094 bash[27803]: cluster 2023-10-09T08:33:22.621518+0000 mgr.y (mgr.24355) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:24.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:23 smithi016 bash[33851]: cluster 2023-10-09T08:33:22.621518+0000 mgr.y (mgr.24355) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:24.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:23 smithi016 bash[27454]: cluster 2023-10-09T08:33:22.621518+0000 mgr.y (mgr.24355) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:24.326 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:24 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:24] "GET /metrics HTTP/1.1" 200 155635 "" "Prometheus/2.33.4" 2023-10-09T08:33:25.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:25 smithi016 bash[27454]: cluster 2023-10-09T08:33:24.622137+0000 mgr.y (mgr.24355) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:25.826 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:25 smithi016 bash[33851]: cluster 2023-10-09T08:33:24.622137+0000 mgr.y (mgr.24355) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:25.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:25 smithi094 bash[27803]: cluster 2023-10-09T08:33:24.622137+0000 mgr.y (mgr.24355) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:27 smithi094 bash[27803]: cluster 2023-10-09T08:33:26.622718+0000 mgr.y (mgr.24355) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:28.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:27 smithi016 bash[27454]: cluster 2023-10-09T08:33:26.622718+0000 mgr.y (mgr.24355) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:28.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:27 smithi016 bash[33851]: cluster 2023-10-09T08:33:26.622718+0000 mgr.y (mgr.24355) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:28.779 INFO:teuthology.orchestra.run.smithi016.stdout:128849018932 2023-10-09T08:33:28.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T08:33:29.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:29 smithi094 bash[27803]: cluster 2023-10-09T08:33:28.623066+0000 mgr.y (mgr.24355) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:29 smithi016 bash[27454]: cluster 2023-10-09T08:33:28.623066+0000 mgr.y (mgr.24355) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:30.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:29 smithi016 bash[33851]: cluster 2023-10-09T08:33:28.623066+0000 mgr.y (mgr.24355) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:31.433 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:31 smithi016 bash[27454]: cluster 2023-10-09T08:33:30.623686+0000 mgr.y (mgr.24355) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:31.433 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:31 smithi016 bash[33851]: cluster 2023-10-09T08:33:30.623686+0000 mgr.y (mgr.24355) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:31 smithi094 bash[27803]: cluster 2023-10-09T08:33:30.623686+0000 mgr.y (mgr.24355) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:31.653 INFO:teuthology.orchestra.run.smithi016.stdout:34359738457 2023-10-09T08:33:31.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T08:33:32.134 INFO:teuthology.orchestra.run.smithi016.stdout:206158430234 2023-10-09T08:33:32.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T08:33:32.624 INFO:teuthology.orchestra.run.smithi016.stdout:107374182462 2023-10-09T08:33:32.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T08:33:32.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:33:32 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:33:33.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:33 smithi016 bash[33851]: cluster 2023-10-09T08:33:32.624148+0000 mgr.y (mgr.24355) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:33.950 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:33 smithi016 bash[27454]: cluster 2023-10-09T08:33:32.624148+0000 mgr.y (mgr.24355) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:33.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:33 smithi094 bash[27803]: cluster 2023-10-09T08:33:32.624148+0000 mgr.y (mgr.24355) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:34.262 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:34 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:34] "GET /metrics HTTP/1.1" 200 155635 "" "Prometheus/2.33.4" 2023-10-09T08:33:34.535 INFO:teuthology.orchestra.run.smithi016.stdout:55834574927 2023-10-09T08:33:34.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T08:33:34.577 INFO:teuthology.orchestra.run.smithi016.stdout:77309411399 2023-10-09T08:33:34.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T08:33:35.684 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:35 smithi016 systemd[1]: Stopping Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:33:35.684 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:35 smithi016 bash[56704]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T08:33:35.685 INFO:teuthology.orchestra.run.smithi016.stdout:180388626468 2023-10-09T08:33:35.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T08:33:35.762 INFO:teuthology.orchestra.run.smithi016.stdout:154618822702 2023-10-09T08:33:35.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T08:33:35.955 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:35 smithi016 bash[27454]: cluster 2023-10-09T08:33:34.624533+0000 mgr.y (mgr.24355) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:35.955 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:35 smithi016 bash[33851]: cluster 2023-10-09T08:33:34.624533+0000 mgr.y (mgr.24355) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:35.956 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:35 smithi016 bash[53269]: level=info ts=2023-10-09T08:33:35.680Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T08:33:35.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:35 smithi094 bash[27803]: cluster 2023-10-09T08:33:34.624533+0000 mgr.y (mgr.24355) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:36.581 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:36 smithi016 bash[56738]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T08:33:36.581 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:36 smithi016 bash[56909]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T08:33:36.582 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:36 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T08:33:36.582 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:36 smithi016 systemd[1]: Stopped Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:33:36.582 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:36 smithi016 systemd[1]: Started Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:33:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:37 smithi094 bash[27803]: audit 2023-10-09T08:33:36.554218+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:37 smithi094 bash[27803]: cephadm 2023-10-09T08:33:36.556308+0000 mgr.y (mgr.24355) 65 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:33:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:37 smithi094 bash[27803]: cephadm 2023-10-09T08:33:36.560680+0000 mgr.y (mgr.24355) 66 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:33:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:37 smithi094 bash[27803]: cluster 2023-10-09T08:33:36.625181+0000 mgr.y (mgr.24355) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:37.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[27454]: audit 2023-10-09T08:33:36.554218+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:37.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[27454]: cephadm 2023-10-09T08:33:36.556308+0000 mgr.y (mgr.24355) 65 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:33:37.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[27454]: cephadm 2023-10-09T08:33:36.560680+0000 mgr.y (mgr.24355) 66 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:33:37.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[27454]: cluster 2023-10-09T08:33:36.625181+0000 mgr.y (mgr.24355) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:37.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[33851]: audit 2023-10-09T08:33:36.554218+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:37.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[33851]: cephadm 2023-10-09T08:33:36.556308+0000 mgr.y (mgr.24355) 65 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:33:37.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[33851]: cephadm 2023-10-09T08:33:36.560680+0000 mgr.y (mgr.24355) 66 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:33:37.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[33851]: cluster 2023-10-09T08:33:36.625181+0000 mgr.y (mgr.24355) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:37.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.608Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T08:33:37.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.608Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T08:33:37.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.611Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.16 port=9094 2023-10-09T08:33:37.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.612Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T08:33:37.753 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.663Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T08:33:37.753 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.664Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T08:33:37.753 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.667Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T08:33:37.753 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:37 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:37.667Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T08:33:39.955 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:39 smithi094 bash[27803]: cluster 2023-10-09T08:33:38.625739+0000 mgr.y (mgr.24355) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:40.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:39 smithi016 bash[27454]: cluster 2023-10-09T08:33:38.625739+0000 mgr.y (mgr.24355) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:40.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:39 smithi016 bash[33851]: cluster 2023-10-09T08:33:38.625739+0000 mgr.y (mgr.24355) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:40.002 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:39 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:39.613Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000590969s 2023-10-09T08:33:40.062 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.063 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.064 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.065 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.065 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.066 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.066 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.070 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:33:40.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 systemd[1]: Stopping Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[48419]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.860Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.860Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.860Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.860Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.860Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.860Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.861Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.860Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.861Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T08:33:40.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.861Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T08:33:40.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.861Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T08:33:40.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:40 smithi094 bash[46955]: ts=2023-10-09T08:33:40.862Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T08:33:41.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:41 smithi094 bash[27803]: cluster 2023-10-09T08:33:40.626394+0000 mgr.y (mgr.24355) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27454]: cluster 2023-10-09T08:33:40.626394+0000 mgr.y (mgr.24355) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:41.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[33851]: cluster 2023-10-09T08:33:40.626394+0000 mgr.y (mgr.24355) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:41.711 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:41 smithi094 bash[48456]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T08:33:41.711 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:41 smithi094 bash[48525]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T08:33:41.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:41 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T08:33:41.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:41 smithi094 systemd[1]: Stopped Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:33:41.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:41 smithi094 systemd[1]: Started Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Bus STOPPING 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Bus STOPPED 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Bus STARTING 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Serving on http://:::9283 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Bus STARTED 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Bus STOPPING 2023-10-09T08:33:42.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Bus STOPPED 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:41 smithi016 bash[27723]: [09/Oct/2023:08:33:41] ENGINE Bus STARTING 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE Serving on http://:::9283 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE Bus STARTED 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE Bus STOPPING 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE Bus STOPPED 2023-10-09T08:33:42.048 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE Bus STARTING 2023-10-09T08:33:42.403 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE Serving on http://:::9283 2023-10-09T08:33:42.404 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27723]: [09/Oct/2023:08:33:42] ENGINE Bus STARTED 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.866Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.866Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.866Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi094 (none))" 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.866Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.866Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.869Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.869Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T08:33:42.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.870Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T08:33:42.995 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.875Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T08:33:42.995 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.875Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.743µs 2023-10-09T08:33:42.995 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.875Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T08:33:42.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.756268+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:42.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.762301+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:33:42.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.762919+0000 mgr.y (mgr.24355) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.764442+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.764875+0000 mgr.y (mgr.24355) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.770978+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.782476+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.783039+0000 mgr.y (mgr.24355) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.786883+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.787530+0000 mgr.y (mgr.24355) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T08:33:42.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.794948+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:42.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.805373+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:33:42.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.806324+0000 mgr.y (mgr.24355) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:33:42.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.812609+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T08:33:42.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.813086+0000 mgr.y (mgr.24355) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T08:33:42.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.820945+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:42.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[27803]: audit 2023-10-09T08:33:41.828272+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:43.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.756268+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.762301+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:33:43.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.762919+0000 mgr.y (mgr.24355) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:33:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.764442+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T08:33:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.764875+0000 mgr.y (mgr.24355) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T08:33:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.770978+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.782476+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:33:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.783039+0000 mgr.y (mgr.24355) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:33:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.786883+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T08:33:43.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.787530+0000 mgr.y (mgr.24355) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.794948+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.805373+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.806324+0000 mgr.y (mgr.24355) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.812609+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.813086+0000 mgr.y (mgr.24355) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.820945+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[27454]: audit 2023-10-09T08:33:41.828272+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:43.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.756268+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.762301+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.762919+0000 mgr.y (mgr.24355) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.764442+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.764875+0000 mgr.y (mgr.24355) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.770978+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.782476+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.783039+0000 mgr.y (mgr.24355) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.786883+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T08:33:43.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.787530+0000 mgr.y (mgr.24355) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T08:33:43.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.794948+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.805373+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:33:43.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.806324+0000 mgr.y (mgr.24355) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:33:43.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.812609+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T08:33:43.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.813086+0000 mgr.y (mgr.24355) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T08:33:43.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.820945+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:43.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:42 smithi016 bash[33851]: audit 2023-10-09T08:33:41.828272+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:43.492 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:42 smithi094 bash[48555]: ts=2023-10-09T08:33:42.999Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T08:33:43.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[48555]: ts=2023-10-09T08:33:42.999Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T08:33:43.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[48555]: ts=2023-10-09T08:33:43.000Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=61.959µs wal_replay_duration=124.056753ms total_replay_duration=124.156231ms 2023-10-09T08:33:43.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[48555]: ts=2023-10-09T08:33:43.003Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T08:33:43.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[48555]: ts=2023-10-09T08:33:43.003Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T08:33:43.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[48555]: ts=2023-10-09T08:33:43.003Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T08:33:43.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[48555]: ts=2023-10-09T08:33:43.036Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=33.052485ms db_storage=1.279µs remote_storage=1.718µs web_handler=606ns query_engine=941ns scrape=343.849µs scrape_sd=52.242µs notify=26.961µs notify_sd=16.864µs rules=32.157787ms 2023-10-09T08:33:43.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[48555]: ts=2023-10-09T08:33:43.036Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T08:33:44.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:43 smithi016 bash[27454]: cluster 2023-10-09T08:33:42.626812+0000 mgr.y (mgr.24355) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:44.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:43 smithi016 bash[33851]: cluster 2023-10-09T08:33:42.626812+0000 mgr.y (mgr.24355) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:44.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:43 smithi094 bash[27803]: cluster 2023-10-09T08:33:42.626812+0000 mgr.y (mgr.24355) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:46.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:45 smithi016 bash[27454]: cluster 2023-10-09T08:33:44.627443+0000 mgr.y (mgr.24355) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:46.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:45 smithi016 bash[33851]: cluster 2023-10-09T08:33:44.627443+0000 mgr.y (mgr.24355) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:45 smithi094 bash[27803]: cluster 2023-10-09T08:33:44.627443+0000 mgr.y (mgr.24355) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:47.871 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:47 smithi016 bash[27454]: cluster 2023-10-09T08:33:46.628075+0000 mgr.y (mgr.24355) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:47.871 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:47 smithi016 bash[27454]: audit 2023-10-09T08:33:47.722241+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:47.871 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:33:47 smithi016 bash[56937]: level=info ts=2023-10-09T08:33:47.616Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003297563s 2023-10-09T08:33:48.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:47 smithi094 bash[27803]: cluster 2023-10-09T08:33:46.628075+0000 mgr.y (mgr.24355) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:48.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:47 smithi094 bash[27803]: audit 2023-10-09T08:33:47.722241+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:48.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:47 smithi016 bash[33851]: cluster 2023-10-09T08:33:46.628075+0000 mgr.y (mgr.24355) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:48.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:47 smithi016 bash[33851]: audit 2023-10-09T08:33:47.722241+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:49 smithi094 bash[27803]: cluster 2023-10-09T08:33:48.628593+0000 mgr.y (mgr.24355) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:50.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:49 smithi016 bash[33851]: cluster 2023-10-09T08:33:48.628593+0000 mgr.y (mgr.24355) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:50.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:49 smithi016 bash[27454]: cluster 2023-10-09T08:33:48.628593+0000 mgr.y (mgr.24355) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:50.942 INFO:teuthology.orchestra.run.smithi016.stdout:77309411402 2023-10-09T08:33:50.949 INFO:teuthology.orchestra.run.smithi016.stdout:34359738461 2023-10-09T08:33:51.248 INFO:teuthology.orchestra.run.smithi016.stdout:180388626472 2023-10-09T08:33:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:51 smithi094 bash[27803]: cluster 2023-10-09T08:33:50.629024+0000 mgr.y (mgr.24355) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:51 smithi094 bash[27803]: audit 2023-10-09T08:33:50.936604+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.16:0/2193788190' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:33:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:51 smithi094 bash[27803]: audit 2023-10-09T08:33:50.946456+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.16:0/2292679364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:33:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:51 smithi016 bash[27454]: cluster 2023-10-09T08:33:50.629024+0000 mgr.y (mgr.24355) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:51 smithi016 bash[27454]: audit 2023-10-09T08:33:50.936604+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.16:0/2193788190' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:33:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:51 smithi016 bash[27454]: audit 2023-10-09T08:33:50.946456+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.16:0/2292679364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:33:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:51 smithi016 bash[33851]: cluster 2023-10-09T08:33:50.629024+0000 mgr.y (mgr.24355) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:51 smithi016 bash[33851]: audit 2023-10-09T08:33:50.936604+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.16:0/2193788190' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:33:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:51 smithi016 bash[33851]: audit 2023-10-09T08:33:50.946456+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.16:0/2292679364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:33:51.911 INFO:teuthology.orchestra.run.smithi016.stdout:154618822706 2023-10-09T08:33:52.150 INFO:teuthology.orchestra.run.smithi016.stdout:206158430239 2023-10-09T08:33:52.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:52 smithi016 bash[27454]: audit 2023-10-09T08:33:51.245918+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.16:0/1878928834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:33:52.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:52 smithi016 bash[27454]: audit 2023-10-09T08:33:51.908350+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.16:0/928496038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:33:52.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:52 smithi016 bash[33851]: audit 2023-10-09T08:33:51.245918+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.16:0/1878928834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:33:52.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:52 smithi016 bash[33851]: audit 2023-10-09T08:33:51.908350+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.16:0/928496038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:33:52.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:52 smithi094 bash[27803]: audit 2023-10-09T08:33:51.245918+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.16:0/1878928834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:33:52.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:52 smithi094 bash[27803]: audit 2023-10-09T08:33:51.908350+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.16:0/928496038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:33:52.912 INFO:teuthology.orchestra.run.smithi016.stdout:128849018938 2023-10-09T08:33:52.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:33:52 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:33:53.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:53 smithi016 bash[27454]: audit 2023-10-09T08:33:52.147915+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.16:0/1700304482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:33:53.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:53 smithi016 bash[27454]: cluster 2023-10-09T08:33:52.629558+0000 mgr.y (mgr.24355) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:53.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:53 smithi016 bash[27454]: audit 2023-10-09T08:33:52.909349+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.16:0/991799890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:33:53.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:53 smithi016 bash[33851]: audit 2023-10-09T08:33:52.147915+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.16:0/1700304482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:33:53.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:53 smithi016 bash[33851]: cluster 2023-10-09T08:33:52.629558+0000 mgr.y (mgr.24355) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:53.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:53 smithi016 bash[33851]: audit 2023-10-09T08:33:52.909349+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.16:0/991799890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:33:53.312 INFO:teuthology.orchestra.run.smithi016.stdout:107374182467 2023-10-09T08:33:53.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:53 smithi094 bash[27803]: audit 2023-10-09T08:33:52.147915+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.16:0/1700304482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:33:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:53 smithi094 bash[27803]: cluster 2023-10-09T08:33:52.629558+0000 mgr.y (mgr.24355) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:53 smithi094 bash[27803]: audit 2023-10-09T08:33:52.909349+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.16:0/991799890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:33:53.609 INFO:teuthology.orchestra.run.smithi016.stdout:55834574932 2023-10-09T08:33:54.021 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738457 got 34359738461 for osd.0 2023-10-09T08:33:54.021 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:54.143 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411399 got 77309411402 for osd.2 2023-10-09T08:33:54.144 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:54.196 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626468 got 180388626472 for osd.6 2023-10-09T08:33:54.196 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:54.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:54 smithi094 bash[27803]: audit 2023-10-09T08:33:53.309911+0000 mon.a (mon.0) 610 : audit [DBG] from='client.? 172.21.15.16:0/2163743343' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:33:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:54 smithi094 bash[27803]: audit 2023-10-09T08:33:53.606811+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.16:0/1622661500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:33:54.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:33:54 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:33:54] "GET /metrics HTTP/1.1" 200 155634 "" "Prometheus/2.33.4" 2023-10-09T08:33:54.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:54 smithi016 bash[33851]: audit 2023-10-09T08:33:53.309911+0000 mon.a (mon.0) 610 : audit [DBG] from='client.? 172.21.15.16:0/2163743343' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:33:54.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:54 smithi016 bash[33851]: audit 2023-10-09T08:33:53.606811+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.16:0/1622661500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:33:54.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:54 smithi016 bash[27454]: audit 2023-10-09T08:33:53.309911+0000 mon.a (mon.0) 610 : audit [DBG] from='client.? 172.21.15.16:0/2163743343' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:33:54.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:54 smithi016 bash[27454]: audit 2023-10-09T08:33:53.606811+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.16:0/1622661500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:33:55.100 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822702 got 154618822706 for osd.5 2023-10-09T08:33:55.101 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:55.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:55 smithi094 bash[27803]: cluster 2023-10-09T08:33:54.630109+0000 mgr.y (mgr.24355) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:55.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:55 smithi016 bash[33851]: cluster 2023-10-09T08:33:54.630109+0000 mgr.y (mgr.24355) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:55.550 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:55 smithi016 bash[27454]: cluster 2023-10-09T08:33:54.630109+0000 mgr.y (mgr.24355) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:55.769 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430234 got 206158430239 for osd.7 2023-10-09T08:33:55.769 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:56.497 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018932 got 128849018938 for osd.4 2023-10-09T08:33:56.497 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:56.636 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182462 got 107374182467 for osd.3 2023-10-09T08:33:56.636 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:56.640 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574927 got 55834574932 for osd.1 2023-10-09T08:33:56.640 DEBUG:teuthology.parallel:result is None 2023-10-09T08:33:56.640 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T08:33:56.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:33:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:57 smithi094 bash[27803]: cluster 2023-10-09T08:33:56.630605+0000 mgr.y (mgr.24355) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:58.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:57 smithi016 bash[27454]: cluster 2023-10-09T08:33:56.630605+0000 mgr.y (mgr.24355) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:58.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:57 smithi016 bash[33851]: cluster 2023-10-09T08:33:56.630605+0000 mgr.y (mgr.24355) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:59.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:59 smithi094 bash[27803]: audit 2023-10-09T08:33:58.053056+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:59.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:59 smithi094 bash[27803]: audit 2023-10-09T08:33:58.055453+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:33:59.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:59 smithi094 bash[27803]: audit 2023-10-09T08:33:58.056945+0000 mon.c (mon.2) 57 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:33:59.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:59 smithi094 bash[27803]: audit 2023-10-09T08:33:58.065756+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:59.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:59 smithi094 bash[27803]: audit 2023-10-09T08:33:58.073282+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:59.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:33:59 smithi094 bash[27803]: cluster 2023-10-09T08:33:58.631135+0000 mgr.y (mgr.24355) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[27454]: audit 2023-10-09T08:33:58.053056+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[27454]: audit 2023-10-09T08:33:58.055453+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:33:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[27454]: audit 2023-10-09T08:33:58.056945+0000 mon.c (mon.2) 57 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:33:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[27454]: audit 2023-10-09T08:33:58.065756+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[27454]: audit 2023-10-09T08:33:58.073282+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[27454]: cluster 2023-10-09T08:33:58.631135+0000 mgr.y (mgr.24355) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:33:59.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[33851]: audit 2023-10-09T08:33:58.053056+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:59.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[33851]: audit 2023-10-09T08:33:58.055453+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:33:59.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[33851]: audit 2023-10-09T08:33:58.056945+0000 mon.c (mon.2) 57 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:33:59.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[33851]: audit 2023-10-09T08:33:58.065756+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:33:59.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[33851]: audit 2023-10-09T08:33:58.073282+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:33:59.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:33:59 smithi016 bash[33851]: cluster 2023-10-09T08:33:58.631135+0000 mgr.y (mgr.24355) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:01.454 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:34:01.486 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:01 smithi016 bash[27454]: cluster 2023-10-09T08:34:00.631742+0000 mgr.y (mgr.24355) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:01.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:01 smithi016 bash[33851]: cluster 2023-10-09T08:34:00.631742+0000 mgr.y (mgr.24355) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:01.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:01 smithi094 bash[27803]: cluster 2023-10-09T08:34:00.631742+0000 mgr.y (mgr.24355) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:02.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:34:02 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:34:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:03 smithi094 bash[27803]: cluster 2023-10-09T08:34:02.632327+0000 mgr.y (mgr.24355) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:04.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:03 smithi016 bash[27454]: cluster 2023-10-09T08:34:02.632327+0000 mgr.y (mgr.24355) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:04.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:03 smithi016 bash[33851]: cluster 2023-10-09T08:34:02.632327+0000 mgr.y (mgr.24355) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:04.472 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:34:04.474 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:34:04.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:04] "GET /metrics HTTP/1.1" 200 155634 "" "Prometheus/2.33.4" 2023-10-09T08:34:05.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:04 smithi094 bash[27803]: audit 2023-10-09T08:34:03.892878+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:04 smithi094 bash[27803]: audit 2023-10-09T08:34:03.895327+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:04 smithi094 bash[27803]: audit 2023-10-09T08:34:03.896852+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:04 smithi094 bash[27803]: audit 2023-10-09T08:34:03.903849+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:04 smithi094 bash[27803]: audit 2023-10-09T08:34:04.459286+0000 mgr.y (mgr.24355) 87 : audit [DBG] from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:34:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:04 smithi094 bash[27803]: cluster 2023-10-09T08:34:04.632656+0000 mgr.y (mgr.24355) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:05.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[27454]: audit 2023-10-09T08:34:03.892878+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[27454]: audit 2023-10-09T08:34:03.895327+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[27454]: audit 2023-10-09T08:34:03.896852+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[27454]: audit 2023-10-09T08:34:03.903849+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[27454]: audit 2023-10-09T08:34:04.459286+0000 mgr.y (mgr.24355) 87 : audit [DBG] from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[27454]: cluster 2023-10-09T08:34:04.632656+0000 mgr.y (mgr.24355) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[33851]: audit 2023-10-09T08:34:03.892878+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[33851]: audit 2023-10-09T08:34:03.895327+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[33851]: audit 2023-10-09T08:34:03.896852+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:05.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[33851]: audit 2023-10-09T08:34:03.903849+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:05.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[33851]: audit 2023-10-09T08:34:04.459286+0000 mgr.y (mgr.24355) 87 : audit [DBG] from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:34:05.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:04 smithi016 bash[33851]: cluster 2023-10-09T08:34:04.632656+0000 mgr.y (mgr.24355) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:05.317 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":66,"stamp":"2023-10-09T08:34:02.632033+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916132+0000","last_change":"2023-10-09T08:32:37.686368+0000","last_active":"2023-10-09T08:32:39.916132+0000","last_peered":"2023-10-09T08:32:39.916132+0000","last_clean":"2023-10-09T08:32:39.916132+0000","last_became_active":"2023-10-09T08:32:37.685786+0000","last_became_peered":"2023-10-09T08:32:37.685786+0000","last_unstale":"2023-10-09T08:32:39.916132+0000","last_undegraded":"2023-10-09T08:32:39.916132+0000","last_fullsized":"2023-10-09T08:32:39.916132+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.688756+0000","last_change":"2023-10-09T08:32:37.776388+0000","last_active":"2023-10-09T08:32:38.688756+0000","last_peered":"2023-10-09T08:32:38.688756+0000","last_clean":"2023-10-09T08:32:38.688756+0000","last_became_active":"2023-10-09T08:32:37.775915+0000","last_became_peered":"2023-10-09T08:32:37.775915+0000","last_unstale":"2023-10-09T08:32:38.688756+0000","last_undegraded":"2023-10-09T08:32:38.688756+0000","last_fullsized":"2023-10-09T08:32:38.688756+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916292+0000","last_change":"2023-10-09T08:32:37.777840+0000","last_active":"2023-10-09T08:32:39.916292+0000","last_peered":"2023-10-09T08:32:39.916292+0000","last_clean":"2023-10-09T08:32:39.916292+0000","last_became_active":"2023-10-09T08:32:37.777235+0000","last_became_peered":"2023-10-09T08:32:37.777235+0000","last_unstale":"2023-10-09T08:32:39.916292+0000","last_undegraded":"2023-10-09T08:32:39.916292+0000","last_fullsized":"2023-10-09T08:32:39.916292+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916116+0000","last_change":"2023-10-09T08:32:37.689006+0000","last_active":"2023-10-09T08:32:39.916116+0000","last_peered":"2023-10-09T08:32:39.916116+0000","last_clean":"2023-10-09T08:32:39.916116+0000","last_became_active":"2023-10-09T08:32:37.688629+0000","last_became_peered":"2023-10-09T08:32:37.688629+0000","last_unstale":"2023-10-09T08:32:39.916116+0000","last_undegraded":"2023-10-09T08:32:39.916116+0000","last_fullsized":"2023-10-09T08:32:39.916116+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916194+0000","last_change":"2023-10-09T08:31:31.839687+0000","last_active":"2023-10-09T08:32:39.916194+0000","last_peered":"2023-10-09T08:32:39.916194+0000","last_clean":"2023-10-09T08:32:39.916194+0000","last_became_active":"2023-10-09T08:31:31.838733+0000","last_became_peered":"2023-10-09T08:31:31.838733+0000","last_unstale":"2023-10-09T08:32:39.916194+0000","last_undegraded":"2023-10-09T08:32:39.916194+0000","last_fullsized":"2023-10-09T08:32:39.916194+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.689887+0000","last_change":"2023-10-09T08:32:37.686289+0000","last_active":"2023-10-09T08:32:38.689887+0000","last_peered":"2023-10-09T08:32:38.689887+0000","last_clean":"2023-10-09T08:32:38.689887+0000","last_became_active":"2023-10-09T08:32:37.685665+0000","last_became_peered":"2023-10-09T08:32:37.685665+0000","last_unstale":"2023-10-09T08:32:38.689887+0000","last_undegraded":"2023-10-09T08:32:38.689887+0000","last_fullsized":"2023-10-09T08:32:38.689887+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.913685+0000","last_change":"2023-10-09T08:32:37.688803+0000","last_active":"2023-10-09T08:32:39.913685+0000","last_peered":"2023-10-09T08:32:39.913685+0000","last_clean":"2023-10-09T08:32:39.913685+0000","last_became_active":"2023-10-09T08:32:37.688433+0000","last_became_peered":"2023-10-09T08:32:37.688433+0000","last_unstale":"2023-10-09T08:32:39.913685+0000","last_undegraded":"2023-10-09T08:32:39.913685+0000","last_fullsized":"2023-10-09T08:32:39.913685+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.689831+0000","last_change":"2023-10-09T08:32:37.682306+0000","last_active":"2023-10-09T08:32:38.689831+0000","last_peered":"2023-10-09T08:32:38.689831+0000","last_clean":"2023-10-09T08:32:38.689831+0000","last_became_active":"2023-10-09T08:32:37.681879+0000","last_became_peered":"2023-10-09T08:32:37.681879+0000","last_unstale":"2023-10-09T08:32:38.689831+0000","last_undegraded":"2023-10-09T08:32:38.689831+0000","last_fullsized":"2023-10-09T08:32:38.689831+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.688776+0000","last_change":"2023-10-09T08:32:37.688617+0000","last_active":"2023-10-09T08:32:38.688776+0000","last_peered":"2023-10-09T08:32:38.688776+0000","last_clean":"2023-10-09T08:32:38.688776+0000","last_became_active":"2023-10-09T08:32:37.688218+0000","last_became_peered":"2023-10-09T08:32:37.688218+0000","last_unstale":"2023-10-09T08:32:38.688776+0000","last_undegraded":"2023-10-09T08:32:38.688776+0000","last_fullsized":"2023-10-09T08:32:38.688776+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.85599999999999998,"15min":0.84699999999999998},"min":{"1min":0.628,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":0.88,"5min":0.92800000000000005,"15min":0.93600000000000005},"min":{"1min":0.70499999999999996,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":1.087}]},{"osd":1,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.92600000000000005,"15min":0.93000000000000005},"min":{"1min":0.66000000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.323,"5min":1.786,"15min":1.786},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89100000000000001,"15min":0.89100000000000001},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1299999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.98899999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.92600000000000005,"15min":0.93700000000000006},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.099,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.94899999999999995,"15min":0.96299999999999997},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.161}]},{"osd":3,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.89800000000000002,"15min":0.89600000000000002},"min":{"1min":0.71499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.214,"5min":1.276,"15min":1.276},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.66100000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.2110000000000001,"5min":1.367,"15min":1.367},"last":0.81699999999999995}]},{"osd":4,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.84199999999999997,"15min":0.84799999999999998},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.1719999999999999,"5min":1.228,"15min":1.228},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.878,"15min":0.877},"min":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.94899999999999995}]},{"osd":5,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92500000000000004,"15min":0.92500000000000004},"min":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.6399999999999999,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.91200000000000003,"15min":0.91400000000000003},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.248,"5min":1.446,"15min":1.446},"last":1.1799999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.97099999999999997,"15min":0.97199999999999998},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.3620000000000001,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.2370000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.97099999999999997,"15min":0.96599999999999997},"min":{"1min":0.72599999999999998,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.337,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":1.0629999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626474,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.88400000000000001,"15min":0.91200000000000003},"min":{"1min":0.63600000000000001,"5min":0.621,"15min":0.621},"max":{"1min":1.1919999999999999,"5min":1.385,"15min":1.385},"last":1.04},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.89400000000000002,"15min":0.90700000000000003},"min":{"1min":0.67000000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.131,"5min":1.327,"15min":1.327},"last":0.71299999999999997}]},{"osd":1,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.88,"15min":0.90400000000000003},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.96999999999999997,"5min":1.496,"15min":1.496},"last":1.0669999999999999},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.89300000000000002,"15min":0.91000000000000003},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.169,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.92600000000000005}]},{"osd":2,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.90100000000000002,"15min":0.94099999999999995},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.133,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.96799999999999997,"15min":0.99199999999999999},"min":{"1min":0.68600000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.1359999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.0089999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.96699999999999997,"15min":0.998},"min":{"1min":0.67400000000000004,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.2090000000000001,"5min":1.774,"15min":1.774},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.98699999999999999,"15min":1.036},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.03,"5min":1.476,"15min":1.476},"last":0.97999999999999998}]},{"osd":4,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":0.83399999999999996},"min":{"1min":0.495,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.2929999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.879,"15min":0.90300000000000002},"min":{"1min":0.52900000000000003,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.2709999999999999,"5min":1.619,"15min":1.619},"last":0.85299999999999998}]},{"osd":5,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.93999999999999995,"15min":0.95799999999999996},"min":{"1min":0.505,"5min":0.504,"15min":0.504},"max":{"1min":1.2330000000000001,"5min":1.544,"15min":1.544},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.93899999999999995,"15min":0.96799999999999997},"min":{"1min":0.57999999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3260000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.94999999999999996}]},{"osd":7,"last update":"Mon Oct 9 08:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.92100000000000004,"15min":0.91200000000000003},"min":{"1min":0.58799999999999997,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.93000000000000005,"15min":0.92800000000000005},"min":{"1min":0.626,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.167,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.998}]}]},{"osd":1,"up_from":13,"seq":55834574934,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:04 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.72699999999999998,"15min":0.59099999999999997},"min":{"1min":0.53000000000000003,"5min":0.47799999999999998,"15min":0.29999999999999999},"max":{"1min":1.1140000000000001,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.751},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.77200000000000002,"15min":0.624},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.80200000000000005}]},{"osd":2,"last update":"Mon Oct 9 08:33:54 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.79200000000000004,"15min":0.70299999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.127,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.78000000000000003,"15min":0.71299999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.98699999999999999,"5min":1.165,"15min":1.165},"last":1.1279999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.84099999999999997,"15min":0.81999999999999995},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84399999999999997,"15min":0.80500000000000005},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.6739999999999999,"5min":1.6739999999999999,"15min":1.6739999999999999},"last":0.91400000000000003}]},{"osd":4,"last update":"Mon Oct 9 08:33:14 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.998,"15min":0.96499999999999997},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":1.056},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0069999999999999,"15min":0.96899999999999997},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":1.153}]},{"osd":5,"last update":"Mon Oct 9 08:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0660000000000001,"15min":1.089},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":1.079},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.0209999999999999,"15min":1.032},"min":{"1min":0.69499999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.4570000000000001,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":0.97399999999999998}]},{"osd":6,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.149,"15min":1.173},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.3740000000000001,"5min":1.639,"15min":1.639},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1639999999999999,"15min":1.1899999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":1.288}]},{"osd":7,"last update":"Mon Oct 9 08:33:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.149,"15min":1.1619999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.4730000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.1910000000000001,"15min":1.1970000000000001},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.456,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.1699999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738464,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:33:04 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.70899999999999996,"15min":0.58699999999999997},"min":{"1min":0.52900000000000003,"5min":0.38500000000000001,"15min":0.33300000000000002},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.78200000000000003,"15min":0.64000000000000001},"min":{"1min":0.55000000000000004,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.149,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.78000000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.78200000000000003,"15min":0.66900000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.80600000000000005,"15min":0.66900000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.1910000000000001,"5min":1.335,"15min":1.335},"last":0.89200000000000002}]},{"osd":3,"last update":"Mon Oct 9 08:33:27 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.81200000000000006,"15min":0.77300000000000002},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.90100000000000002,"15min":0.85299999999999998},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2989999999999999,"5min":1.544,"15min":1.544},"last":1.081}]},{"osd":4,"last update":"Mon Oct 9 08:33:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.028,"15min":1.0229999999999999},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.2589999999999999,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":1.266},{"interface":"front","average":{"1min":1.024,"5min":1.0089999999999999,"15min":0.98699999999999999},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.5720000000000001,"5min":1.75,"15min":1.75},"last":1.4930000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0469999999999999,"15min":1.0309999999999999},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.3340000000000001,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.089,"5min":1.075,"15min":1.0609999999999999},"min":{"1min":0.85999999999999999,"5min":0.83399999999999996,"15min":0.83399999999999996},"max":{"1min":1.4299999999999999,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.3100000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.159,"15min":1.1859999999999999},"min":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.4099999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":1.411},{"interface":"front","average":{"1min":1.024,"5min":1.1419999999999999,"15min":1.1679999999999999},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.4319999999999999,"5min":1.649,"15min":1.649},"last":1.387}]},{"osd":7,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.1890000000000001,"15min":1.2110000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":1.4390000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.1779999999999999,"15min":1.196},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.3959999999999999,"5min":1.7230000000000001,"15min":1.7230000000000001},"last":1.288}]}]},{"osd":2,"up_from":18,"seq":77309411405,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:50 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.71799999999999997,"15min":0.64900000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.125,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.747,"15min":0.67100000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.034,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.75800000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:33:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.75,"15min":0.67600000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.99299999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.88},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.84199999999999997,"15min":0.71599999999999997},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.46899999999999997},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.99099999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.83999999999999997,"15min":0.82599999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1679999999999999,"5min":1.349,"15min":1.349},"last":1.0569999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.879,"15min":0.85899999999999999},"min":{"1min":0.53000000000000003,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.173,"5min":1.173,"15min":1.173},"last":0.83699999999999997}]},{"osd":4,"last update":"Mon Oct 9 08:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.97799999999999998,"15min":0.97599999999999998},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":1.286},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.028,"15min":1.01},"min":{"1min":0.84199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.92300000000000004}]},{"osd":5,"last update":"Mon Oct 9 08:33:58 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.996,"15min":1.0169999999999999},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.2110000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0740000000000001,"15min":1.0580000000000001},"min":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.204}]},{"osd":6,"last update":"Mon Oct 9 08:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.1279999999999999,"15min":1.1339999999999999},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.335,"5min":1.714,"15min":1.714},"last":1.331},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1519999999999999,"15min":1.147},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.6000000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":1.0920000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:33:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.143,"15min":1.1579999999999999},"min":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.4219999999999999,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.177},{"interface":"front","average":{"1min":1.129,"5min":1.2290000000000001,"15min":1.246},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":1.3740000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182469,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.69299999999999995,"15min":0.67100000000000004},"min":{"1min":0.45400000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.96999999999999997,"5min":1.2,"15min":1.2},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.81100000000000005,"15min":0.78300000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.3620000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.71699999999999997}]},{"osd":1,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.83199999999999996,"15min":0.81999999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.2689999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.76800000000000002,"15min":0.754},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.82699999999999996,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":1.026}]},{"osd":2,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.86299999999999999,"15min":0.85099999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.052,"5min":1.8400000000000001,"15min":1.8400000000000001},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80500000000000005,"15min":0.77800000000000002},"min":{"1min":0.48599999999999999,"5min":0.44,"15min":0.44},"max":{"1min":1.514,"5min":1.514,"15min":1.514},"last":0.95899999999999996}]},{"osd":4,"last update":"Mon Oct 9 08:33:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.96299999999999997,"15min":0.92700000000000005},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.2529999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":1.105},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.99299999999999999,"15min":0.97499999999999998},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.51,"5min":1.542,"15min":1.542},"last":1.145}]},{"osd":5,"last update":"Mon Oct 9 08:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.0129999999999999,"15min":1.0009999999999999},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.472,"5min":1.472,"15min":1.472},"last":1.1679999999999999},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.0309999999999999,"15min":1.0169999999999999},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.4279999999999999,"5min":1.663,"15min":1.663},"last":1.0700000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.105,"15min":1.129},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3660000000000001,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":1.329},{"interface":"front","average":{"1min":1.107,"5min":1.1359999999999999,"15min":1.155},"min":{"1min":0.79300000000000004,"5min":0.78900000000000003,"15min":0.78900000000000003},"max":{"1min":1.905,"5min":1.905,"15min":1.905},"last":1.264}]},{"osd":7,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.165,"15min":1.173},"min":{"1min":0.86399999999999999,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.46,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":1.222},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.093,"15min":1.105},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.4299999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":1.2929999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018940,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.80400000000000005,"15min":0.78400000000000003},"min":{"1min":0.59999999999999998,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.113,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.24},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.88200000000000001,"15min":0.88800000000000001},"min":{"1min":0.68000000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.093,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.81000000000000005}]},{"osd":1,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.81399999999999995,"15min":0.79800000000000004},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.113,"5min":1.244,"15min":1.244},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.873,"15min":0.873},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.0700000000000001,"5min":1.2,"15min":1.2},"last":0.96999999999999997}]},{"osd":2,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.84999999999999998,"15min":0.84299999999999997},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.147,"5min":1.3,"15min":1.3},"last":1.163},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.90500000000000003,"15min":0.89300000000000002},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.91300000000000003}]},{"osd":3,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.89900000000000002,"15min":0.88500000000000001},"min":{"1min":0.63700000000000001,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1240000000000001,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.1259999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.90200000000000002,"15min":0.90000000000000002},"min":{"1min":0.64700000000000002,"5min":0.623,"15min":0.623},"max":{"1min":1.095,"5min":1.302,"15min":1.302},"last":1.0289999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.78400000000000003,"15min":0.72399999999999998},"min":{"1min":0.58399999999999996,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.087},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.84199999999999997,"15min":0.78100000000000003},"min":{"1min":0.50600000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.355,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.2130000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:33:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.83199999999999996,"15min":0.82999999999999996},"min":{"1min":0.49099999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2909999999999999,"5min":1.645,"15min":1.645},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.89000000000000001,"15min":0.875},"min":{"1min":0.58299999999999996,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.298}]},{"osd":7,"last update":"Mon Oct 9 08:33:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.96499999999999997,"15min":0.96099999999999997},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.2769999999999999,"5min":1.278,"15min":1.278},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.004,"5min":0.96699999999999997,"15min":0.96099999999999997},"min":{"1min":0.56000000000000005,"5min":0.5,"15min":0.5},"max":{"1min":1.3839999999999999,"5min":1.724,"15min":1.724},"last":1.103}]}]},{"osd":5,"up_from":36,"seq":154618822708,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:32:57 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.81599999999999995,"15min":0.80900000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2290000000000001,"5min":1.238,"15min":1.238},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.876,"15min":0.88700000000000001},"min":{"1min":0.63900000000000001,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.0429999999999999,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":1.0209999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:32:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.86699999999999999,"15min":0.86899999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.0940000000000001,"5min":1.2,"15min":1.2},"last":1.2949999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.84299999999999997,"15min":0.83199999999999996},"min":{"1min":0.65100000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.0089999999999999,"5min":1.198,"15min":1.198},"last":0.91100000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:32:57 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.86699999999999999,"15min":0.86499999999999999},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":1.173},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.90200000000000002,"15min":0.89300000000000002},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.873}]},{"osd":3,"last update":"Mon Oct 9 08:32:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.91200000000000003,"15min":0.92700000000000005},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.095,"5min":1.276,"15min":1.276},"last":1.0549999999999999},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.93100000000000005,"15min":0.91800000000000004},"min":{"1min":0.60499999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.282,"5min":1.323,"15min":1.323},"last":1.1399999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:32:57 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.73999999999999999,"15min":0.70999999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.77900000000000003,"15min":0.751},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":0.84299999999999997}]},{"osd":6,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.88300000000000001,"15min":0.86699999999999999},"min":{"1min":0.59399999999999997,"5min":0.439,"15min":0.439},"max":{"1min":1.2689999999999999,"5min":1.349,"15min":1.349},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.87,"5min":0.83699999999999997,"15min":0.82099999999999995},"min":{"1min":0.55700000000000005,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.155,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.2490000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.88100000000000001,"15min":0.86099999999999999},"min":{"1min":0.64300000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.095,"5min":0.92400000000000004,"15min":0.89600000000000002},"min":{"1min":0.69599999999999995,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":1.2210000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:34:05.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:34:07.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:07 smithi094 bash[27803]: cluster 2023-10-09T08:34:06.633288+0000 mgr.y (mgr.24355) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:08.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:07 smithi016 bash[27454]: cluster 2023-10-09T08:34:06.633288+0000 mgr.y (mgr.24355) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:08.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:07 smithi016 bash[33851]: cluster 2023-10-09T08:34:06.633288+0000 mgr.y (mgr.24355) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:09.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:09 smithi094 bash[27803]: cluster 2023-10-09T08:34:08.633804+0000 mgr.y (mgr.24355) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:10.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:09 smithi016 bash[27454]: cluster 2023-10-09T08:34:08.633804+0000 mgr.y (mgr.24355) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:10.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:09 smithi016 bash[33851]: cluster 2023-10-09T08:34:08.633804+0000 mgr.y (mgr.24355) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:10.092 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:34:11.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:11 smithi016 bash[27454]: cluster 2023-10-09T08:34:10.634461+0000 mgr.y (mgr.24355) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:11.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:11 smithi016 bash[33851]: cluster 2023-10-09T08:34:10.634461+0000 mgr.y (mgr.24355) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:11.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:11 smithi094 bash[27803]: cluster 2023-10-09T08:34:10.634461+0000 mgr.y (mgr.24355) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:12.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:34:12 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:34:13.291 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:34:13.295 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:34:13.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:13 smithi094 bash[27803]: cluster 2023-10-09T08:34:12.635035+0000 mgr.y (mgr.24355) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:14.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:13 smithi016 bash[27454]: cluster 2023-10-09T08:34:12.635035+0000 mgr.y (mgr.24355) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:14.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:13 smithi016 bash[33851]: cluster 2023-10-09T08:34:12.635035+0000 mgr.y (mgr.24355) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:14.096 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":71,"stamp":"2023-10-09T08:34:12.634757+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916132+0000","last_change":"2023-10-09T08:32:37.686368+0000","last_active":"2023-10-09T08:32:39.916132+0000","last_peered":"2023-10-09T08:32:39.916132+0000","last_clean":"2023-10-09T08:32:39.916132+0000","last_became_active":"2023-10-09T08:32:37.685786+0000","last_became_peered":"2023-10-09T08:32:37.685786+0000","last_unstale":"2023-10-09T08:32:39.916132+0000","last_undegraded":"2023-10-09T08:32:39.916132+0000","last_fullsized":"2023-10-09T08:32:39.916132+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.688756+0000","last_change":"2023-10-09T08:32:37.776388+0000","last_active":"2023-10-09T08:32:38.688756+0000","last_peered":"2023-10-09T08:32:38.688756+0000","last_clean":"2023-10-09T08:32:38.688756+0000","last_became_active":"2023-10-09T08:32:37.775915+0000","last_became_peered":"2023-10-09T08:32:37.775915+0000","last_unstale":"2023-10-09T08:32:38.688756+0000","last_undegraded":"2023-10-09T08:32:38.688756+0000","last_fullsized":"2023-10-09T08:32:38.688756+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916292+0000","last_change":"2023-10-09T08:32:37.777840+0000","last_active":"2023-10-09T08:32:39.916292+0000","last_peered":"2023-10-09T08:32:39.916292+0000","last_clean":"2023-10-09T08:32:39.916292+0000","last_became_active":"2023-10-09T08:32:37.777235+0000","last_became_peered":"2023-10-09T08:32:37.777235+0000","last_unstale":"2023-10-09T08:32:39.916292+0000","last_undegraded":"2023-10-09T08:32:39.916292+0000","last_fullsized":"2023-10-09T08:32:39.916292+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916116+0000","last_change":"2023-10-09T08:32:37.689006+0000","last_active":"2023-10-09T08:32:39.916116+0000","last_peered":"2023-10-09T08:32:39.916116+0000","last_clean":"2023-10-09T08:32:39.916116+0000","last_became_active":"2023-10-09T08:32:37.688629+0000","last_became_peered":"2023-10-09T08:32:37.688629+0000","last_unstale":"2023-10-09T08:32:39.916116+0000","last_undegraded":"2023-10-09T08:32:39.916116+0000","last_fullsized":"2023-10-09T08:32:39.916116+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.916194+0000","last_change":"2023-10-09T08:31:31.839687+0000","last_active":"2023-10-09T08:32:39.916194+0000","last_peered":"2023-10-09T08:32:39.916194+0000","last_clean":"2023-10-09T08:32:39.916194+0000","last_became_active":"2023-10-09T08:31:31.838733+0000","last_became_peered":"2023-10-09T08:31:31.838733+0000","last_unstale":"2023-10-09T08:32:39.916194+0000","last_undegraded":"2023-10-09T08:32:39.916194+0000","last_fullsized":"2023-10-09T08:32:39.916194+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.689887+0000","last_change":"2023-10-09T08:32:37.686289+0000","last_active":"2023-10-09T08:32:38.689887+0000","last_peered":"2023-10-09T08:32:38.689887+0000","last_clean":"2023-10-09T08:32:38.689887+0000","last_became_active":"2023-10-09T08:32:37.685665+0000","last_became_peered":"2023-10-09T08:32:37.685665+0000","last_unstale":"2023-10-09T08:32:38.689887+0000","last_undegraded":"2023-10-09T08:32:38.689887+0000","last_fullsized":"2023-10-09T08:32:38.689887+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:39.913685+0000","last_change":"2023-10-09T08:32:37.688803+0000","last_active":"2023-10-09T08:32:39.913685+0000","last_peered":"2023-10-09T08:32:39.913685+0000","last_clean":"2023-10-09T08:32:39.913685+0000","last_became_active":"2023-10-09T08:32:37.688433+0000","last_became_peered":"2023-10-09T08:32:37.688433+0000","last_unstale":"2023-10-09T08:32:39.913685+0000","last_undegraded":"2023-10-09T08:32:39.913685+0000","last_fullsized":"2023-10-09T08:32:39.913685+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.689831+0000","last_change":"2023-10-09T08:32:37.682306+0000","last_active":"2023-10-09T08:32:38.689831+0000","last_peered":"2023-10-09T08:32:38.689831+0000","last_clean":"2023-10-09T08:32:38.689831+0000","last_became_active":"2023-10-09T08:32:37.681879+0000","last_became_peered":"2023-10-09T08:32:37.681879+0000","last_unstale":"2023-10-09T08:32:38.689831+0000","last_undegraded":"2023-10-09T08:32:38.689831+0000","last_fullsized":"2023-10-09T08:32:38.689831+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T08:32:38.688776+0000","last_change":"2023-10-09T08:32:37.688617+0000","last_active":"2023-10-09T08:32:38.688776+0000","last_peered":"2023-10-09T08:32:38.688776+0000","last_clean":"2023-10-09T08:32:38.688776+0000","last_became_active":"2023-10-09T08:32:37.688218+0000","last_became_peered":"2023-10-09T08:32:37.688218+0000","last_unstale":"2023-10-09T08:32:38.688776+0000","last_undegraded":"2023-10-09T08:32:38.688776+0000","last_fullsized":"2023-10-09T08:32:38.688776+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430243,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.85599999999999998,"15min":0.84699999999999998},"min":{"1min":0.628,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.88,"5min":0.92800000000000005,"15min":0.93600000000000005},"min":{"1min":0.70499999999999996,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":0.72699999999999998}]},{"osd":1,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.92600000000000005,"15min":0.93000000000000005},"min":{"1min":0.66000000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.323,"5min":1.786,"15min":1.786},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89100000000000001,"15min":0.89100000000000001},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1299999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.86099999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.92600000000000005,"15min":0.93700000000000006},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.099,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.94899999999999995,"15min":0.96299999999999997},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.90900000000000003}]},{"osd":3,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.89800000000000002,"15min":0.89600000000000002},"min":{"1min":0.71499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.214,"5min":1.276,"15min":1.276},"last":1.139},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.66100000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.2110000000000001,"5min":1.367,"15min":1.367},"last":1.2350000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.84199999999999997,"15min":0.84799999999999998},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.1719999999999999,"5min":1.228,"15min":1.228},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.878,"15min":0.877},"min":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":1.0609999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92500000000000004,"15min":0.92500000000000004},"min":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.6399999999999999,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.123},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.91200000000000003,"15min":0.91400000000000003},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.248,"5min":1.446,"15min":1.446},"last":1.2909999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.97099999999999997,"15min":0.97199999999999998},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.3620000000000001,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.97099999999999997,"15min":0.96599999999999997},"min":{"1min":0.72599999999999998,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.337,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":1.2070000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626476,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.88400000000000001,"15min":0.91200000000000003},"min":{"1min":0.63600000000000001,"5min":0.621,"15min":0.621},"max":{"1min":1.1919999999999999,"5min":1.385,"15min":1.385},"last":1.081},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.89400000000000002,"15min":0.90700000000000003},"min":{"1min":0.67000000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.131,"5min":1.327,"15min":1.327},"last":0.85999999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.88,"15min":0.90400000000000003},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.96999999999999997,"5min":1.496,"15min":1.496},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.89300000000000002,"15min":0.91000000000000003},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.169,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.70299999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.90100000000000002,"15min":0.94099999999999995},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.133,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.96799999999999997,"15min":0.99199999999999999},"min":{"1min":0.68600000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.1359999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":0.96799999999999997}]},{"osd":3,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.96699999999999997,"15min":0.998},"min":{"1min":0.67400000000000004,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.2090000000000001,"5min":1.774,"15min":1.774},"last":1.02},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.98699999999999999,"15min":1.036},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.03,"5min":1.476,"15min":1.476},"last":0.90800000000000003}]},{"osd":4,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":0.83399999999999996},"min":{"1min":0.495,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.2929999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.879,"15min":0.90300000000000002},"min":{"1min":0.52900000000000003,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.2709999999999999,"5min":1.619,"15min":1.619},"last":0.93700000000000006}]},{"osd":5,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.93999999999999995,"15min":0.95799999999999996},"min":{"1min":0.505,"5min":0.504,"15min":0.504},"max":{"1min":1.2330000000000001,"5min":1.544,"15min":1.544},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.93899999999999995,"15min":0.96799999999999997},"min":{"1min":0.57999999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3260000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.1020000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.92100000000000004,"15min":0.91200000000000003},"min":{"1min":0.58799999999999997,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.93000000000000005,"15min":0.92800000000000005},"min":{"1min":0.626,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.167,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":1.149}]}]},{"osd":1,"up_from":13,"seq":55834574936,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.72099999999999997,"15min":0.59499999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.96699999999999997,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.78100000000000003,"15min":0.63500000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.69199999999999995}]},{"osd":2,"last update":"Mon Oct 9 08:33:54 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.79200000000000004,"15min":0.70299999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.127,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.78000000000000003,"15min":0.71299999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.98699999999999999,"5min":1.165,"15min":1.165},"last":0.76700000000000002}]},{"osd":3,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.84099999999999997,"15min":0.81999999999999995},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84399999999999997,"15min":0.80500000000000005},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.6739999999999999,"5min":1.6739999999999999,"15min":1.6739999999999999},"last":1.0369999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:33:14 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.998,"15min":0.96499999999999997},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":1.204},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0069999999999999,"15min":0.96899999999999997},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.90200000000000002}]},{"osd":5,"last update":"Mon Oct 9 08:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0660000000000001,"15min":1.089},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.0209999999999999,"15min":1.032},"min":{"1min":0.69499999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.4570000000000001,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":1.1200000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.149,"15min":1.173},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.3740000000000001,"5min":1.639,"15min":1.639},"last":1.371},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1639999999999999,"15min":1.1899999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":1.155}]},{"osd":7,"last update":"Mon Oct 9 08:33:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.149,"15min":1.1619999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.4730000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.155,"5min":1.1910000000000001,"15min":1.1970000000000001},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.456,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.296}]}]},{"osd":0,"up_from":8,"seq":34359738466,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.72899999999999998,"15min":0.59799999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.373,"5min":1.373,"15min":1.373},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.79500000000000004,"15min":0.64700000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.39000000000000001},"max":{"1min":0.98599999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.93799999999999994}]},{"osd":2,"last update":"Mon Oct 9 08:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.78200000000000003,"15min":0.66900000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.80600000000000005,"15min":0.66900000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.1910000000000001,"5min":1.335,"15min":1.335},"last":0.85999999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:33:27 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.81200000000000006,"15min":0.77300000000000002},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.90100000000000002,"15min":0.85299999999999998},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2989999999999999,"5min":1.544,"15min":1.544},"last":1.149}]},{"osd":4,"last update":"Mon Oct 9 08:33:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.028,"15min":1.0229999999999999},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.2589999999999999,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.024,"5min":1.0089999999999999,"15min":0.98699999999999999},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.5720000000000001,"5min":1.75,"15min":1.75},"last":1.1200000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.028,"15min":1.0249999999999999},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.26,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.3580000000000001},{"interface":"front","average":{"1min":1.006,"5min":1.0649999999999999,"15min":1.0569999999999999},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.492,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.90200000000000002}]},{"osd":6,"last update":"Mon Oct 9 08:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.159,"15min":1.1859999999999999},"min":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.4099999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":1.2230000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.1419999999999999,"15min":1.1679999999999999},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.4319999999999999,"5min":1.649,"15min":1.649},"last":1.272}]},{"osd":7,"last update":"Mon Oct 9 08:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.1890000000000001,"15min":1.2110000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.1779999999999999,"15min":1.196},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.3959999999999999,"5min":1.7230000000000001,"15min":1.7230000000000001},"last":1.2949999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411407,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:50 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.71799999999999997,"15min":0.64900000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.125,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.747,"15min":0.67100000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.034,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.67600000000000005}]},{"osd":1,"last update":"Mon Oct 9 08:33:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.75,"15min":0.67600000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.99299999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.84199999999999997,"15min":0.71599999999999997},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.46899999999999997},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.72699999999999998}]},{"osd":3,"last update":"Mon Oct 9 08:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.83999999999999997,"15min":0.82599999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1679999999999999,"5min":1.349,"15min":1.349},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.879,"15min":0.85899999999999999},"min":{"1min":0.53000000000000003,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.173,"5min":1.173,"15min":1.173},"last":1.1100000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.97799999999999998,"15min":0.97599999999999998},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.028,"15min":1.01},"min":{"1min":0.84199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.2190000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:33:58 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.996,"15min":1.0169999999999999},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.2110000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.3600000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0740000000000001,"15min":1.0580000000000001},"min":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.194}]},{"osd":6,"last update":"Mon Oct 9 08:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.1279999999999999,"15min":1.1339999999999999},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.335,"5min":1.714,"15min":1.714},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1519999999999999,"15min":1.147},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.6000000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":1.306}]},{"osd":7,"last update":"Mon Oct 9 08:33:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.143,"15min":1.1579999999999999},"min":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.4219999999999999,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.4410000000000001},{"interface":"front","average":{"1min":1.129,"5min":1.2290000000000001,"15min":1.246},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":1.4079999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182471,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.69299999999999995,"15min":0.67100000000000004},"min":{"1min":0.45400000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.96999999999999997,"5min":1.2,"15min":1.2},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.81100000000000005,"15min":0.78300000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.3620000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.68000000000000005}]},{"osd":1,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.83199999999999996,"15min":0.81999999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.2689999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.76800000000000002,"15min":0.754},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.82699999999999996,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.95299999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.86299999999999999,"15min":0.85099999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.052,"5min":1.8400000000000001,"15min":1.8400000000000001},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80500000000000005,"15min":0.77800000000000002},"min":{"1min":0.48599999999999999,"5min":0.44,"15min":0.44},"max":{"1min":1.514,"5min":1.514,"15min":1.514},"last":0.82199999999999995}]},{"osd":4,"last update":"Mon Oct 9 08:33:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.96299999999999997,"15min":0.92700000000000005},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.2529999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.99299999999999999,"15min":0.97499999999999998},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.51,"5min":1.542,"15min":1.542},"last":0.98099999999999998}]},{"osd":5,"last update":"Mon Oct 9 08:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0109999999999999,"15min":1},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.022,"15min":1.014},"min":{"1min":0.69999999999999996,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.3220000000000001,"5min":1.663,"15min":1.663},"last":1.157}]},{"osd":6,"last update":"Mon Oct 9 08:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.105,"15min":1.129},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3660000000000001,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":1.048},{"interface":"front","average":{"1min":1.107,"5min":1.1359999999999999,"15min":1.155},"min":{"1min":0.79300000000000004,"5min":0.78900000000000003,"15min":0.78900000000000003},"max":{"1min":1.905,"5min":1.905,"15min":1.905},"last":1.1830000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.165,"15min":1.173},"min":{"1min":0.86399999999999999,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.46,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":1.2210000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.093,"15min":1.105},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.4299999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":1.2430000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018942,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.80400000000000005,"15min":0.78400000000000003},"min":{"1min":0.59999999999999998,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.113,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.88200000000000001,"15min":0.88800000000000001},"min":{"1min":0.68000000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.093,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.1759999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.81399999999999995,"15min":0.79800000000000004},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.113,"5min":1.244,"15min":1.244},"last":1.03},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.873,"15min":0.873},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.0700000000000001,"5min":1.2,"15min":1.2},"last":0.76800000000000002}]},{"osd":2,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.84999999999999998,"15min":0.84299999999999997},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.147,"5min":1.3,"15min":1.3},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.90500000000000003,"15min":0.89300000000000002},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.98499999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.89900000000000002,"15min":0.88500000000000001},"min":{"1min":0.63700000000000001,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1240000000000001,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.90200000000000002,"15min":0.90000000000000002},"min":{"1min":0.64700000000000002,"5min":0.623,"15min":0.623},"max":{"1min":1.095,"5min":1.302,"15min":1.302},"last":1.22}]},{"osd":5,"last update":"Mon Oct 9 08:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.78400000000000003,"15min":0.72399999999999998},"min":{"1min":0.58399999999999996,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.84199999999999997,"15min":0.78100000000000003},"min":{"1min":0.50600000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.355,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.2450000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:33:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.83199999999999996,"15min":0.82999999999999996},"min":{"1min":0.49099999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2909999999999999,"5min":1.645,"15min":1.645},"last":1.123},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.89000000000000001,"15min":0.875},"min":{"1min":0.58299999999999996,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.2689999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:33:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.96499999999999997,"15min":0.96099999999999997},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.2769999999999999,"5min":1.278,"15min":1.278},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.004,"5min":0.96699999999999997,"15min":0.96099999999999997},"min":{"1min":0.56000000000000005,"5min":0.5,"15min":0.5},"max":{"1min":1.3839999999999999,"5min":1.724,"15min":1.724},"last":1.2989999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822710,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:34:01 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.83399999999999996,"15min":0.81499999999999995},"min":{"1min":0.68000000000000005,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.147,"5min":1.238,"15min":1.238},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.86199999999999999,"15min":0.88200000000000001},"min":{"1min":0.61699999999999999,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":1.0669999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:34:01 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.88100000000000001,"15min":0.873},"min":{"1min":0.65600000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.84399999999999997,"15min":0.83199999999999996},"min":{"1min":0.68000000000000005,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.1499999999999999,"5min":1.198,"15min":1.198},"last":1.016}]},{"osd":2,"last update":"Mon Oct 9 08:34:01 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.86899999999999999,"15min":0.86499999999999999},"min":{"1min":0.60099999999999998,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.173,"5min":1.282,"15min":1.282},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.90600000000000003,"15min":0.89400000000000002},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.1930000000000001,"5min":1.224,"15min":1.224},"last":1.0940000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:34:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.90700000000000003,"15min":0.92500000000000004},"min":{"1min":0.68400000000000005,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.2569999999999999,"5min":1.276,"15min":1.276},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.028,"5min":0.95499999999999996,"15min":0.92600000000000005},"min":{"1min":0.77600000000000002,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.1879999999999999,"5min":1.323,"15min":1.323},"last":0.998}]},{"osd":4,"last update":"Mon Oct 9 08:34:01 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.76400000000000001,"15min":0.71799999999999997},"min":{"1min":0.48399999999999999,"5min":0.439,"15min":0.439},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.88600000000000001,"5min":0.80900000000000005,"15min":0.76100000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.2529999999999999,"5min":1.387,"15min":1.387},"last":1.044}]},{"osd":6,"last update":"Mon Oct 9 08:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.88300000000000001,"15min":0.86699999999999999},"min":{"1min":0.59399999999999997,"5min":0.439,"15min":0.439},"max":{"1min":1.2689999999999999,"5min":1.349,"15min":1.349},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.87,"5min":0.83699999999999997,"15min":0.82099999999999995},"min":{"1min":0.55700000000000005,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.155,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.871}]},{"osd":7,"last update":"Mon Oct 9 08:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.88100000000000001,"15min":0.86099999999999999},"min":{"1min":0.64300000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.095,"5min":0.92400000000000004,"15min":0.89600000000000002},"min":{"1min":0.69599999999999995,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":0.90800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:34:14.100 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T08:34:14.100 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T08:34:14.101 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T08:34:14.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T08:34:14.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:34:14 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:14] "GET /metrics HTTP/1.1" 200 155646 "" "Prometheus/2.33.4" 2023-10-09T08:34:14.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:14 smithi094 bash[27803]: audit 2023-10-09T08:34:13.278070+0000 mgr.y (mgr.24355) 93 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:34:15.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:14 smithi016 bash[27454]: audit 2023-10-09T08:34:13.278070+0000 mgr.y (mgr.24355) 93 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:34:15.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:14 smithi016 bash[33851]: audit 2023-10-09T08:34:13.278070+0000 mgr.y (mgr.24355) 93 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:34:15.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:15 smithi094 bash[27803]: cluster 2023-10-09T08:34:14.635616+0000 mgr.y (mgr.24355) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:16.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:15 smithi016 bash[27454]: cluster 2023-10-09T08:34:14.635616+0000 mgr.y (mgr.24355) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:16.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:15 smithi016 bash[33851]: cluster 2023-10-09T08:34:14.635616+0000 mgr.y (mgr.24355) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:17.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:17 smithi094 bash[27803]: cluster 2023-10-09T08:34:16.636326+0000 mgr.y (mgr.24355) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:18.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:17 smithi016 bash[27454]: cluster 2023-10-09T08:34:16.636326+0000 mgr.y (mgr.24355) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:18.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:17 smithi016 bash[33851]: cluster 2023-10-09T08:34:16.636326+0000 mgr.y (mgr.24355) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:18.887 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:34:19.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:19 smithi094 bash[27803]: cluster 2023-10-09T08:34:18.636889+0000 mgr.y (mgr.24355) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:20.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:19 smithi016 bash[27454]: cluster 2023-10-09T08:34:18.636889+0000 mgr.y (mgr.24355) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:20.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:19 smithi016 bash[33851]: cluster 2023-10-09T08:34:18.636889+0000 mgr.y (mgr.24355) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:21.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:21 smithi016 bash[27454]: cluster 2023-10-09T08:34:20.637555+0000 mgr.y (mgr.24355) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:21.464 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:21 smithi016 bash[33851]: cluster 2023-10-09T08:34:20.637555+0000 mgr.y (mgr.24355) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:21 smithi094 bash[27803]: cluster 2023-10-09T08:34:20.637555+0000 mgr.y (mgr.24355) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:21.905 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:34:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:22 smithi016 bash[33851]: audit 2023-10-09T08:34:21.902337+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.16:0/2669484136' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:34:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:22 smithi016 bash[27454]: audit 2023-10-09T08:34:21.902337+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.16:0/2669484136' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:34:22.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:22 smithi094 bash[27803]: audit 2023-10-09T08:34:21.902337+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.16:0/2669484136' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:34:22.617 INFO:teuthology.orchestra.run.smithi016.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T08:34:22.618 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T08:34:22.619 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T08:34:22.619 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T08:34:22.629 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi016.front.sepia.ceph.com 2023-10-09T08:34:22.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T08:34:22.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:34:22 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:34:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:23 smithi094 bash[27803]: cluster 2023-10-09T08:34:22.638198+0000 mgr.y (mgr.24355) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:23.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:23 smithi016 bash[27454]: cluster 2023-10-09T08:34:22.638198+0000 mgr.y (mgr.24355) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:23.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:23 smithi016 bash[33851]: cluster 2023-10-09T08:34:22.638198+0000 mgr.y (mgr.24355) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:24.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:34:24 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:24] "GET /metrics HTTP/1.1" 200 155654 "" "Prometheus/2.33.4" 2023-10-09T08:34:25.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:25 smithi016 bash[27454]: cluster 2023-10-09T08:34:24.638768+0000 mgr.y (mgr.24355) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:25.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:25 smithi016 bash[33851]: cluster 2023-10-09T08:34:24.638768+0000 mgr.y (mgr.24355) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:25.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:25 smithi094 bash[27803]: cluster 2023-10-09T08:34:24.638768+0000 mgr.y (mgr.24355) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:26.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:26 smithi094 bash[27803]: audit 2023-10-09T08:34:25.951346+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T08:34:26.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:26 smithi094 bash[27803]: audit 2023-10-09T08:34:25.952169+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T08:34:27.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:26 smithi016 bash[33851]: audit 2023-10-09T08:34:25.951346+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T08:34:27.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:26 smithi016 bash[33851]: audit 2023-10-09T08:34:25.952169+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T08:34:27.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:26 smithi016 bash[27454]: audit 2023-10-09T08:34:25.951346+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T08:34:27.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:26 smithi016 bash[27454]: audit 2023-10-09T08:34:25.952169+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T08:34:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:27 smithi094 bash[27803]: audit 2023-10-09T08:34:25.950283+0000 mgr.y (mgr.24355) 100 : audit [DBG] from='client.24548 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:34:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:27 smithi094 bash[27803]: cluster 2023-10-09T08:34:26.639402+0000 mgr.y (mgr.24355) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:27 smithi094 bash[27803]: audit 2023-10-09T08:34:26.710058+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T08:34:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:27 smithi094 bash[27803]: cluster 2023-10-09T08:34:26.710154+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T08:34:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:27 smithi094 bash[27803]: audit 2023-10-09T08:34:26.715639+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:27 smithi094 bash[27803]: audit 2023-10-09T08:34:26.717137+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:28.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[33851]: audit 2023-10-09T08:34:25.950283+0000 mgr.y (mgr.24355) 100 : audit [DBG] from='client.24548 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:34:28.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[33851]: cluster 2023-10-09T08:34:26.639402+0000 mgr.y (mgr.24355) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:28.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[33851]: audit 2023-10-09T08:34:26.710058+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T08:34:28.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[33851]: cluster 2023-10-09T08:34:26.710154+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[33851]: audit 2023-10-09T08:34:26.715639+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[33851]: audit 2023-10-09T08:34:26.717137+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[27454]: audit 2023-10-09T08:34:25.950283+0000 mgr.y (mgr.24355) 100 : audit [DBG] from='client.24548 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[27454]: cluster 2023-10-09T08:34:26.639402+0000 mgr.y (mgr.24355) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[27454]: audit 2023-10-09T08:34:26.710058+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[27454]: cluster 2023-10-09T08:34:26.710154+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[27454]: audit 2023-10-09T08:34:26.715639+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[27454]: audit 2023-10-09T08:34:26.717137+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:28.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:27 smithi016 bash[27454]: debug 2023-10-09T08:34:27.734+0000 7f4b58864700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T08:34:28.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T08:34:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:27.716058+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T08:34:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: cluster 2023-10-09T08:34:27.716182+0000 mon.a (mon.0) 622 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T08:34:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:27.719757+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:27.738444+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: cluster 2023-10-09T08:34:27.739326+0000 mon.a (mon.0) 624 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: cluster 2023-10-09T08:34:27.739363+0000 mon.a (mon.0) 625 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: cluster 2023-10-09T08:34:27.744471+0000 mon.a (mon.0) 626 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:27.745040+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: cluster 2023-10-09T08:34:27.745167+0000 mon.a (mon.0) 628 : cluster [DBG] fsmap foo:0 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: cephadm 2023-10-09T08:34:27.749409+0000 mgr.y (mgr.24355) 102 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:27.755823+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:27.793032+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.067062+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.337549+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.338620+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:28.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.345062+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:28.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.356972+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:28.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.358369+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:28.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.358883+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:28.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.363183+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T08:34:28.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:28 smithi094 bash[27803]: audit 2023-10-09T08:34:28.364556+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:29.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:27.716058+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T08:34:29.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: cluster 2023-10-09T08:34:27.716182+0000 mon.a (mon.0) 622 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T08:34:29.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:27.719757+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:29.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:27.738444+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:29.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: cluster 2023-10-09T08:34:27.739326+0000 mon.a (mon.0) 624 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T08:34:29.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: cluster 2023-10-09T08:34:27.739363+0000 mon.a (mon.0) 625 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: cluster 2023-10-09T08:34:27.744471+0000 mon.a (mon.0) 626 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:27.745040+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: cluster 2023-10-09T08:34:27.745167+0000 mon.a (mon.0) 628 : cluster [DBG] fsmap foo:0 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: cephadm 2023-10-09T08:34:27.749409+0000 mgr.y (mgr.24355) 102 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:27.755823+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:27.793032+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.067062+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.337549+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.338620+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.345062+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.356972+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.358369+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.358883+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.363183+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[33851]: audit 2023-10-09T08:34:28.364556+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:27.716058+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: cluster 2023-10-09T08:34:27.716182+0000 mon.a (mon.0) 622 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T08:34:29.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:27.719757+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:27.738444+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: cluster 2023-10-09T08:34:27.739326+0000 mon.a (mon.0) 624 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: cluster 2023-10-09T08:34:27.739363+0000 mon.a (mon.0) 625 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: cluster 2023-10-09T08:34:27.744471+0000 mon.a (mon.0) 626 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:27.745040+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: cluster 2023-10-09T08:34:27.745167+0000 mon.a (mon.0) 628 : cluster [DBG] fsmap foo:0 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: cephadm 2023-10-09T08:34:27.749409+0000 mgr.y (mgr.24355) 102 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:27.755823+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:27.793032+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.067062+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.337549+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.338620+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.345062+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.356972+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.358369+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.358883+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.363183+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T08:34:29.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:28 smithi016 bash[27454]: audit 2023-10-09T08:34:28.364556+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:30.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:29 smithi016 bash[33851]: cephadm 2023-10-09T08:34:28.365422+0000 mgr.y (mgr.24355) 103 : cephadm [INF] Deploying daemon mds.foo.smithi016.ccdktz on smithi016 2023-10-09T08:34:30.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:29 smithi016 bash[33851]: cluster 2023-10-09T08:34:28.639804+0000 mgr.y (mgr.24355) 104 : cluster [DBG] pgmap v82: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:30.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:29 smithi016 bash[33851]: cluster 2023-10-09T08:34:28.745468+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T08:34:30.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:29 smithi016 bash[27454]: cephadm 2023-10-09T08:34:28.365422+0000 mgr.y (mgr.24355) 103 : cephadm [INF] Deploying daemon mds.foo.smithi016.ccdktz on smithi016 2023-10-09T08:34:30.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:29 smithi016 bash[27454]: cluster 2023-10-09T08:34:28.639804+0000 mgr.y (mgr.24355) 104 : cluster [DBG] pgmap v82: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:30.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:29 smithi016 bash[27454]: cluster 2023-10-09T08:34:28.745468+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T08:34:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:29 smithi094 bash[27803]: cephadm 2023-10-09T08:34:28.365422+0000 mgr.y (mgr.24355) 103 : cephadm [INF] Deploying daemon mds.foo.smithi016.ccdktz on smithi016 2023-10-09T08:34:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:29 smithi094 bash[27803]: cluster 2023-10-09T08:34:28.639804+0000 mgr.y (mgr.24355) 104 : cluster [DBG] pgmap v82: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:29 smithi094 bash[27803]: cluster 2023-10-09T08:34:28.745468+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T08:34:31.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:30 smithi094 bash[27803]: cluster 2023-10-09T08:34:29.755493+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T08:34:31.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:30 smithi016 bash[27454]: cluster 2023-10-09T08:34:29.755493+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T08:34:31.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:30 smithi016 bash[33851]: cluster 2023-10-09T08:34:29.755493+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T08:34:31.792 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:31 smithi016 bash[27454]: cluster 2023-10-09T08:34:30.640670+0000 mgr.y (mgr.24355) 105 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:31.792 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:31 smithi016 bash[33851]: cluster 2023-10-09T08:34:30.640670+0000 mgr.y (mgr.24355) 105 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:31 smithi094 bash[27803]: cluster 2023-10-09T08:34:30.640670+0000 mgr.y (mgr.24355) 105 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:32.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:34:32 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:34:33.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[27454]: audit 2023-10-09T08:34:32.047479+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:33.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[27454]: audit 2023-10-09T08:34:32.050105+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[27454]: audit 2023-10-09T08:34:32.050487+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[27454]: audit 2023-10-09T08:34:32.055281+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[27454]: audit 2023-10-09T08:34:32.056591+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[27454]: cephadm 2023-10-09T08:34:32.057450+0000 mgr.y (mgr.24355) 106 : cephadm [INF] Deploying daemon mds.foo.smithi094.zfssci on smithi094 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[27454]: cluster 2023-10-09T08:34:32.641264+0000 mgr.y (mgr.24355) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[33851]: audit 2023-10-09T08:34:32.047479+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[33851]: audit 2023-10-09T08:34:32.050105+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:33.371 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[33851]: audit 2023-10-09T08:34:32.050487+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:33.372 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[33851]: audit 2023-10-09T08:34:32.055281+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T08:34:33.372 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[33851]: audit 2023-10-09T08:34:32.056591+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:33.372 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[33851]: cephadm 2023-10-09T08:34:32.057450+0000 mgr.y (mgr.24355) 106 : cephadm [INF] Deploying daemon mds.foo.smithi094.zfssci on smithi094 2023-10-09T08:34:33.372 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:33 smithi016 bash[33851]: cluster 2023-10-09T08:34:32.641264+0000 mgr.y (mgr.24355) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:33.395 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:33 smithi094 bash[27803]: audit 2023-10-09T08:34:32.047479+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:33.395 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:33 smithi094 bash[27803]: audit 2023-10-09T08:34:32.050105+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:33.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:33 smithi094 bash[27803]: audit 2023-10-09T08:34:32.050487+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:34:33.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:33 smithi094 bash[27803]: audit 2023-10-09T08:34:32.055281+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T08:34:33.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:33 smithi094 bash[27803]: audit 2023-10-09T08:34:32.056591+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:33.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:33 smithi094 bash[27803]: cephadm 2023-10-09T08:34:32.057450+0000 mgr.y (mgr.24355) 106 : cephadm [INF] Deploying daemon mds.foo.smithi094.zfssci on smithi094 2023-10-09T08:34:33.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:33 smithi094 bash[27803]: cluster 2023-10-09T08:34:32.641264+0000 mgr.y (mgr.24355) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:34.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:34:34 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:34] "GET /metrics HTTP/1.1" 200 155654 "" "Prometheus/2.33.4" 2023-10-09T08:34:34.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: audit 2023-10-09T08:34:34.059617+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.061227+0000 mon.a (mon.0) 640 : cluster [DBG] mds.? [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] up:boot 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.061329+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi016.ccdktz assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.061596+0000 mon.a (mon.0) 642 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.061674+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.061786+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.062123+0000 mon.a (mon.0) 645 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.066716+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:creating} 2023-10-09T08:34:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.085891+0000 mon.a (mon.0) 647 : cluster [INF] daemon mds.foo.smithi016.ccdktz is now active in filesystem foo as rank 0 2023-10-09T08:34:35.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: cluster 2023-10-09T08:34:34.641984+0000 mgr.y (mgr.24355) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:35.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: audit 2023-10-09T08:34:34.909928+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:35.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:35 smithi094 bash[27803]: audit 2023-10-09T08:34:34.912648+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:35.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: audit 2023-10-09T08:34:34.059617+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:34:35.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.061227+0000 mon.a (mon.0) 640 : cluster [DBG] mds.? [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] up:boot 2023-10-09T08:34:35.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.061329+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi016.ccdktz assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T08:34:35.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.061596+0000 mon.a (mon.0) 642 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T08:34:35.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.061674+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T08:34:35.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.061786+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2023-10-09T08:34:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.062123+0000 mon.a (mon.0) 645 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T08:34:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.066716+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:creating} 2023-10-09T08:34:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.085891+0000 mon.a (mon.0) 647 : cluster [INF] daemon mds.foo.smithi016.ccdktz is now active in filesystem foo as rank 0 2023-10-09T08:34:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: cluster 2023-10-09T08:34:34.641984+0000 mgr.y (mgr.24355) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: audit 2023-10-09T08:34:34.909928+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[27454]: audit 2023-10-09T08:34:34.912648+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: audit 2023-10-09T08:34:34.059617+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.061227+0000 mon.a (mon.0) 640 : cluster [DBG] mds.? [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] up:boot 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.061329+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi016.ccdktz assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.061596+0000 mon.a (mon.0) 642 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.061674+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.061786+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.062123+0000 mon.a (mon.0) 645 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.066716+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:creating} 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.085891+0000 mon.a (mon.0) 647 : cluster [INF] daemon mds.foo.smithi016.ccdktz is now active in filesystem foo as rank 0 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: cluster 2023-10-09T08:34:34.641984+0000 mgr.y (mgr.24355) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:35.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: audit 2023-10-09T08:34:34.909928+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:35.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:35 smithi016 bash[33851]: audit 2023-10-09T08:34:34.912648+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:36.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:36 smithi094 bash[27803]: cluster 2023-10-09T08:34:35.068975+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] up:active 2023-10-09T08:34:36.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:36 smithi094 bash[27803]: cluster 2023-10-09T08:34:35.069039+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 2023-10-09T08:34:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:36 smithi016 bash[27454]: cluster 2023-10-09T08:34:35.068975+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] up:active 2023-10-09T08:34:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:36 smithi016 bash[27454]: cluster 2023-10-09T08:34:35.069039+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 2023-10-09T08:34:36.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:36 smithi016 bash[33851]: cluster 2023-10-09T08:34:35.068975+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] up:active 2023-10-09T08:34:36.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:36 smithi016 bash[33851]: cluster 2023-10-09T08:34:35.069039+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 2023-10-09T08:34:37.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:37 smithi094 bash[27803]: cluster 2023-10-09T08:34:36.643101+0000 mgr.y (mgr.24355) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:37.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:37 smithi016 bash[27454]: cluster 2023-10-09T08:34:36.643101+0000 mgr.y (mgr.24355) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:37 smithi016 bash[33851]: cluster 2023-10-09T08:34:36.643101+0000 mgr.y (mgr.24355) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:38.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[27454]: cluster 2023-10-09T08:34:37.082331+0000 mon.a (mon.0) 651 : cluster [DBG] mds.? [v2:172.21.15.94:6832/1000218818,v1:172.21.15.94:6833/1000218818] up:boot 2023-10-09T08:34:38.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[27454]: audit 2023-10-09T08:34:37.082446+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:34:38.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[27454]: cluster 2023-10-09T08:34:37.082798+0000 mon.a (mon.0) 652 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:34:38.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[27454]: cluster 2023-10-09T08:34:37.086499+0000 mon.a (mon.0) 653 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:34:38.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[33851]: cluster 2023-10-09T08:34:37.082331+0000 mon.a (mon.0) 651 : cluster [DBG] mds.? [v2:172.21.15.94:6832/1000218818,v1:172.21.15.94:6833/1000218818] up:boot 2023-10-09T08:34:38.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[33851]: audit 2023-10-09T08:34:37.082446+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:34:38.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[33851]: cluster 2023-10-09T08:34:37.082798+0000 mon.a (mon.0) 652 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:34:38.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:38 smithi016 bash[33851]: cluster 2023-10-09T08:34:37.086499+0000 mon.a (mon.0) 653 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:34:38.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:38 smithi094 bash[27803]: cluster 2023-10-09T08:34:37.082331+0000 mon.a (mon.0) 651 : cluster [DBG] mds.? [v2:172.21.15.94:6832/1000218818,v1:172.21.15.94:6833/1000218818] up:boot 2023-10-09T08:34:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:38 smithi094 bash[27803]: audit 2023-10-09T08:34:37.082446+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:34:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:38 smithi094 bash[27803]: cluster 2023-10-09T08:34:37.082798+0000 mon.a (mon.0) 652 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:34:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:38 smithi094 bash[27803]: cluster 2023-10-09T08:34:37.086499+0000 mon.a (mon.0) 653 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:34:39.028 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T08:34:39.037 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T08:34:39.037 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T08:34:39.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T08:34:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:39 smithi094 bash[27803]: cluster 2023-10-09T08:34:38.643745+0000 mgr.y (mgr.24355) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:39.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:39 smithi016 bash[27454]: cluster 2023-10-09T08:34:38.643745+0000 mgr.y (mgr.24355) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:39.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:39 smithi016 bash[33851]: cluster 2023-10-09T08:34:38.643745+0000 mgr.y (mgr.24355) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:41.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:40 smithi094 bash[27803]: audit 2023-10-09T08:34:39.906230+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:41.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:40 smithi094 bash[27803]: cluster 2023-10-09T08:34:40.644432+0000 mgr.y (mgr.24355) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:41.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:40 smithi094 bash[27803]: audit 2023-10-09T08:34:40.662390+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:41.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:40 smithi016 bash[33851]: audit 2023-10-09T08:34:39.906230+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:41.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:40 smithi016 bash[33851]: cluster 2023-10-09T08:34:40.644432+0000 mgr.y (mgr.24355) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:41.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:40 smithi016 bash[33851]: audit 2023-10-09T08:34:40.662390+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:41.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:40 smithi016 bash[27454]: audit 2023-10-09T08:34:39.906230+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:41.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:40 smithi016 bash[27454]: cluster 2023-10-09T08:34:40.644432+0000 mgr.y (mgr.24355) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:41.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:40 smithi016 bash[27454]: audit 2023-10-09T08:34:40.662390+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:42.793 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:34:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.790740+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.793692+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.795085+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.802818+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.810911+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.813260+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:43.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.814559+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:43.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:42 smithi094 bash[27803]: audit 2023-10-09T08:34:41.821799+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.790740+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.793692+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.795085+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.802818+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.810911+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.813260+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.814559+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[27454]: audit 2023-10-09T08:34:41.821799+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.790740+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.793692+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:43.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.795085+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:43.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.802818+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.810911+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:34:43.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.813260+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:34:43.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.814559+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:34:43.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:42 smithi016 bash[33851]: audit 2023-10-09T08:34:41.821799+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:43.767 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:34:44.143 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:43 smithi016 bash[27454]: cluster 2023-10-09T08:34:42.645034+0000 mgr.y (mgr.24355) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:44.143 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:43 smithi016 bash[33851]: cluster 2023-10-09T08:34:42.645034+0000 mgr.y (mgr.24355) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:44.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:43 smithi094 bash[27803]: cluster 2023-10-09T08:34:42.645034+0000 mgr.y (mgr.24355) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:44.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:34:44 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:44] "GET /metrics HTTP/1.1" 200 190739 "" "Prometheus/2.33.4" 2023-10-09T08:34:46.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:45 smithi016 bash[27454]: cluster 2023-10-09T08:34:44.645681+0000 mgr.y (mgr.24355) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:46.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:45 smithi016 bash[27454]: audit 2023-10-09T08:34:44.921349+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:46.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:45 smithi016 bash[33851]: cluster 2023-10-09T08:34:44.645681+0000 mgr.y (mgr.24355) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:46.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:45 smithi016 bash[33851]: audit 2023-10-09T08:34:44.921349+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:45 smithi094 bash[27803]: cluster 2023-10-09T08:34:44.645681+0000 mgr.y (mgr.24355) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:45 smithi094 bash[27803]: audit 2023-10-09T08:34:44.921349+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:34:47.068 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:34:47.100 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:46 smithi016 bash[27454]: cluster 2023-10-09T08:34:46.646701+0000 mgr.y (mgr.24355) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:47.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:46 smithi016 bash[33851]: cluster 2023-10-09T08:34:46.646701+0000 mgr.y (mgr.24355) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:47.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:46 smithi094 bash[27803]: cluster 2023-10-09T08:34:46.646701+0000 mgr.y (mgr.24355) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:47.849 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":19,"active_gid":24355,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6800","nonce":1710200957},{"type":"v1","addr":"172.21.15.16:6801","nonce":1710200957}]},"active_addr":"172.21.15.16:6801/1710200957","active_change":"2023-10-09T08:32:04.579738+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":1478513102}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":1021312597}]}]} 2023-10-09T08:34:47.853 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T08:34:47.853 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T08:34:47.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T08:34:48.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:47 smithi094 bash[27803]: audit 2023-10-09T08:34:47.059928+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.16:0/1913581559' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:34:48.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:47 smithi016 bash[27454]: audit 2023-10-09T08:34:47.059928+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.16:0/1913581559' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:34:48.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:47 smithi016 bash[33851]: audit 2023-10-09T08:34:47.059928+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.16:0/1913581559' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:34:49.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:48 smithi094 bash[27803]: cluster 2023-10-09T08:34:48.647406+0000 mgr.y (mgr.24355) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:49.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:48 smithi016 bash[27454]: cluster 2023-10-09T08:34:48.647406+0000 mgr.y (mgr.24355) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:48 smithi016 bash[33851]: cluster 2023-10-09T08:34:48.647406+0000 mgr.y (mgr.24355) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:51 smithi094 bash[27803]: cluster 2023-10-09T08:34:50.648428+0000 mgr.y (mgr.24355) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:51 smithi016 bash[27454]: cluster 2023-10-09T08:34:50.648428+0000 mgr.y (mgr.24355) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:51 smithi016 bash[33851]: cluster 2023-10-09T08:34:50.648428+0000 mgr.y (mgr.24355) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T08:34:52.637 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:34:52.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:34:52 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:34:53.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:53 smithi094 bash[27803]: cluster 2023-10-09T08:34:52.648989+0000 mgr.y (mgr.24355) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:53.995 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:53 smithi016 bash[27454]: cluster 2023-10-09T08:34:52.648989+0000 mgr.y (mgr.24355) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:53.996 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:53 smithi016 bash[33851]: cluster 2023-10-09T08:34:52.648989+0000 mgr.y (mgr.24355) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:54.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:34:54 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:34:54] "GET /metrics HTTP/1.1" 200 191342 "" "Prometheus/2.33.4" 2023-10-09T08:34:55.668 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:34:55.669 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":60,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:34:29.743323+0000","last_up_change":"2023-10-09T08:31:29.882559+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T08:34:26.406514+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T08:34:27.172366+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6803","nonce":818359725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6805","nonce":818359725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6809","nonce":818359725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6807","nonce":818359725}]},"public_addr":"172.21.15.16:6803/818359725","cluster_addr":"172.21.15.16:6805/818359725","heartbeat_back_addr":"172.21.15.16:6809/818359725","heartbeat_front_addr":"172.21.15.16:6807/818359725","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6811","nonce":664142092}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6813","nonce":664142092}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6817","nonce":664142092}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6815","nonce":664142092}]},"public_addr":"172.21.15.16:6811/664142092","cluster_addr":"172.21.15.16:6813/664142092","heartbeat_back_addr":"172.21.15.16:6817/664142092","heartbeat_front_addr":"172.21.15.16:6815/664142092","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6819","nonce":3541024001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6821","nonce":3541024001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6825","nonce":3541024001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6823","nonce":3541024001}]},"public_addr":"172.21.15.16:6819/3541024001","cluster_addr":"172.21.15.16:6821/3541024001","heartbeat_back_addr":"172.21.15.16:6825/3541024001","heartbeat_front_addr":"172.21.15.16:6823/3541024001","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6827","nonce":3124750952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6829","nonce":3124750952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6833","nonce":3124750952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6831","nonce":3124750952}]},"public_addr":"172.21.15.16:6827/3124750952","cluster_addr":"172.21.15.16:6829/3124750952","heartbeat_back_addr":"172.21.15.16:6833/3124750952","heartbeat_front_addr":"172.21.15.16:6831/3124750952","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:34:55.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:55 smithi094 bash[27803]: cluster 2023-10-09T08:34:54.649534+0000 mgr.y (mgr.24355) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:55.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:55 smithi094 bash[27803]: audit 2023-10-09T08:34:55.662957+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.16:0/1507589413' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:34:56.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:55 smithi016 bash[27454]: cluster 2023-10-09T08:34:54.649534+0000 mgr.y (mgr.24355) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:56.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:55 smithi016 bash[27454]: audit 2023-10-09T08:34:55.662957+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.16:0/1507589413' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:34:56.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:55 smithi016 bash[33851]: cluster 2023-10-09T08:34:54.649534+0000 mgr.y (mgr.24355) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:56.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:55 smithi016 bash[33851]: audit 2023-10-09T08:34:55.662957+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.16:0/1507589413' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:34:56.395 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T08:34:56.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T08:34:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:57 smithi094 bash[27803]: cluster 2023-10-09T08:34:56.650573+0000 mgr.y (mgr.24355) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:58.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:57 smithi016 bash[27454]: cluster 2023-10-09T08:34:56.650573+0000 mgr.y (mgr.24355) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:58.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:57 smithi016 bash[33851]: cluster 2023-10-09T08:34:56.650573+0000 mgr.y (mgr.24355) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:34:59.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:34:59 smithi094 bash[27803]: cluster 2023-10-09T08:34:58.651240+0000 mgr.y (mgr.24355) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:34:59 smithi016 bash[27454]: cluster 2023-10-09T08:34:58.651240+0000 mgr.y (mgr.24355) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:00.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:34:59 smithi016 bash[33851]: cluster 2023-10-09T08:34:58.651240+0000 mgr.y (mgr.24355) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:01.174 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:01.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:01 smithi094 bash[27803]: cluster 2023-10-09T08:35:00.652258+0000 mgr.y (mgr.24355) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:01.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:01 smithi016 bash[33851]: cluster 2023-10-09T08:35:00.652258+0000 mgr.y (mgr.24355) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:01.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:01 smithi016 bash[27454]: cluster 2023-10-09T08:35:00.652258+0000 mgr.y (mgr.24355) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:02.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:35:02 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:35:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:03 smithi094 bash[27803]: cluster 2023-10-09T08:35:02.652935+0000 mgr.y (mgr.24355) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:04.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:03 smithi016 bash[27454]: cluster 2023-10-09T08:35:02.652935+0000 mgr.y (mgr.24355) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:04.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:03 smithi016 bash[33851]: cluster 2023-10-09T08:35:02.652935+0000 mgr.y (mgr.24355) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:04.345 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:35:04.345 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":60,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:34:29.743323+0000","last_up_change":"2023-10-09T08:31:29.882559+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T08:34:26.406514+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T08:34:27.172366+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6803","nonce":818359725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6805","nonce":818359725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6809","nonce":818359725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":818359725},{"type":"v1","addr":"172.21.15.16:6807","nonce":818359725}]},"public_addr":"172.21.15.16:6803/818359725","cluster_addr":"172.21.15.16:6805/818359725","heartbeat_back_addr":"172.21.15.16:6809/818359725","heartbeat_front_addr":"172.21.15.16:6807/818359725","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6811","nonce":664142092}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6813","nonce":664142092}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6817","nonce":664142092}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":664142092},{"type":"v1","addr":"172.21.15.16:6815","nonce":664142092}]},"public_addr":"172.21.15.16:6811/664142092","cluster_addr":"172.21.15.16:6813/664142092","heartbeat_back_addr":"172.21.15.16:6817/664142092","heartbeat_front_addr":"172.21.15.16:6815/664142092","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6819","nonce":3541024001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6821","nonce":3541024001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6825","nonce":3541024001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":3541024001},{"type":"v1","addr":"172.21.15.16:6823","nonce":3541024001}]},"public_addr":"172.21.15.16:6819/3541024001","cluster_addr":"172.21.15.16:6821/3541024001","heartbeat_back_addr":"172.21.15.16:6825/3541024001","heartbeat_front_addr":"172.21.15.16:6823/3541024001","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6827","nonce":3124750952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6829","nonce":3124750952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6833","nonce":3124750952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3124750952},{"type":"v1","addr":"172.21.15.16:6831","nonce":3124750952}]},"public_addr":"172.21.15.16:6827/3124750952","cluster_addr":"172.21.15.16:6829/3124750952","heartbeat_back_addr":"172.21.15.16:6833/3124750952","heartbeat_front_addr":"172.21.15.16:6831/3124750952","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:35:04.398 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:04] "GET /metrics HTTP/1.1" 200 191342 "" "Prometheus/2.33.4" 2023-10-09T08:35:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:04 smithi094 bash[27803]: audit 2023-10-09T08:35:04.339396+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.16:0/3020724693' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:35:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:04 smithi094 bash[27803]: audit 2023-10-09T08:35:04.618338+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T08:35:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:04 smithi094 bash[27803]: audit 2023-10-09T08:35:04.619069+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T08:35:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:04 smithi094 bash[27803]: audit 2023-10-09T08:35:04.619081+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T08:35:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:04 smithi094 bash[27803]: audit 2023-10-09T08:35:04.619610+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T08:35:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:04 smithi094 bash[27803]: audit 2023-10-09T08:35:04.619752+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T08:35:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:04 smithi094 bash[27803]: audit 2023-10-09T08:35:04.620412+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[33851]: audit 2023-10-09T08:35:04.339396+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.16:0/3020724693' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[33851]: audit 2023-10-09T08:35:04.618338+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[33851]: audit 2023-10-09T08:35:04.619069+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[33851]: audit 2023-10-09T08:35:04.619081+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[33851]: audit 2023-10-09T08:35:04.619610+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[33851]: audit 2023-10-09T08:35:04.619752+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[33851]: audit 2023-10-09T08:35:04.620412+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27454]: audit 2023-10-09T08:35:04.339396+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.16:0/3020724693' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:35:05.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27454]: audit 2023-10-09T08:35:04.618338+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T08:35:05.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27454]: audit 2023-10-09T08:35:04.619069+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T08:35:05.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27454]: audit 2023-10-09T08:35:04.619081+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27454]: audit 2023-10-09T08:35:04.619610+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27454]: audit 2023-10-09T08:35:04.619752+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:04 smithi016 bash[27454]: audit 2023-10-09T08:35:04.620412+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T08:35:05.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T08:35:05.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T08:35:05.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T08:35:05.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T08:35:05.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T08:35:05.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T08:35:05.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T08:35:05.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T08:35:05.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:05 smithi094 bash[27803]: cluster 2023-10-09T08:35:04.653546+0000 mgr.y (mgr.24355) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:05.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:05 smithi094 bash[27803]: audit 2023-10-09T08:35:04.723819+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T08:35:05.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:05 smithi094 bash[27803]: audit 2023-10-09T08:35:04.723987+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T08:35:05.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:05 smithi094 bash[27803]: audit 2023-10-09T08:35:04.724124+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T08:35:05.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:05 smithi094 bash[27803]: cluster 2023-10-09T08:35:04.724250+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T08:35:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[27454]: cluster 2023-10-09T08:35:04.653546+0000 mgr.y (mgr.24355) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[27454]: audit 2023-10-09T08:35:04.723819+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T08:35:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[27454]: audit 2023-10-09T08:35:04.723987+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T08:35:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[27454]: audit 2023-10-09T08:35:04.724124+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T08:35:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[27454]: cluster 2023-10-09T08:35:04.724250+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T08:35:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[33851]: cluster 2023-10-09T08:35:04.653546+0000 mgr.y (mgr.24355) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[33851]: audit 2023-10-09T08:35:04.723819+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T08:35:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[33851]: audit 2023-10-09T08:35:04.723987+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T08:35:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[33851]: audit 2023-10-09T08:35:04.724124+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T08:35:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:05 smithi016 bash[33851]: cluster 2023-10-09T08:35:04.724250+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T08:35:06.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:06 smithi094 bash[27803]: cluster 2023-10-09T08:35:05.728326+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T08:35:07.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:06 smithi016 bash[27454]: cluster 2023-10-09T08:35:05.728326+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T08:35:07.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:06 smithi016 bash[33851]: cluster 2023-10-09T08:35:05.728326+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T08:35:08.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:07 smithi016 bash[27454]: cluster 2023-10-09T08:35:06.654218+0000 mgr.y (mgr.24355) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:08.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:07 smithi016 bash[33851]: cluster 2023-10-09T08:35:06.654218+0000 mgr.y (mgr.24355) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:07 smithi094 bash[27803]: cluster 2023-10-09T08:35:06.654218+0000 mgr.y (mgr.24355) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:10.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:09 smithi016 bash[27454]: cluster 2023-10-09T08:35:08.654879+0000 mgr.y (mgr.24355) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:10.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:09 smithi016 bash[33851]: cluster 2023-10-09T08:35:08.654879+0000 mgr.y (mgr.24355) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:10.020 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.021 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.021 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.021 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.022 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.023 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.027 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.027 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:10.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:09 smithi094 bash[27803]: cluster 2023-10-09T08:35:08.654879+0000 mgr.y (mgr.24355) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:11.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:11 smithi094 bash[27803]: cluster 2023-10-09T08:35:10.655975+0000 mgr.y (mgr.24355) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:11.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:11 smithi016 bash[27454]: cluster 2023-10-09T08:35:10.655975+0000 mgr.y (mgr.24355) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:11.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:11 smithi016 bash[33851]: cluster 2023-10-09T08:35:10.655975+0000 mgr.y (mgr.24355) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:12.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:35:12 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:35:13.823 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:13 smithi016 bash[27454]: cluster 2023-10-09T08:35:12.656565+0000 mgr.y (mgr.24355) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:13.823 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:13 smithi016 bash[33851]: cluster 2023-10-09T08:35:12.656565+0000 mgr.y (mgr.24355) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:13.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:13 smithi094 bash[27803]: cluster 2023-10-09T08:35:12.656565+0000 mgr.y (mgr.24355) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:14.400 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:35:14 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:14] "GET /metrics HTTP/1.1" 200 191344 "" "Prometheus/2.33.4" 2023-10-09T08:35:15.862 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:15 smithi016 bash[27454]: cluster 2023-10-09T08:35:14.657114+0000 mgr.y (mgr.24355) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:15.862 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:15 smithi016 bash[33851]: cluster 2023-10-09T08:35:14.657114+0000 mgr.y (mgr.24355) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:15.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:15 smithi094 bash[27803]: cluster 2023-10-09T08:35:14.657114+0000 mgr.y (mgr.24355) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:17.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:17 smithi094 bash[27803]: cluster 2023-10-09T08:35:16.657974+0000 mgr.y (mgr.24355) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:18.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:17 smithi016 bash[27454]: cluster 2023-10-09T08:35:16.657974+0000 mgr.y (mgr.24355) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:18.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:17 smithi016 bash[33851]: cluster 2023-10-09T08:35:16.657974+0000 mgr.y (mgr.24355) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:19.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:19 smithi094 bash[27803]: cluster 2023-10-09T08:35:18.658345+0000 mgr.y (mgr.24355) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:19.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:19 smithi016 bash[27454]: cluster 2023-10-09T08:35:18.658345+0000 mgr.y (mgr.24355) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:19.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:19 smithi016 bash[33851]: cluster 2023-10-09T08:35:18.658345+0000 mgr.y (mgr.24355) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:21.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:21 smithi016 bash[27454]: cluster 2023-10-09T08:35:20.659352+0000 mgr.y (mgr.24355) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:21.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:21 smithi016 bash[33851]: cluster 2023-10-09T08:35:20.659352+0000 mgr.y (mgr.24355) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:21 smithi094 bash[27803]: cluster 2023-10-09T08:35:20.659352+0000 mgr.y (mgr.24355) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:22.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:35:22 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:35:23.410 INFO:teuthology.orchestra.run.smithi016.stdout:34359738481 2023-10-09T08:35:23.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T08:35:23.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:23 smithi094 bash[27803]: cluster 2023-10-09T08:35:22.660106+0000 mgr.y (mgr.24355) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:23.995 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:23 smithi016 bash[27454]: cluster 2023-10-09T08:35:22.660106+0000 mgr.y (mgr.24355) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:23.995 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:23 smithi016 bash[33851]: cluster 2023-10-09T08:35:22.660106+0000 mgr.y (mgr.24355) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:24.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:35:24 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:24] "GET /metrics HTTP/1.1" 200 191361 "" "Prometheus/2.33.4" 2023-10-09T08:35:24.274 INFO:teuthology.orchestra.run.smithi016.stdout:107374182486 2023-10-09T08:35:24.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T08:35:25.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:25 smithi016 bash[27454]: cluster 2023-10-09T08:35:24.660633+0000 mgr.y (mgr.24355) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:25.978 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:25 smithi016 bash[33851]: cluster 2023-10-09T08:35:24.660633+0000 mgr.y (mgr.24355) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:25.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:25 smithi094 bash[27803]: cluster 2023-10-09T08:35:24.660633+0000 mgr.y (mgr.24355) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:26.209 INFO:teuthology.orchestra.run.smithi016.stdout:55834574951 2023-10-09T08:35:26.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T08:35:26.274 INFO:teuthology.orchestra.run.smithi016.stdout:154618822725 2023-10-09T08:35:26.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T08:35:26.757 INFO:teuthology.orchestra.run.smithi016.stdout:180388626491 2023-10-09T08:35:26.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T08:35:27.449 INFO:teuthology.orchestra.run.smithi016.stdout:206158430258 2023-10-09T08:35:27.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T08:35:27.630 INFO:teuthology.orchestra.run.smithi016.stdout:128849018957 2023-10-09T08:35:27.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T08:35:27.685 INFO:teuthology.orchestra.run.smithi016.stdout:77309411423 2023-10-09T08:35:27.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T08:35:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:27 smithi094 bash[27803]: cluster 2023-10-09T08:35:26.661706+0000 mgr.y (mgr.24355) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:28.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:27 smithi016 bash[27454]: cluster 2023-10-09T08:35:26.661706+0000 mgr.y (mgr.24355) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:28.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:27 smithi016 bash[33851]: cluster 2023-10-09T08:35:26.661706+0000 mgr.y (mgr.24355) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:29 smithi094 bash[27803]: cluster 2023-10-09T08:35:28.662478+0000 mgr.y (mgr.24355) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:30.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:29 smithi016 bash[27454]: cluster 2023-10-09T08:35:28.662478+0000 mgr.y (mgr.24355) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:30.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:29 smithi016 bash[33851]: cluster 2023-10-09T08:35:28.662478+0000 mgr.y (mgr.24355) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:30.207 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:30.207 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:30.209 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:30.214 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:31.221 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:31.222 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:31 smithi094 bash[27803]: cluster 2023-10-09T08:35:30.663423+0000 mgr.y (mgr.24355) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:31.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:31 smithi016 bash[27454]: cluster 2023-10-09T08:35:30.663423+0000 mgr.y (mgr.24355) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:31.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:31 smithi016 bash[33851]: cluster 2023-10-09T08:35:30.663423+0000 mgr.y (mgr.24355) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:32.234 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:32.235 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:32.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:35:32 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:35:33.990 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:33 smithi016 bash[27454]: cluster 2023-10-09T08:35:32.664031+0000 mgr.y (mgr.24355) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:33.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:33 smithi016 bash[33851]: cluster 2023-10-09T08:35:32.664031+0000 mgr.y (mgr.24355) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:33.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:33 smithi094 bash[27803]: cluster 2023-10-09T08:35:32.664031+0000 mgr.y (mgr.24355) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:34.251 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:35:34 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:34] "GET /metrics HTTP/1.1" 200 191361 "" "Prometheus/2.33.4" 2023-10-09T08:35:35.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:35 smithi094 bash[27803]: cluster 2023-10-09T08:35:34.664520+0000 mgr.y (mgr.24355) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:36.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:35 smithi016 bash[33851]: cluster 2023-10-09T08:35:34.664520+0000 mgr.y (mgr.24355) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:36.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:35 smithi016 bash[27454]: cluster 2023-10-09T08:35:34.664520+0000 mgr.y (mgr.24355) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:37.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:37 smithi094 bash[27803]: cluster 2023-10-09T08:35:36.665619+0000 mgr.y (mgr.24355) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:38.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:37 smithi016 bash[27454]: cluster 2023-10-09T08:35:36.665619+0000 mgr.y (mgr.24355) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:38.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:37 smithi016 bash[33851]: cluster 2023-10-09T08:35:36.665619+0000 mgr.y (mgr.24355) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:39.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:39 smithi094 bash[27803]: cluster 2023-10-09T08:35:38.666082+0000 mgr.y (mgr.24355) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:40.026 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:39 smithi016 bash[27454]: cluster 2023-10-09T08:35:38.666082+0000 mgr.y (mgr.24355) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:40.026 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:39 smithi016 bash[33851]: cluster 2023-10-09T08:35:38.666082+0000 mgr.y (mgr.24355) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:41.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:41 smithi094 bash[27803]: cluster 2023-10-09T08:35:40.667201+0000 mgr.y (mgr.24355) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:41 smithi016 bash[27454]: cluster 2023-10-09T08:35:40.667201+0000 mgr.y (mgr.24355) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:41.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:41 smithi016 bash[33851]: cluster 2023-10-09T08:35:40.667201+0000 mgr.y (mgr.24355) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:42.334 INFO:teuthology.orchestra.run.smithi016.stdout:34359738485 2023-10-09T08:35:42.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:42 smithi094 bash[27803]: audit 2023-10-09T08:35:41.830390+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:35:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:42 smithi094 bash[27803]: audit 2023-10-09T08:35:42.099458+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:35:42.529 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:42 smithi016 bash[27454]: audit 2023-10-09T08:35:41.830390+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:35:42.529 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:42 smithi016 bash[27454]: audit 2023-10-09T08:35:42.099458+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:35:42.530 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:42 smithi016 bash[33851]: audit 2023-10-09T08:35:41.830390+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:35:42.530 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:42 smithi016 bash[33851]: audit 2023-10-09T08:35:42.099458+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:35:42.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:35:42 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:35:43.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:43 smithi094 bash[27803]: audit 2023-10-09T08:35:42.331388+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.16:0/970738050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:35:43.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:43 smithi094 bash[27803]: audit 2023-10-09T08:35:42.379821+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:35:43.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:43 smithi094 bash[27803]: audit 2023-10-09T08:35:42.380735+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:35:43.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:43 smithi094 bash[27803]: audit 2023-10-09T08:35:42.388227+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:35:43.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:43 smithi094 bash[27803]: cluster 2023-10-09T08:35:42.667897+0000 mgr.y (mgr.24355) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[33851]: audit 2023-10-09T08:35:42.331388+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.16:0/970738050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[33851]: audit 2023-10-09T08:35:42.379821+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[33851]: audit 2023-10-09T08:35:42.380735+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[33851]: audit 2023-10-09T08:35:42.388227+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[33851]: cluster 2023-10-09T08:35:42.667897+0000 mgr.y (mgr.24355) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[27454]: audit 2023-10-09T08:35:42.331388+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.16:0/970738050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[27454]: audit 2023-10-09T08:35:42.379821+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:35:43.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[27454]: audit 2023-10-09T08:35:42.380735+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:35:43.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[27454]: audit 2023-10-09T08:35:42.388227+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:35:43.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:43 smithi016 bash[27454]: cluster 2023-10-09T08:35:42.667897+0000 mgr.y (mgr.24355) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:43.651 INFO:teuthology.orchestra.run.smithi016.stdout:55834574955 2023-10-09T08:35:44.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:44 smithi016 bash[33851]: audit 2023-10-09T08:35:43.647942+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.16:0/2004985483' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:35:44.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:44 smithi016 bash[27454]: audit 2023-10-09T08:35:43.647942+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.16:0/2004985483' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:35:44.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:35:44 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:44] "GET /metrics HTTP/1.1" 200 191384 "" "Prometheus/2.33.4" 2023-10-09T08:35:44.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:44 smithi094 bash[27803]: audit 2023-10-09T08:35:43.647942+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.16:0/2004985483' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:35:45.174 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738481 got 34359738485 for osd.0 2023-10-09T08:35:45.175 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:45.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:45 smithi016 bash[27454]: cluster 2023-10-09T08:35:44.668358+0000 mgr.y (mgr.24355) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:45.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:45 smithi016 bash[33851]: cluster 2023-10-09T08:35:44.668358+0000 mgr.y (mgr.24355) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:45 smithi094 bash[27803]: cluster 2023-10-09T08:35:44.668358+0000 mgr.y (mgr.24355) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:46.143 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574951 got 55834574955 for osd.1 2023-10-09T08:35:46.144 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:46.424 INFO:teuthology.orchestra.run.smithi016.stdout:107374182490 2023-10-09T08:35:46.569 INFO:teuthology.orchestra.run.smithi016.stdout:77309411426 2023-10-09T08:35:46.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:46 smithi094 bash[27803]: audit 2023-10-09T08:35:46.420948+0000 mon.a (mon.0) 672 : audit [DBG] from='client.? 172.21.15.16:0/2162913321' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:35:46.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:46 smithi016 bash[27454]: audit 2023-10-09T08:35:46.420948+0000 mon.a (mon.0) 672 : audit [DBG] from='client.? 172.21.15.16:0/2162913321' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:35:46.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:46 smithi016 bash[33851]: audit 2023-10-09T08:35:46.420948+0000 mon.a (mon.0) 672 : audit [DBG] from='client.? 172.21.15.16:0/2162913321' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:35:46.796 INFO:teuthology.orchestra.run.smithi016.stdout:154618822729 2023-10-09T08:35:47.399 INFO:teuthology.orchestra.run.smithi016.stdout:180388626495 2023-10-09T08:35:47.634 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[27454]: audit 2023-10-09T08:35:46.565608+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.16:0/1221538640' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:35:47.634 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[27454]: cluster 2023-10-09T08:35:46.669348+0000 mgr.y (mgr.24355) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:47.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[27454]: audit 2023-10-09T08:35:46.793544+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.16:0/1256951704' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:35:47.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[27454]: audit 2023-10-09T08:35:47.396165+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.16:0/3696991286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:35:47.635 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[33851]: audit 2023-10-09T08:35:46.565608+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.16:0/1221538640' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:35:47.635 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[33851]: cluster 2023-10-09T08:35:46.669348+0000 mgr.y (mgr.24355) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:47.635 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[33851]: audit 2023-10-09T08:35:46.793544+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.16:0/1256951704' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:35:47.635 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:47 smithi016 bash[33851]: audit 2023-10-09T08:35:47.396165+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.16:0/3696991286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:35:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:47 smithi094 bash[27803]: audit 2023-10-09T08:35:46.565608+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.16:0/1221538640' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:35:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:47 smithi094 bash[27803]: cluster 2023-10-09T08:35:46.669348+0000 mgr.y (mgr.24355) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:47.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:47 smithi094 bash[27803]: audit 2023-10-09T08:35:46.793544+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.16:0/1256951704' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:35:47.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:47 smithi094 bash[27803]: audit 2023-10-09T08:35:47.396165+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.16:0/3696991286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:35:48.041 INFO:teuthology.orchestra.run.smithi016.stdout:206158430263 2023-10-09T08:35:48.427 INFO:teuthology.orchestra.run.smithi016.stdout:128849018962 2023-10-09T08:35:48.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:48 smithi094 bash[27803]: audit 2023-10-09T08:35:48.038211+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.16:0/1636476138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:35:48.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:48 smithi094 bash[27803]: audit 2023-10-09T08:35:48.423795+0000 mon.a (mon.0) 673 : audit [DBG] from='client.? 172.21.15.16:0/2741156440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:35:48.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:48 smithi016 bash[33851]: audit 2023-10-09T08:35:48.038211+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.16:0/1636476138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:35:48.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:48 smithi016 bash[33851]: audit 2023-10-09T08:35:48.423795+0000 mon.a (mon.0) 673 : audit [DBG] from='client.? 172.21.15.16:0/2741156440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:35:48.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:48 smithi016 bash[27454]: audit 2023-10-09T08:35:48.038211+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.16:0/1636476138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:35:48.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:48 smithi016 bash[27454]: audit 2023-10-09T08:35:48.423795+0000 mon.a (mon.0) 673 : audit [DBG] from='client.? 172.21.15.16:0/2741156440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:35:49.377 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182486 got 107374182490 for osd.3 2023-10-09T08:35:49.377 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:49.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:49 smithi016 bash[27454]: cluster 2023-10-09T08:35:48.670040+0000 mgr.y (mgr.24355) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:49.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:49 smithi016 bash[33851]: cluster 2023-10-09T08:35:48.670040+0000 mgr.y (mgr.24355) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:49.767 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822725 got 154618822729 for osd.5 2023-10-09T08:35:49.767 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:49.769 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411423 got 77309411426 for osd.2 2023-10-09T08:35:49.769 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:49.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:49 smithi094 bash[27803]: cluster 2023-10-09T08:35:48.670040+0000 mgr.y (mgr.24355) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:50.384 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626491 got 180388626495 for osd.6 2023-10-09T08:35:50.385 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:50.801 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430258 got 206158430263 for osd.7 2023-10-09T08:35:50.801 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:50.844 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018957 got 128849018962 for osd.4 2023-10-09T08:35:50.844 DEBUG:teuthology.parallel:result is None 2023-10-09T08:35:50.844 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T08:35:50.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:35:51.181 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:51 smithi016 bash[33851]: cluster 2023-10-09T08:35:50.671098+0000 mgr.y (mgr.24355) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:51 smithi094 bash[27803]: cluster 2023-10-09T08:35:50.671098+0000 mgr.y (mgr.24355) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:51 smithi016 bash[27454]: cluster 2023-10-09T08:35:50.671098+0000 mgr.y (mgr.24355) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:52.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:35:52 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:35:53.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:53 smithi094 bash[27803]: cluster 2023-10-09T08:35:52.671841+0000 mgr.y (mgr.24355) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:54.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:53 smithi016 bash[27454]: cluster 2023-10-09T08:35:52.671841+0000 mgr.y (mgr.24355) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:54.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:53 smithi016 bash[33851]: cluster 2023-10-09T08:35:52.671841+0000 mgr.y (mgr.24355) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:54.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:35:54 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:35:54] "GET /metrics HTTP/1.1" 200 191380 "" "Prometheus/2.33.4" 2023-10-09T08:35:55.619 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:35:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:55 smithi094 bash[27803]: cluster 2023-10-09T08:35:54.672544+0000 mgr.y (mgr.24355) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:56.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:55 smithi016 bash[27454]: cluster 2023-10-09T08:35:54.672544+0000 mgr.y (mgr.24355) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:56.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:55 smithi016 bash[33851]: cluster 2023-10-09T08:35:54.672544+0000 mgr.y (mgr.24355) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:58.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:57 smithi016 bash[33851]: cluster 2023-10-09T08:35:56.673624+0000 mgr.y (mgr.24355) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:58.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:57 smithi016 bash[27454]: cluster 2023-10-09T08:35:56.673624+0000 mgr.y (mgr.24355) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:58.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:57 smithi094 bash[27803]: cluster 2023-10-09T08:35:56.673624+0000 mgr.y (mgr.24355) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:35:58.903 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:35:58.907 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:35:59.755 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":131,"stamp":"2023-10-09T08:35:58.673931+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381752,"kb_used_data":4984,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579464,"statfs":{"total":767960285184,"available":765521371136,"internally_reserved":0,"allocated":5103616,"data_stored":2165922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005359"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.726503+0000","last_change":"2023-10-09T08:34:27.755384+0000","last_active":"2023-10-09T08:35:04.726503+0000","last_peered":"2023-10-09T08:35:04.726503+0000","last_clean":"2023-10-09T08:35:04.726503+0000","last_became_active":"2023-10-09T08:34:27.755041+0000","last_became_peered":"2023-10-09T08:34:27.755041+0000","last_unstale":"2023-10-09T08:35:04.726503+0000","last_undegraded":"2023-10-09T08:35:04.726503+0000","last_fullsized":"2023-10-09T08:35:04.726503+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721089+0000","last_change":"2023-10-09T08:34:27.728243+0000","last_active":"2023-10-09T08:35:04.721089+0000","last_peered":"2023-10-09T08:35:04.721089+0000","last_clean":"2023-10-09T08:35:04.721089+0000","last_became_active":"2023-10-09T08:34:27.727645+0000","last_became_peered":"2023-10-09T08:34:27.727645+0000","last_unstale":"2023-10-09T08:35:04.721089+0000","last_undegraded":"2023-10-09T08:35:04.721089+0000","last_fullsized":"2023-10-09T08:35:04.721089+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724276+0000","last_change":"2023-10-09T08:34:28.994471+0000","last_active":"2023-10-09T08:35:04.724276+0000","last_peered":"2023-10-09T08:35:04.724276+0000","last_clean":"2023-10-09T08:35:04.724276+0000","last_became_active":"2023-10-09T08:34:28.993827+0000","last_became_peered":"2023-10-09T08:34:28.993827+0000","last_unstale":"2023-10-09T08:35:04.724276+0000","last_undegraded":"2023-10-09T08:35:04.724276+0000","last_fullsized":"2023-10-09T08:35:04.724276+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723730+0000","last_change":"2023-10-09T08:34:27.735557+0000","last_active":"2023-10-09T08:35:04.723730+0000","last_peered":"2023-10-09T08:35:04.723730+0000","last_clean":"2023-10-09T08:35:04.723730+0000","last_became_active":"2023-10-09T08:34:27.734936+0000","last_became_peered":"2023-10-09T08:34:27.734936+0000","last_unstale":"2023-10-09T08:35:04.723730+0000","last_undegraded":"2023-10-09T08:35:04.723730+0000","last_fullsized":"2023-10-09T08:35:04.723730+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.726821+0000","last_change":"2023-10-09T08:34:27.757258+0000","last_active":"2023-10-09T08:35:04.726821+0000","last_peered":"2023-10-09T08:35:04.726821+0000","last_clean":"2023-10-09T08:35:04.726821+0000","last_became_active":"2023-10-09T08:34:27.756970+0000","last_became_peered":"2023-10-09T08:34:27.756970+0000","last_unstale":"2023-10-09T08:35:04.726821+0000","last_undegraded":"2023-10-09T08:35:04.726821+0000","last_fullsized":"2023-10-09T08:35:04.726821+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723642+0000","last_change":"2023-10-09T08:34:27.739113+0000","last_active":"2023-10-09T08:35:04.723642+0000","last_peered":"2023-10-09T08:35:04.723642+0000","last_clean":"2023-10-09T08:35:04.723642+0000","last_became_active":"2023-10-09T08:34:27.738645+0000","last_became_peered":"2023-10-09T08:34:27.738645+0000","last_unstale":"2023-10-09T08:35:04.723642+0000","last_undegraded":"2023-10-09T08:35:04.723642+0000","last_fullsized":"2023-10-09T08:35:04.723642+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727529+0000","last_change":"2023-10-09T08:34:27.757165+0000","last_active":"2023-10-09T08:35:04.727529+0000","last_peered":"2023-10-09T08:35:04.727529+0000","last_clean":"2023-10-09T08:35:04.727529+0000","last_became_active":"2023-10-09T08:34:27.756898+0000","last_became_peered":"2023-10-09T08:34:27.756898+0000","last_unstale":"2023-10-09T08:35:04.727529+0000","last_undegraded":"2023-10-09T08:35:04.727529+0000","last_fullsized":"2023-10-09T08:35:04.727529+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723398+0000","last_change":"2023-10-09T08:34:27.733839+0000","last_active":"2023-10-09T08:35:04.723398+0000","last_peered":"2023-10-09T08:35:04.723398+0000","last_clean":"2023-10-09T08:35:04.723398+0000","last_became_active":"2023-10-09T08:34:27.733406+0000","last_became_peered":"2023-10-09T08:34:27.733406+0000","last_unstale":"2023-10-09T08:35:04.723398+0000","last_undegraded":"2023-10-09T08:35:04.723398+0000","last_fullsized":"2023-10-09T08:35:04.723398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727911+0000","last_change":"2023-10-09T08:34:27.761231+0000","last_active":"2023-10-09T08:35:04.727911+0000","last_peered":"2023-10-09T08:35:04.727911+0000","last_clean":"2023-10-09T08:35:04.727911+0000","last_became_active":"2023-10-09T08:34:27.760876+0000","last_became_peered":"2023-10-09T08:34:27.760876+0000","last_unstale":"2023-10-09T08:35:04.727911+0000","last_undegraded":"2023-10-09T08:35:04.727911+0000","last_fullsized":"2023-10-09T08:35:04.727911+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721556+0000","last_change":"2023-10-09T08:34:27.722989+0000","last_active":"2023-10-09T08:35:04.721556+0000","last_peered":"2023-10-09T08:35:04.721556+0000","last_clean":"2023-10-09T08:35:04.721556+0000","last_became_active":"2023-10-09T08:34:27.722547+0000","last_became_peered":"2023-10-09T08:34:27.722547+0000","last_unstale":"2023-10-09T08:35:04.721556+0000","last_undegraded":"2023-10-09T08:35:04.721556+0000","last_fullsized":"2023-10-09T08:35:04.721556+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727946+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:35:04.727946+0000","last_peered":"2023-10-09T08:35:04.727946+0000","last_clean":"2023-10-09T08:35:04.727946+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:35:04.727946+0000","last_undegraded":"2023-10-09T08:35:04.727946+0000","last_fullsized":"2023-10-09T08:35:04.727946+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721405+0000","last_change":"2023-10-09T08:34:27.724012+0000","last_active":"2023-10-09T08:35:04.721405+0000","last_peered":"2023-10-09T08:35:04.721405+0000","last_clean":"2023-10-09T08:35:04.721405+0000","last_became_active":"2023-10-09T08:34:27.723578+0000","last_became_peered":"2023-10-09T08:34:27.723578+0000","last_unstale":"2023-10-09T08:35:04.721405+0000","last_undegraded":"2023-10-09T08:35:04.721405+0000","last_fullsized":"2023-10-09T08:35:04.721405+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.732190+0000","last_change":"2023-10-09T08:32:37.686368+0000","last_active":"2023-10-09T08:35:04.732190+0000","last_peered":"2023-10-09T08:35:04.732190+0000","last_clean":"2023-10-09T08:35:04.732190+0000","last_became_active":"2023-10-09T08:32:37.685786+0000","last_became_peered":"2023-10-09T08:32:37.685786+0000","last_unstale":"2023-10-09T08:35:04.732190+0000","last_undegraded":"2023-10-09T08:35:04.732190+0000","last_fullsized":"2023-10-09T08:35:04.732190+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723908+0000","last_change":"2023-10-09T08:34:29.489088+0000","last_active":"2023-10-09T08:35:04.723908+0000","last_peered":"2023-10-09T08:35:04.723908+0000","last_clean":"2023-10-09T08:35:04.723908+0000","last_became_active":"2023-10-09T08:34:29.488737+0000","last_became_peered":"2023-10-09T08:34:29.488737+0000","last_unstale":"2023-10-09T08:35:04.723908+0000","last_undegraded":"2023-10-09T08:35:04.723908+0000","last_fullsized":"2023-10-09T08:35:04.723908+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729527+0000","last_change":"2023-10-09T08:32:37.776388+0000","last_active":"2023-10-09T08:35:04.729527+0000","last_peered":"2023-10-09T08:35:04.729527+0000","last_clean":"2023-10-09T08:35:04.729527+0000","last_became_active":"2023-10-09T08:32:37.775915+0000","last_became_peered":"2023-10-09T08:32:37.775915+0000","last_unstale":"2023-10-09T08:35:04.729527+0000","last_undegraded":"2023-10-09T08:35:04.729527+0000","last_fullsized":"2023-10-09T08:35:04.729527+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724424+0000","last_change":"2023-10-09T08:34:27.737193+0000","last_active":"2023-10-09T08:35:04.724424+0000","last_peered":"2023-10-09T08:35:04.724424+0000","last_clean":"2023-10-09T08:35:04.724424+0000","last_became_active":"2023-10-09T08:34:27.736818+0000","last_became_peered":"2023-10-09T08:34:27.736818+0000","last_unstale":"2023-10-09T08:35:04.724424+0000","last_undegraded":"2023-10-09T08:35:04.724424+0000","last_fullsized":"2023-10-09T08:35:04.724424+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721287+0000","last_change":"2023-10-09T08:34:27.859401+0000","last_active":"2023-10-09T08:35:04.721287+0000","last_peered":"2023-10-09T08:35:04.721287+0000","last_clean":"2023-10-09T08:35:04.721287+0000","last_became_active":"2023-10-09T08:34:27.858669+0000","last_became_peered":"2023-10-09T08:34:27.858669+0000","last_unstale":"2023-10-09T08:35:04.721287+0000","last_undegraded":"2023-10-09T08:35:04.721287+0000","last_fullsized":"2023-10-09T08:35:04.721287+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.730112+0000","last_change":"2023-10-09T08:34:27.729156+0000","last_active":"2023-10-09T08:35:04.730112+0000","last_peered":"2023-10-09T08:35:04.730112+0000","last_clean":"2023-10-09T08:35:04.730112+0000","last_became_active":"2023-10-09T08:34:27.728872+0000","last_became_peered":"2023-10-09T08:34:27.728872+0000","last_unstale":"2023-10-09T08:35:04.730112+0000","last_undegraded":"2023-10-09T08:35:04.730112+0000","last_fullsized":"2023-10-09T08:35:04.730112+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733843+0000","last_change":"2023-10-09T08:32:37.777840+0000","last_active":"2023-10-09T08:35:04.733843+0000","last_peered":"2023-10-09T08:35:04.733843+0000","last_clean":"2023-10-09T08:35:04.733843+0000","last_became_active":"2023-10-09T08:32:37.777235+0000","last_became_peered":"2023-10-09T08:32:37.777235+0000","last_unstale":"2023-10-09T08:35:04.733843+0000","last_undegraded":"2023-10-09T08:35:04.733843+0000","last_fullsized":"2023-10-09T08:35:04.733843+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721480+0000","last_change":"2023-10-09T08:34:27.759741+0000","last_active":"2023-10-09T08:35:04.721480+0000","last_peered":"2023-10-09T08:35:04.721480+0000","last_clean":"2023-10-09T08:35:04.721480+0000","last_became_active":"2023-10-09T08:34:27.759413+0000","last_became_peered":"2023-10-09T08:34:27.759413+0000","last_unstale":"2023-10-09T08:35:04.721480+0000","last_undegraded":"2023-10-09T08:35:04.721480+0000","last_fullsized":"2023-10-09T08:35:04.721480+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721594+0000","last_change":"2023-10-09T08:34:27.734843+0000","last_active":"2023-10-09T08:35:04.721594+0000","last_peered":"2023-10-09T08:35:04.721594+0000","last_clean":"2023-10-09T08:35:04.721594+0000","last_became_active":"2023-10-09T08:34:27.734497+0000","last_became_peered":"2023-10-09T08:34:27.734497+0000","last_unstale":"2023-10-09T08:35:04.721594+0000","last_undegraded":"2023-10-09T08:35:04.721594+0000","last_fullsized":"2023-10-09T08:35:04.721594+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733559+0000","last_change":"2023-10-09T08:32:37.689006+0000","last_active":"2023-10-09T08:35:04.733559+0000","last_peered":"2023-10-09T08:35:04.733559+0000","last_clean":"2023-10-09T08:35:04.733559+0000","last_became_active":"2023-10-09T08:32:37.688629+0000","last_became_peered":"2023-10-09T08:32:37.688629+0000","last_unstale":"2023-10-09T08:35:04.733559+0000","last_undegraded":"2023-10-09T08:35:04.733559+0000","last_fullsized":"2023-10-09T08:35:04.733559+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.732105+0000","last_change":"2023-10-09T08:34:27.860287+0000","last_active":"2023-10-09T08:35:04.732105+0000","last_peered":"2023-10-09T08:35:04.732105+0000","last_clean":"2023-10-09T08:35:04.732105+0000","last_became_active":"2023-10-09T08:34:27.859851+0000","last_became_peered":"2023-10-09T08:34:27.859851+0000","last_unstale":"2023-10-09T08:35:04.732105+0000","last_undegraded":"2023-10-09T08:35:04.732105+0000","last_fullsized":"2023-10-09T08:35:04.732105+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.722908+0000","last_change":"2023-10-09T08:34:27.738058+0000","last_active":"2023-10-09T08:35:04.722908+0000","last_peered":"2023-10-09T08:35:04.722908+0000","last_clean":"2023-10-09T08:35:04.722908+0000","last_became_active":"2023-10-09T08:34:27.737006+0000","last_became_peered":"2023-10-09T08:34:27.737006+0000","last_unstale":"2023-10-09T08:35:04.722908+0000","last_undegraded":"2023-10-09T08:35:04.722908+0000","last_fullsized":"2023-10-09T08:35:04.722908+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733498+0000","last_change":"2023-10-09T08:31:31.839687+0000","last_active":"2023-10-09T08:35:04.733498+0000","last_peered":"2023-10-09T08:35:04.733498+0000","last_clean":"2023-10-09T08:35:04.733498+0000","last_became_active":"2023-10-09T08:31:31.838733+0000","last_became_peered":"2023-10-09T08:31:31.838733+0000","last_unstale":"2023-10-09T08:35:04.733498+0000","last_undegraded":"2023-10-09T08:35:04.733498+0000","last_fullsized":"2023-10-09T08:35:04.733498+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723201+0000","last_change":"2023-10-09T08:32:37.686289+0000","last_active":"2023-10-09T08:35:04.723201+0000","last_peered":"2023-10-09T08:35:04.723201+0000","last_clean":"2023-10-09T08:35:04.723201+0000","last_became_active":"2023-10-09T08:32:37.685665+0000","last_became_peered":"2023-10-09T08:32:37.685665+0000","last_unstale":"2023-10-09T08:35:04.723201+0000","last_undegraded":"2023-10-09T08:35:04.723201+0000","last_fullsized":"2023-10-09T08:35:04.723201+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729590+0000","last_change":"2023-10-09T08:34:27.758762+0000","last_active":"2023-10-09T08:35:04.729590+0000","last_peered":"2023-10-09T08:35:04.729590+0000","last_clean":"2023-10-09T08:35:04.729590+0000","last_became_active":"2023-10-09T08:34:27.758358+0000","last_became_peered":"2023-10-09T08:34:27.758358+0000","last_unstale":"2023-10-09T08:35:04.729590+0000","last_undegraded":"2023-10-09T08:35:04.729590+0000","last_fullsized":"2023-10-09T08:35:04.729590+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728562+0000","last_change":"2023-10-09T08:34:27.724082+0000","last_active":"2023-10-09T08:35:04.728562+0000","last_peered":"2023-10-09T08:35:04.728562+0000","last_clean":"2023-10-09T08:35:04.728562+0000","last_became_active":"2023-10-09T08:34:27.723719+0000","last_became_peered":"2023-10-09T08:34:27.723719+0000","last_unstale":"2023-10-09T08:35:04.728562+0000","last_undegraded":"2023-10-09T08:35:04.728562+0000","last_fullsized":"2023-10-09T08:35:04.728562+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727912+0000","last_change":"2023-10-09T08:32:37.688803+0000","last_active":"2023-10-09T08:35:04.727912+0000","last_peered":"2023-10-09T08:35:04.727912+0000","last_clean":"2023-10-09T08:35:04.727912+0000","last_became_active":"2023-10-09T08:32:37.688433+0000","last_became_peered":"2023-10-09T08:32:37.688433+0000","last_unstale":"2023-10-09T08:35:04.727912+0000","last_undegraded":"2023-10-09T08:35:04.727912+0000","last_fullsized":"2023-10-09T08:35:04.727912+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T08:35:05.732328+0000","last_change":"2023-10-09T08:35:05.732328+0000","last_active":"2023-10-09T08:35:05.732328+0000","last_peered":"2023-10-09T08:35:05.732328+0000","last_clean":"2023-10-09T08:35:05.732328+0000","last_became_active":"2023-10-09T08:35:05.731634+0000","last_became_peered":"2023-10-09T08:35:05.731634+0000","last_unstale":"2023-10-09T08:35:05.732328+0000","last_undegraded":"2023-10-09T08:35:05.732328+0000","last_fullsized":"2023-10-09T08:35:05.732328+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727247+0000","last_change":"2023-10-09T08:34:27.730834+0000","last_active":"2023-10-09T08:35:04.727247+0000","last_peered":"2023-10-09T08:35:04.727247+0000","last_clean":"2023-10-09T08:35:04.727247+0000","last_became_active":"2023-10-09T08:34:27.730149+0000","last_became_peered":"2023-10-09T08:34:27.730149+0000","last_unstale":"2023-10-09T08:35:04.727247+0000","last_undegraded":"2023-10-09T08:35:04.727247+0000","last_fullsized":"2023-10-09T08:35:04.727247+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723154+0000","last_change":"2023-10-09T08:32:37.682306+0000","last_active":"2023-10-09T08:35:04.723154+0000","last_peered":"2023-10-09T08:35:04.723154+0000","last_clean":"2023-10-09T08:35:04.723154+0000","last_became_active":"2023-10-09T08:32:37.681879+0000","last_became_peered":"2023-10-09T08:32:37.681879+0000","last_unstale":"2023-10-09T08:35:04.723154+0000","last_undegraded":"2023-10-09T08:35:04.723154+0000","last_fullsized":"2023-10-09T08:35:04.723154+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728676+0000","last_change":"2023-10-09T08:34:27.762067+0000","last_active":"2023-10-09T08:35:04.728676+0000","last_peered":"2023-10-09T08:35:04.728676+0000","last_clean":"2023-10-09T08:35:04.728676+0000","last_became_active":"2023-10-09T08:34:27.761669+0000","last_became_peered":"2023-10-09T08:34:27.761669+0000","last_unstale":"2023-10-09T08:35:04.728676+0000","last_undegraded":"2023-10-09T08:35:04.728676+0000","last_fullsized":"2023-10-09T08:35:04.728676+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723557+0000","last_change":"2023-10-09T08:34:27.735008+0000","last_active":"2023-10-09T08:35:04.723557+0000","last_peered":"2023-10-09T08:35:04.723557+0000","last_clean":"2023-10-09T08:35:04.723557+0000","last_became_active":"2023-10-09T08:34:27.734642+0000","last_became_peered":"2023-10-09T08:34:27.734642+0000","last_unstale":"2023-10-09T08:35:04.723557+0000","last_undegraded":"2023-10-09T08:35:04.723557+0000","last_fullsized":"2023-10-09T08:35:04.723557+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.730035+0000","last_change":"2023-10-09T08:32:37.688617+0000","last_active":"2023-10-09T08:35:04.730035+0000","last_peered":"2023-10-09T08:35:04.730035+0000","last_clean":"2023-10-09T08:35:04.730035+0000","last_became_active":"2023-10-09T08:32:37.688218+0000","last_became_peered":"2023-10-09T08:32:37.688218+0000","last_unstale":"2023-10-09T08:35:04.730035+0000","last_undegraded":"2023-10-09T08:35:04.730035+0000","last_fullsized":"2023-10-09T08:35:04.730035+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729475+0000","last_change":"2023-10-09T08:34:27.763824+0000","last_active":"2023-10-09T08:35:04.729475+0000","last_peered":"2023-10-09T08:35:04.729475+0000","last_clean":"2023-10-09T08:35:04.729475+0000","last_became_active":"2023-10-09T08:34:27.763343+0000","last_became_peered":"2023-10-09T08:34:27.763343+0000","last_unstale":"2023-10-09T08:35:04.729475+0000","last_undegraded":"2023-10-09T08:35:04.729475+0000","last_fullsized":"2023-10-09T08:35:04.729475+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724403+0000","last_change":"2023-10-09T08:34:27.738097+0000","last_active":"2023-10-09T08:35:04.724403+0000","last_peered":"2023-10-09T08:35:04.724403+0000","last_clean":"2023-10-09T08:35:04.724403+0000","last_became_active":"2023-10-09T08:34:27.737587+0000","last_became_peered":"2023-10-09T08:34:27.737587+0000","last_unstale":"2023-10-09T08:35:04.724403+0000","last_undegraded":"2023-10-09T08:35:04.724403+0000","last_fullsized":"2023-10-09T08:35:04.724403+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728202+0000","last_change":"2023-10-09T08:34:27.755514+0000","last_active":"2023-10-09T08:35:04.728202+0000","last_peered":"2023-10-09T08:35:04.728202+0000","last_clean":"2023-10-09T08:35:04.728202+0000","last_became_active":"2023-10-09T08:34:27.754976+0000","last_became_peered":"2023-10-09T08:34:27.754976+0000","last_unstale":"2023-10-09T08:35:04.728202+0000","last_undegraded":"2023-10-09T08:35:04.728202+0000","last_fullsized":"2023-10-09T08:35:04.728202+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727819+0000","last_change":"2023-10-09T08:34:27.727750+0000","last_active":"2023-10-09T08:35:04.727819+0000","last_peered":"2023-10-09T08:35:04.727819+0000","last_clean":"2023-10-09T08:35:04.727819+0000","last_became_active":"2023-10-09T08:34:27.727113+0000","last_became_peered":"2023-10-09T08:34:27.727113+0000","last_unstale":"2023-10-09T08:35:04.727819+0000","last_undegraded":"2023-10-09T08:35:04.727819+0000","last_fullsized":"2023-10-09T08:35:04.727819+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727621+0000","last_change":"2023-10-09T08:34:27.759549+0000","last_active":"2023-10-09T08:35:04.727621+0000","last_peered":"2023-10-09T08:35:04.727621+0000","last_clean":"2023-10-09T08:35:04.727621+0000","last_became_active":"2023-10-09T08:34:27.759253+0000","last_became_peered":"2023-10-09T08:34:27.759253+0000","last_unstale":"2023-10-09T08:35:04.727621+0000","last_undegraded":"2023-10-09T08:35:04.727621+0000","last_fullsized":"2023-10-09T08:35:04.727621+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.732287+0000","last_change":"2023-10-09T08:34:27.722692+0000","last_active":"2023-10-09T08:35:04.732287+0000","last_peered":"2023-10-09T08:35:04.732287+0000","last_clean":"2023-10-09T08:35:04.732287+0000","last_became_active":"2023-10-09T08:34:27.722326+0000","last_became_peered":"2023-10-09T08:34:27.722326+0000","last_unstale":"2023-10-09T08:35:04.732287+0000","last_undegraded":"2023-10-09T08:35:04.732287+0000","last_fullsized":"2023-10-09T08:35:04.732287+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727419+0000","last_change":"2023-10-09T08:34:27.758797+0000","last_active":"2023-10-09T08:35:04.727419+0000","last_peered":"2023-10-09T08:35:04.727419+0000","last_clean":"2023-10-09T08:35:04.727419+0000","last_became_active":"2023-10-09T08:34:27.758390+0000","last_became_peered":"2023-10-09T08:34:27.758390+0000","last_unstale":"2023-10-09T08:35:04.727419+0000","last_undegraded":"2023-10-09T08:35:04.727419+0000","last_fullsized":"2023-10-09T08:35:04.727419+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723336+0000","last_change":"2023-10-09T08:34:27.738199+0000","last_active":"2023-10-09T08:35:04.723336+0000","last_peered":"2023-10-09T08:35:04.723336+0000","last_clean":"2023-10-09T08:35:04.723336+0000","last_became_active":"2023-10-09T08:34:27.737208+0000","last_became_peered":"2023-10-09T08:34:27.737208+0000","last_unstale":"2023-10-09T08:35:04.723336+0000","last_undegraded":"2023-10-09T08:35:04.723336+0000","last_fullsized":"2023-10-09T08:35:04.723336+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721180+0000","last_change":"2023-10-09T08:34:27.752491+0000","last_active":"2023-10-09T08:35:04.721180+0000","last_peered":"2023-10-09T08:35:04.721180+0000","last_clean":"2023-10-09T08:35:04.721180+0000","last_became_active":"2023-10-09T08:34:27.752099+0000","last_became_peered":"2023-10-09T08:34:27.752099+0000","last_unstale":"2023-10-09T08:35:04.721180+0000","last_undegraded":"2023-10-09T08:35:04.721180+0000","last_fullsized":"2023-10-09T08:35:04.721180+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723275+0000","last_change":"2023-10-09T08:34:27.739060+0000","last_active":"2023-10-09T08:35:04.723275+0000","last_peered":"2023-10-09T08:35:04.723275+0000","last_clean":"2023-10-09T08:35:04.723275+0000","last_became_active":"2023-10-09T08:34:27.738704+0000","last_became_peered":"2023-10-09T08:34:27.738704+0000","last_unstale":"2023-10-09T08:35:04.723275+0000","last_undegraded":"2023-10-09T08:35:04.723275+0000","last_fullsized":"2023-10-09T08:35:04.723275+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T08:35:05.733763+0000","last_change":"2023-10-09T08:35:05.733763+0000","last_active":"2023-10-09T08:35:05.733763+0000","last_peered":"2023-10-09T08:35:05.733763+0000","last_clean":"2023-10-09T08:35:05.733763+0000","last_became_active":"2023-10-09T08:35:05.733184+0000","last_became_peered":"2023-10-09T08:35:05.733184+0000","last_unstale":"2023-10-09T08:35:05.733763+0000","last_undegraded":"2023-10-09T08:35:05.733763+0000","last_fullsized":"2023-10-09T08:35:05.733763+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733977+0000","last_change":"2023-10-09T08:34:27.733891+0000","last_active":"2023-10-09T08:35:04.733977+0000","last_peered":"2023-10-09T08:35:04.733977+0000","last_clean":"2023-10-09T08:35:04.733977+0000","last_became_active":"2023-10-09T08:34:27.733379+0000","last_became_peered":"2023-10-09T08:34:27.733379+0000","last_unstale":"2023-10-09T08:35:04.733977+0000","last_undegraded":"2023-10-09T08:35:04.733977+0000","last_fullsized":"2023-10-09T08:35:04.733977+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T08:35:05.734125+0000","last_change":"2023-10-09T08:35:05.734125+0000","last_active":"2023-10-09T08:35:05.734125+0000","last_peered":"2023-10-09T08:35:05.734125+0000","last_clean":"2023-10-09T08:35:05.734125+0000","last_became_active":"2023-10-09T08:35:05.733657+0000","last_became_peered":"2023-10-09T08:35:05.733657+0000","last_unstale":"2023-10-09T08:35:05.734125+0000","last_undegraded":"2023-10-09T08:35:05.734125+0000","last_fullsized":"2023-10-09T08:35:05.734125+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733952+0000","last_change":"2023-10-09T08:34:27.728108+0000","last_active":"2023-10-09T08:35:04.733952+0000","last_peered":"2023-10-09T08:35:04.733952+0000","last_clean":"2023-10-09T08:35:04.733952+0000","last_became_active":"2023-10-09T08:34:27.727139+0000","last_became_peered":"2023-10-09T08:34:27.727139+0000","last_unstale":"2023-10-09T08:35:04.733952+0000","last_undegraded":"2023-10-09T08:35:04.733952+0000","last_fullsized":"2023-10-09T08:35:04.733952+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.722900+0000","last_change":"2023-10-09T08:34:29.490932+0000","last_active":"2023-10-09T08:35:04.722900+0000","last_peered":"2023-10-09T08:35:04.722900+0000","last_clean":"2023-10-09T08:35:04.722900+0000","last_became_active":"2023-10-09T08:34:29.489841+0000","last_became_peered":"2023-10-09T08:34:29.489841+0000","last_unstale":"2023-10-09T08:35:04.722900+0000","last_undegraded":"2023-10-09T08:35:04.722900+0000","last_fullsized":"2023-10-09T08:35:04.722900+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.734081+0000","last_change":"2023-10-09T08:34:27.728079+0000","last_active":"2023-10-09T08:35:04.734081+0000","last_peered":"2023-10-09T08:35:04.734081+0000","last_clean":"2023-10-09T08:35:04.734081+0000","last_became_active":"2023-10-09T08:34:27.727601+0000","last_became_peered":"2023-10-09T08:34:27.727601+0000","last_unstale":"2023-10-09T08:35:04.734081+0000","last_undegraded":"2023-10-09T08:35:04.734081+0000","last_fullsized":"2023-10-09T08:35:04.734081+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724168+0000","last_change":"2023-10-09T08:34:29.490108+0000","last_active":"2023-10-09T08:35:04.724168+0000","last_peered":"2023-10-09T08:35:04.724168+0000","last_clean":"2023-10-09T08:35:04.724168+0000","last_became_active":"2023-10-09T08:34:29.489133+0000","last_became_peered":"2023-10-09T08:34:29.489133+0000","last_unstale":"2023-10-09T08:35:04.724168+0000","last_undegraded":"2023-10-09T08:35:04.724168+0000","last_fullsized":"2023-10-09T08:35:04.724168+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.731477+0000","last_change":"2023-10-09T08:34:27.735071+0000","last_active":"2023-10-09T08:35:04.731477+0000","last_peered":"2023-10-09T08:35:04.731477+0000","last_clean":"2023-10-09T08:35:04.731477+0000","last_became_active":"2023-10-09T08:34:27.734567+0000","last_became_peered":"2023-10-09T08:34:27.734567+0000","last_unstale":"2023-10-09T08:35:04.731477+0000","last_undegraded":"2023-10-09T08:35:04.731477+0000","last_fullsized":"2023-10-09T08:35:04.731477+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721208+0000","last_change":"2023-10-09T08:34:27.859165+0000","last_active":"2023-10-09T08:35:04.721208+0000","last_peered":"2023-10-09T08:35:04.721208+0000","last_clean":"2023-10-09T08:35:04.721208+0000","last_became_active":"2023-10-09T08:34:27.858349+0000","last_became_peered":"2023-10-09T08:34:27.858349+0000","last_unstale":"2023-10-09T08:35:04.721208+0000","last_undegraded":"2023-10-09T08:35:04.721208+0000","last_fullsized":"2023-10-09T08:35:04.721208+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733421+0000","last_change":"2023-10-09T08:34:27.726666+0000","last_active":"2023-10-09T08:35:04.733421+0000","last_peered":"2023-10-09T08:35:04.733421+0000","last_clean":"2023-10-09T08:35:04.733421+0000","last_became_active":"2023-10-09T08:34:27.726189+0000","last_became_peered":"2023-10-09T08:34:27.726189+0000","last_unstale":"2023-10-09T08:35:04.733421+0000","last_undegraded":"2023-10-09T08:35:04.733421+0000","last_fullsized":"2023-10-09T08:35:04.733421+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723442+0000","last_change":"2023-10-09T08:34:29.490096+0000","last_active":"2023-10-09T08:35:04.723442+0000","last_peered":"2023-10-09T08:35:04.723442+0000","last_clean":"2023-10-09T08:35:04.723442+0000","last_became_active":"2023-10-09T08:34:29.489525+0000","last_became_peered":"2023-10-09T08:34:29.489525+0000","last_unstale":"2023-10-09T08:35:04.723442+0000","last_undegraded":"2023-10-09T08:35:04.723442+0000","last_fullsized":"2023-10-09T08:35:04.723442+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721039+0000","last_change":"2023-10-09T08:34:27.732427+0000","last_active":"2023-10-09T08:35:04.721039+0000","last_peered":"2023-10-09T08:35:04.721039+0000","last_clean":"2023-10-09T08:35:04.721039+0000","last_became_active":"2023-10-09T08:34:27.730518+0000","last_became_peered":"2023-10-09T08:34:27.730518+0000","last_unstale":"2023-10-09T08:35:04.721039+0000","last_undegraded":"2023-10-09T08:35:04.721039+0000","last_fullsized":"2023-10-09T08:35:04.721039+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.725322+0000","last_change":"2023-10-09T08:34:29.489335+0000","last_active":"2023-10-09T08:35:04.725322+0000","last_peered":"2023-10-09T08:35:04.725322+0000","last_clean":"2023-10-09T08:35:04.725322+0000","last_became_active":"2023-10-09T08:34:29.488468+0000","last_became_peered":"2023-10-09T08:34:29.488468+0000","last_unstale":"2023-10-09T08:35:04.725322+0000","last_undegraded":"2023-10-09T08:35:04.725322+0000","last_fullsized":"2023-10-09T08:35:04.725322+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733545+0000","last_change":"2023-10-09T08:34:27.731381+0000","last_active":"2023-10-09T08:35:04.733545+0000","last_peered":"2023-10-09T08:35:04.733545+0000","last_clean":"2023-10-09T08:35:04.733545+0000","last_became_active":"2023-10-09T08:34:27.731065+0000","last_became_peered":"2023-10-09T08:34:27.731065+0000","last_unstale":"2023-10-09T08:35:04.733545+0000","last_undegraded":"2023-10-09T08:35:04.733545+0000","last_fullsized":"2023-10-09T08:35:04.733545+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727757+0000","last_change":"2023-10-09T08:34:27.753177+0000","last_active":"2023-10-09T08:35:04.727757+0000","last_peered":"2023-10-09T08:35:04.727757+0000","last_clean":"2023-10-09T08:35:04.727757+0000","last_became_active":"2023-10-09T08:34:27.752856+0000","last_became_peered":"2023-10-09T08:34:27.752856+0000","last_unstale":"2023-10-09T08:35:04.727757+0000","last_undegraded":"2023-10-09T08:35:04.727757+0000","last_fullsized":"2023-10-09T08:35:04.727757+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727356+0000","last_change":"2023-10-09T08:34:27.730961+0000","last_active":"2023-10-09T08:35:04.727356+0000","last_peered":"2023-10-09T08:35:04.727356+0000","last_clean":"2023-10-09T08:35:04.727356+0000","last_became_active":"2023-10-09T08:34:27.730198+0000","last_became_peered":"2023-10-09T08:34:27.730198+0000","last_unstale":"2023-10-09T08:35:04.727356+0000","last_undegraded":"2023-10-09T08:35:04.727356+0000","last_fullsized":"2023-10-09T08:35:04.727356+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728084+0000","last_change":"2023-10-09T08:34:27.761929+0000","last_active":"2023-10-09T08:35:04.728084+0000","last_peered":"2023-10-09T08:35:04.728084+0000","last_clean":"2023-10-09T08:35:04.728084+0000","last_became_active":"2023-10-09T08:34:27.761331+0000","last_became_peered":"2023-10-09T08:34:27.761331+0000","last_unstale":"2023-10-09T08:35:04.728084+0000","last_undegraded":"2023-10-09T08:35:04.728084+0000","last_fullsized":"2023-10-09T08:35:04.728084+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733853+0000","last_change":"2023-10-09T08:34:27.733630+0000","last_active":"2023-10-09T08:35:04.733853+0000","last_peered":"2023-10-09T08:35:04.733853+0000","last_clean":"2023-10-09T08:35:04.733853+0000","last_became_active":"2023-10-09T08:34:27.733125+0000","last_became_peered":"2023-10-09T08:34:27.733125+0000","last_unstale":"2023-10-09T08:35:04.733853+0000","last_undegraded":"2023-10-09T08:35:04.733853+0000","last_fullsized":"2023-10-09T08:35:04.733853+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724448+0000","last_change":"2023-10-09T08:34:29.489279+0000","last_active":"2023-10-09T08:35:04.724448+0000","last_peered":"2023-10-09T08:35:04.724448+0000","last_clean":"2023-10-09T08:35:04.724448+0000","last_became_active":"2023-10-09T08:34:29.488626+0000","last_became_peered":"2023-10-09T08:34:29.488626+0000","last_unstale":"2023-10-09T08:35:04.724448+0000","last_undegraded":"2023-10-09T08:35:04.724448+0000","last_fullsized":"2023-10-09T08:35:04.724448+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723446+0000","last_change":"2023-10-09T08:34:27.733217+0000","last_active":"2023-10-09T08:35:04.723446+0000","last_peered":"2023-10-09T08:35:04.723446+0000","last_clean":"2023-10-09T08:35:04.723446+0000","last_became_active":"2023-10-09T08:34:27.732878+0000","last_became_peered":"2023-10-09T08:34:27.732878+0000","last_unstale":"2023-10-09T08:35:04.723446+0000","last_undegraded":"2023-10-09T08:35:04.723446+0000","last_fullsized":"2023-10-09T08:35:04.723446+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724291+0000","last_change":"2023-10-09T08:34:29.173037+0000","last_active":"2023-10-09T08:35:04.724291+0000","last_peered":"2023-10-09T08:35:04.724291+0000","last_clean":"2023-10-09T08:35:04.724291+0000","last_became_active":"2023-10-09T08:34:29.172274+0000","last_became_peered":"2023-10-09T08:34:29.172274+0000","last_unstale":"2023-10-09T08:35:04.724291+0000","last_undegraded":"2023-10-09T08:35:04.724291+0000","last_fullsized":"2023-10-09T08:35:04.724291+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721232+0000","last_change":"2023-10-09T08:34:27.732056+0000","last_active":"2023-10-09T08:35:04.721232+0000","last_peered":"2023-10-09T08:35:04.721232+0000","last_clean":"2023-10-09T08:35:04.721232+0000","last_became_active":"2023-10-09T08:34:27.731578+0000","last_became_peered":"2023-10-09T08:34:27.731578+0000","last_unstale":"2023-10-09T08:35:04.721232+0000","last_undegraded":"2023-10-09T08:35:04.721232+0000","last_fullsized":"2023-10-09T08:35:04.721232+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.731984+0000","last_change":"2023-10-09T08:34:27.760239+0000","last_active":"2023-10-09T08:35:04.731984+0000","last_peered":"2023-10-09T08:35:04.731984+0000","last_clean":"2023-10-09T08:35:04.731984+0000","last_became_active":"2023-10-09T08:34:27.759819+0000","last_became_peered":"2023-10-09T08:34:27.759819+0000","last_unstale":"2023-10-09T08:35:04.731984+0000","last_undegraded":"2023-10-09T08:35:04.731984+0000","last_fullsized":"2023-10-09T08:35:04.731984+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724430+0000","last_change":"2023-10-09T08:34:27.737609+0000","last_active":"2023-10-09T08:35:04.724430+0000","last_peered":"2023-10-09T08:35:04.724430+0000","last_clean":"2023-10-09T08:35:04.724430+0000","last_became_active":"2023-10-09T08:34:27.737283+0000","last_became_peered":"2023-10-09T08:34:27.737283+0000","last_unstale":"2023-10-09T08:35:04.724430+0000","last_undegraded":"2023-10-09T08:35:04.724430+0000","last_fullsized":"2023-10-09T08:35:04.724430+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721392+0000","last_change":"2023-10-09T08:34:27.860387+0000","last_active":"2023-10-09T08:35:04.721392+0000","last_peered":"2023-10-09T08:35:04.721392+0000","last_clean":"2023-10-09T08:35:04.721392+0000","last_became_active":"2023-10-09T08:34:27.859698+0000","last_became_peered":"2023-10-09T08:34:27.859698+0000","last_unstale":"2023-10-09T08:35:04.721392+0000","last_undegraded":"2023-10-09T08:35:04.721392+0000","last_fullsized":"2023-10-09T08:35:04.721392+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729752+0000","last_change":"2023-10-09T08:34:27.735616+0000","last_active":"2023-10-09T08:35:04.729752+0000","last_peered":"2023-10-09T08:35:04.729752+0000","last_clean":"2023-10-09T08:35:04.729752+0000","last_became_active":"2023-10-09T08:34:27.735286+0000","last_became_peered":"2023-10-09T08:34:27.735286+0000","last_unstale":"2023-10-09T08:35:04.729752+0000","last_undegraded":"2023-10-09T08:35:04.729752+0000","last_fullsized":"2023-10-09T08:35:04.729752+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723322+0000","last_change":"2023-10-09T08:34:29.234364+0000","last_active":"2023-10-09T08:35:04.723322+0000","last_peered":"2023-10-09T08:35:04.723322+0000","last_clean":"2023-10-09T08:35:04.723322+0000","last_became_active":"2023-10-09T08:34:29.233384+0000","last_became_peered":"2023-10-09T08:34:29.233384+0000","last_unstale":"2023-10-09T08:35:04.723322+0000","last_undegraded":"2023-10-09T08:35:04.723322+0000","last_fullsized":"2023-10-09T08:35:04.723322+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.726835+0000","last_change":"2023-10-09T08:34:27.727597+0000","last_active":"2023-10-09T08:35:04.726835+0000","last_peered":"2023-10-09T08:35:04.726835+0000","last_clean":"2023-10-09T08:35:04.726835+0000","last_became_active":"2023-10-09T08:34:27.727176+0000","last_became_peered":"2023-10-09T08:34:27.727176+0000","last_unstale":"2023-10-09T08:35:04.726835+0000","last_undegraded":"2023-10-09T08:35:04.726835+0000","last_fullsized":"2023-10-09T08:35:04.726835+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430265,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.89000000000000001,"15min":0.85799999999999998},"min":{"1min":0.65100000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.2509999999999999,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.97699999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.93100000000000005,"15min":0.93700000000000006},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.81599999999999995}]},{"osd":1,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.93500000000000005,"15min":0.93300000000000005},"min":{"1min":0.68200000000000005,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.1499999999999999,"5min":1.786,"15min":1.786},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.92300000000000004,"15min":0.90100000000000002},"min":{"1min":0.71699999999999997,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.216,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.91100000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.93300000000000005,"15min":0.93899999999999995},"min":{"1min":0.76800000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.123,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.95399999999999996,"15min":0.96399999999999997},"min":{"1min":0.65600000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.359,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.84599999999999997}]},{"osd":3,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91700000000000004,"15min":0.90200000000000002},"min":{"1min":0.71499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.0860000000000001,"5min":1.276,"15min":1.276},"last":1.252},{"interface":"front","average":{"1min":1.038,"5min":1.0049999999999999,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.268,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":1.196}]},{"osd":4,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.88500000000000001,"15min":0.86199999999999999},"min":{"1min":0.61699999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.4279999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.92100000000000004,"15min":0.89100000000000001},"min":{"1min":0.68999999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.22,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":1.399}]},{"osd":5,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":0.98499999999999999,"15min":0.94499999999999995},"min":{"1min":0.69599999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.2509999999999999,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.034,"5min":0.96699999999999997,"15min":0.93300000000000005},"min":{"1min":0.78500000000000003,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.4890000000000001,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.03}]},{"osd":6,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.024,"15min":0.98999999999999999},"min":{"1min":0.72599999999999998,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.5309999999999999,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":1.165},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.0169999999999999,"15min":0.98099999999999998},"min":{"1min":0.81499999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.337}]}]},{"osd":6,"up_from":42,"seq":180388626498,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":271239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.85399999999999998,"15min":0.90200000000000002},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.179,"5min":1.385,"15min":1.385},"last":0.876},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.90800000000000003,"15min":0.91200000000000003},"min":{"1min":0.69299999999999995,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.256,"5min":1.327,"15min":1.327},"last":1.1439999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.873,"15min":0.90200000000000002},"min":{"1min":0.56699999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.1859999999999999,"5min":1.496,"15min":1.496},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.89400000000000002,"15min":0.91000000000000003},"min":{"1min":0.71699999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.236,"5min":1.329,"15min":1.329},"last":0.96899999999999997}]},{"osd":2,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.89000000000000001,"15min":0.93700000000000006},"min":{"1min":0.755,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.1870000000000001,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.93999999999999995,"15min":0.98299999999999998},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.2050000000000001,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.2969999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.93200000000000005,"15min":0.98599999999999999},"min":{"1min":0.73699999999999999,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.1000000000000001,"5min":1.774,"15min":1.774},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.94899999999999995,"15min":1.0229999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.24,"5min":1.476,"15min":1.476},"last":1.0600000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.86099999999999999,"15min":0.84599999999999997},"min":{"1min":0.63200000000000001,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.123,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90500000000000003,"15min":0.91200000000000003},"min":{"1min":0.65800000000000003,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.175,"5min":1.619,"15min":1.619},"last":1.1839999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.95699999999999996,"15min":0.96399999999999997},"min":{"1min":0.71199999999999997,"5min":0.504,"15min":0.504},"max":{"1min":1.1830000000000001,"5min":1.544,"15min":1.544},"last":1.224},{"interface":"front","average":{"1min":1.0369999999999999,"5min":0.96499999999999997,"15min":0.97699999999999998},"min":{"1min":0.80600000000000005,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.218,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.1000000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.98099999999999998,"15min":0.93200000000000005},"min":{"1min":0.78400000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.232,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.357},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.0089999999999999,"15min":0.95399999999999996},"min":{"1min":0.76000000000000001,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":1.252}]}]},{"osd":1,"up_from":13,"seq":55834574958,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.71599999999999997,"15min":0.60699999999999998},"min":{"1min":0.47799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.91600000000000004,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.747},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.78500000000000003,"15min":0.65100000000000002},"min":{"1min":0.58499999999999996,"5min":0.379,"15min":0.379},"max":{"1min":1.0780000000000001,"5min":1.264,"15min":1.264},"last":0.81899999999999995}]},{"osd":2,"last update":"Mon Oct 9 08:34:54 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.79600000000000004,"15min":0.71199999999999997},"min":{"1min":0.38400000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.0249999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.77600000000000002,"15min":0.71899999999999997},"min":{"1min":0.54800000000000004,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.1279999999999999,"5min":1.165,"15min":1.165},"last":0.96499999999999997}]},{"osd":3,"last update":"Mon Oct 9 08:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.82399999999999995,"15min":0.81899999999999995},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.0960000000000001,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.874},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.83799999999999997,"15min":0.80700000000000005},"min":{"1min":0.47699999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.97999999999999998,"5min":1.6739999999999999,"15min":1.6739999999999999},"last":0.64800000000000002}]},{"osd":4,"last update":"Mon Oct 9 08:35:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.004,"15min":0.96699999999999997},"min":{"1min":0.78600000000000003,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.1930000000000001,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.022,"5min":1.0369999999999999,"15min":0.97899999999999998},"min":{"1min":0.745,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.405,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":1.077}]},{"osd":5,"last update":"Mon Oct 9 08:35:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0589999999999999,"15min":1.087},"min":{"1min":0.83999999999999997,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3240000000000001,"5min":1.599,"15min":1.599},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.165,"5min":1.046,"15min":1.04},"min":{"1min":0.85399999999999998,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.4730000000000001,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":1.1819999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.1000000000000001,"15min":1.1559999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.1699999999999999,"5min":1.639,"15min":1.639},"last":1.145},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.0980000000000001,"15min":1.1679999999999999},"min":{"1min":0.70699999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.3879999999999999,"5min":1.724,"15min":1.724},"last":1.0069999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:39 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.127,"15min":1.155},"min":{"1min":0.84199999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.4830000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.095},{"interface":"front","average":{"1min":1.145,"5min":1.171,"15min":1.1910000000000001},"min":{"1min":0.85799999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"max":{"1min":1.4610000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.04}]}]},{"osd":0,"up_from":8,"seq":34359738488,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:35:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.61199999999999999},"min":{"1min":0.57899999999999996,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0580000000000001,"5min":1.373,"15min":1.373},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.80100000000000005,"15min":0.66300000000000003},"min":{"1min":0.58399999999999996,"5min":0.41299999999999998,"15min":0.39000000000000001},"max":{"1min":1.1200000000000001,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.82699999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.78000000000000003,"15min":0.69299999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.242,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.78000000000000003,"15min":0.68600000000000005},"min":{"1min":0.40799999999999997,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.93400000000000005,"5min":1.335,"15min":1.335},"last":0.90800000000000003}]},{"osd":3,"last update":"Mon Oct 9 08:35:30 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.77600000000000002,"15min":0.76700000000000002},"min":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.002,"5min":1.228,"15min":1.228},"last":1.25},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.90400000000000003,"15min":0.85599999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.175,"5min":1.544,"15min":1.544},"last":0.78000000000000003}]},{"osd":4,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.024,"15min":1.022},"min":{"1min":0.69299999999999995,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.458,"5min":1.458,"15min":1.458},"last":1.141},{"interface":"front","average":{"1min":1.115,"5min":1.032,"15min":0.99399999999999999},"min":{"1min":0.78100000000000003,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.5620000000000001,"5min":1.75,"15min":1.75},"last":0.96099999999999997}]},{"osd":5,"last update":"Mon Oct 9 08:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.0580000000000001,"15min":1.0349999999999999},"min":{"1min":0.90900000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.538,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.0860000000000001,"15min":1.0640000000000001},"min":{"1min":0.84399999999999997,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":0.88}]},{"osd":6,"last update":"Mon Oct 9 08:35:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.119,"15min":1.173},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.5920000000000001,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.113,"15min":1.1579999999999999},"min":{"1min":0.71899999999999997,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.6830000000000001,"5min":1.6830000000000001,"15min":1.6830000000000001},"last":1.4099999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:39 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.1559999999999999,"15min":1.2},"min":{"1min":0.69099999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.6459999999999999,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":1.504},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1579999999999999,"15min":1.1890000000000001},"min":{"1min":0.70899999999999996,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.7549999999999999,"5min":1.7549999999999999,"15min":1.7549999999999999},"last":1.214}]}]},{"osd":2,"up_from":18,"seq":77309411429,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.72799999999999998,"15min":0.66500000000000004},"min":{"1min":0.48199999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.96799999999999997,"5min":1.595,"15min":1.595},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.71799999999999997,"15min":0.67700000000000005},"min":{"1min":0.39100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.95399999999999996,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.95399999999999996}]},{"osd":1,"last update":"Mon Oct 9 08:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.73999999999999999,"15min":0.69099999999999995},"min":{"1min":0.42399999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.99299999999999999,"5min":1.708,"15min":1.708},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.83499999999999996,"15min":0.73399999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.0920000000000001,"5min":1.3,"15min":1.3},"last":1.042}]},{"osd":3,"last update":"Mon Oct 9 08:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.81899999999999995,"15min":0.81999999999999995},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.042,"5min":1.349,"15min":1.349},"last":1.208},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.88300000000000001,"15min":0.86099999999999999},"min":{"1min":0.59199999999999997,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.2569999999999999,"5min":1.464,"15min":1.464},"last":0.91200000000000003}]},{"osd":4,"last update":"Mon Oct 9 08:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.99299999999999999,"15min":0.98099999999999998},"min":{"1min":0.70599999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.613,"5min":1.613,"15min":1.613},"last":1.2589999999999999},{"interface":"front","average":{"1min":1.052,"5min":1.0469999999999999,"15min":1.0169999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":1.115}]},{"osd":5,"last update":"Mon Oct 9 08:35:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.0089999999999999,"15min":1.0209999999999999},"min":{"1min":0.72299999999999998,"5min":0.63,"15min":0.63},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":1.071},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.1060000000000001,"15min":1.0680000000000001},"min":{"1min":0.76100000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.9550000000000001,"5min":1.9550000000000001,"15min":1.9550000000000001},"last":1.3}]},{"osd":6,"last update":"Mon Oct 9 08:35:48 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.127,"15min":1.133},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.4159999999999999,"5min":2.1120000000000001,"15min":2.1120000000000001},"last":1.1599999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.1599999999999999,"15min":1.1499999999999999},"min":{"1min":0.87,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.579,"5min":2.1789999999999998,"15min":2.1789999999999998},"last":1.1779999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.117,"15min":1.149},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.371,"5min":2.0310000000000001,"15min":2.0310000000000001},"last":1.125},{"interface":"front","average":{"1min":1.121,"5min":1.206,"15min":1.238},"min":{"1min":0.70399999999999996,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.458,"5min":2.202,"15min":2.202},"last":1.371}]}]},{"osd":3,"up_from":25,"seq":107374182493,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.68600000000000005,"15min":0.67000000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83699999999999997,"5min":1.2,"15min":1.2},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78600000000000003,"15min":0.77700000000000002},"min":{"1min":0.46300000000000002,"5min":0.31,"15min":0.31},"max":{"1min":1.0049999999999999,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.96199999999999997}]},{"osd":1,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.80200000000000005,"15min":0.80900000000000005},"min":{"1min":0.53300000000000003,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0589999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.145},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.72899999999999998,"15min":0.745},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.024,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":1.0649999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.82799999999999996,"15min":0.83899999999999997},"min":{"1min":0.439,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.163,"5min":1.8400000000000001,"15min":1.8400000000000001},"last":1.21},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.79200000000000004,"15min":0.77600000000000002},"min":{"1min":0.503,"5min":0.373,"15min":0.373},"max":{"1min":1.2150000000000001,"5min":1.514,"15min":1.514},"last":1.2410000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:35:19 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.98799999999999999,"15min":0.93500000000000005},"min":{"1min":0.755,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2669999999999999,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.1930000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.0009999999999999,"15min":0.97799999999999998},"min":{"1min":0.73699999999999999,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.4179999999999999,"5min":1.542,"15min":1.542},"last":1.27}]},{"osd":5,"last update":"Mon Oct 9 08:35:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0229999999999999,"15min":1.004},"min":{"1min":0.75700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.4790000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":1.3540000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.042,"15min":1.02},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.4399999999999999,"5min":1.663,"15min":1.663},"last":1.403}]},{"osd":6,"last update":"Mon Oct 9 08:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.087,"15min":1.123},"min":{"1min":0.71299999999999997,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.4390000000000001,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":1.494},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1220000000000001,"15min":1.1499999999999999},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.3839999999999999,"5min":1.905,"15min":1.905},"last":1.468}]},{"osd":7,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.143,"15min":1.165},"min":{"1min":0.78200000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.605,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":1.575},{"interface":"front","average":{"1min":1.081,"5min":1.0880000000000001,"15min":1.1040000000000001},"min":{"1min":0.72899999999999998,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3420000000000001,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":1.4470000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018964,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.85299999999999998,"15min":0.80100000000000005},"min":{"1min":0.51400000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.163,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.90300000000000002,"15min":0.89500000000000002},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2689999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.98099999999999998}]},{"osd":1,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.84299999999999997,"15min":0.80700000000000005},"min":{"1min":0.59399999999999997,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2170000000000001,"5min":1.244,"15min":1.244},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.89000000000000001,"15min":0.878},"min":{"1min":0.57699999999999996,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.90800000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.89700000000000002,"15min":0.85899999999999999},"min":{"1min":0.66500000000000004,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.95299999999999996,"15min":0.90900000000000003},"min":{"1min":0.59799999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.1990000000000001,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":0.81000000000000005}]},{"osd":3,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.95299999999999996,"15min":0.90300000000000002},"min":{"1min":0.69699999999999995,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.2889999999999999,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.91700000000000004,"15min":0.90500000000000003},"min":{"1min":0.627,"5min":0.625,"15min":0.623},"max":{"1min":1.2809999999999999,"5min":1.302,"15min":1.302},"last":1.0449999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:35:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.82799999999999996,"15min":0.73799999999999999},"min":{"1min":0.51400000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.1950000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.008},{"interface":"front","average":{"1min":1.022,"5min":0.89600000000000002,"15min":0.79900000000000004},"min":{"1min":0.69599999999999995,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.306,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.70499999999999996}]},{"osd":6,"last update":"Mon Oct 9 08:35:47 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.89600000000000002,"15min":0.85199999999999998},"min":{"1min":0.79400000000000004,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.3819999999999999,"5min":1.645,"15min":1.645},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.127,"5min":0.98099999999999998,"15min":0.90500000000000003},"min":{"1min":0.81699999999999995,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.1180000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.0269999999999999,"15min":0.98199999999999998},"min":{"1min":0.88600000000000001,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.014,"15min":0.97599999999999998},"min":{"1min":0.91200000000000003,"5min":0.5,"15min":0.5},"max":{"1min":1.454,"5min":1.724,"15min":1.724},"last":1.133}]}]},{"osd":5,"up_from":36,"seq":154618822732,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.85999999999999999,"15min":0.82399999999999995},"min":{"1min":0.60299999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.1970000000000001,"5min":1.238,"15min":1.238},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.872,"15min":0.88600000000000001},"min":{"1min":0.60499999999999998,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.0920000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.98099999999999998}]},{"osd":1,"last update":"Mon Oct 9 08:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88500000000000001,"15min":0.875},"min":{"1min":0.57999999999999996,"5min":0.502,"15min":0.502},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.875,"15min":0.84299999999999997},"min":{"1min":0.77800000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":1.075}]},{"osd":2,"last update":"Mon Oct 9 08:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":0.91100000000000003,"15min":0.879},"min":{"1min":0.75700000000000001,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":2.9950000000000001,"5min":2.9950000000000001,"15min":2.9950000000000001},"last":1.145},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.92300000000000004,"15min":0.90000000000000002},"min":{"1min":0.70999999999999996,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.95499999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.91800000000000004,"15min":0.92900000000000005},"min":{"1min":0.66200000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.125,"5min":1.276,"15min":1.276},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":0.98399999999999999,"15min":0.93600000000000005},"min":{"1min":0.73399999999999999,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":1.117}]},{"osd":4,"last update":"Mon Oct 9 08:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.79900000000000004,"15min":0.72899999999999998},"min":{"1min":0.65100000000000002,"5min":0.439,"15min":0.439},"max":{"1min":1.1639999999999999,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.872},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.85599999999999998,"15min":0.77600000000000002},"min":{"1min":0.71299999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.26,"5min":1.387,"15min":1.387},"last":0.72799999999999998}]},{"osd":6,"last update":"Mon Oct 9 08:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.92900000000000005,"15min":0.88300000000000001},"min":{"1min":0.45200000000000001,"5min":0.439,"15min":0.439},"max":{"1min":1.24,"5min":1.349,"15min":1.349},"last":1.028},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.91900000000000004,"15min":0.84899999999999998},"min":{"1min":0.68700000000000006,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1859999999999999,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.0069999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.96499999999999997,"15min":0.88900000000000001},"min":{"1min":0.79100000000000004,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.29,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.93500000000000005},{"interface":"front","average":{"1min":1.083,"5min":1.004,"15min":0.92200000000000004},"min":{"1min":0.83399999999999996,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.351,"5min":1.421,"15min":1.421},"last":1.1950000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:35:59.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:36:00.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:35:59 smithi016 bash[27454]: cluster 2023-10-09T08:35:58.674341+0000 mgr.y (mgr.24355) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:00.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:35:59 smithi016 bash[33851]: cluster 2023-10-09T08:35:58.674341+0000 mgr.y (mgr.24355) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:00.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:35:59 smithi094 bash[27803]: cluster 2023-10-09T08:35:58.674341+0000 mgr.y (mgr.24355) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:01.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:00 smithi016 bash[27454]: audit 2023-10-09T08:35:58.881448+0000 mgr.y (mgr.24355) 151 : audit [DBG] from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:36:01.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:00 smithi016 bash[33851]: audit 2023-10-09T08:35:58.881448+0000 mgr.y (mgr.24355) 151 : audit [DBG] from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:36:01.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:00 smithi094 bash[27803]: audit 2023-10-09T08:35:58.881448+0000 mgr.y (mgr.24355) 151 : audit [DBG] from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:36:02.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:01 smithi016 bash[27454]: cluster 2023-10-09T08:36:00.675417+0000 mgr.y (mgr.24355) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:02.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:01 smithi016 bash[33851]: cluster 2023-10-09T08:36:00.675417+0000 mgr.y (mgr.24355) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:02.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:01 smithi094 bash[27803]: cluster 2023-10-09T08:36:00.675417+0000 mgr.y (mgr.24355) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:02.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:36:02 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:36:04.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:03 smithi016 bash[27454]: cluster 2023-10-09T08:36:02.676091+0000 mgr.y (mgr.24355) 153 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:04.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:03 smithi016 bash[33851]: cluster 2023-10-09T08:36:02.676091+0000 mgr.y (mgr.24355) 153 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:04.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:03 smithi094 bash[27803]: cluster 2023-10-09T08:36:02.676091+0000 mgr.y (mgr.24355) 153 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:04.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:04 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:04] "GET /metrics HTTP/1.1" 200 191380 "" "Prometheus/2.33.4" 2023-10-09T08:36:04.587 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:36:05.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:05 smithi016 bash[27454]: cluster 2023-10-09T08:36:04.676777+0000 mgr.y (mgr.24355) 154 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:05 smithi094 bash[27803]: cluster 2023-10-09T08:36:04.676777+0000 mgr.y (mgr.24355) 154 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:06.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:05 smithi016 bash[33851]: cluster 2023-10-09T08:36:04.676777+0000 mgr.y (mgr.24355) 154 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:07.607 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:36:07.609 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:36:08.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:07 smithi016 bash[33851]: cluster 2023-10-09T08:36:06.677907+0000 mgr.y (mgr.24355) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:08.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:07 smithi016 bash[27454]: cluster 2023-10-09T08:36:06.677907+0000 mgr.y (mgr.24355) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:07 smithi094 bash[27803]: cluster 2023-10-09T08:36:06.677907+0000 mgr.y (mgr.24355) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:08.445 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":135,"stamp":"2023-10-09T08:36:06.677107+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381752,"kb_used_data":4984,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579464,"statfs":{"total":767960285184,"available":765521371136,"internally_reserved":0,"allocated":5103616,"data_stored":2165922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004880"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.726503+0000","last_change":"2023-10-09T08:34:27.755384+0000","last_active":"2023-10-09T08:35:04.726503+0000","last_peered":"2023-10-09T08:35:04.726503+0000","last_clean":"2023-10-09T08:35:04.726503+0000","last_became_active":"2023-10-09T08:34:27.755041+0000","last_became_peered":"2023-10-09T08:34:27.755041+0000","last_unstale":"2023-10-09T08:35:04.726503+0000","last_undegraded":"2023-10-09T08:35:04.726503+0000","last_fullsized":"2023-10-09T08:35:04.726503+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721089+0000","last_change":"2023-10-09T08:34:27.728243+0000","last_active":"2023-10-09T08:35:04.721089+0000","last_peered":"2023-10-09T08:35:04.721089+0000","last_clean":"2023-10-09T08:35:04.721089+0000","last_became_active":"2023-10-09T08:34:27.727645+0000","last_became_peered":"2023-10-09T08:34:27.727645+0000","last_unstale":"2023-10-09T08:35:04.721089+0000","last_undegraded":"2023-10-09T08:35:04.721089+0000","last_fullsized":"2023-10-09T08:35:04.721089+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724276+0000","last_change":"2023-10-09T08:34:28.994471+0000","last_active":"2023-10-09T08:35:04.724276+0000","last_peered":"2023-10-09T08:35:04.724276+0000","last_clean":"2023-10-09T08:35:04.724276+0000","last_became_active":"2023-10-09T08:34:28.993827+0000","last_became_peered":"2023-10-09T08:34:28.993827+0000","last_unstale":"2023-10-09T08:35:04.724276+0000","last_undegraded":"2023-10-09T08:35:04.724276+0000","last_fullsized":"2023-10-09T08:35:04.724276+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723730+0000","last_change":"2023-10-09T08:34:27.735557+0000","last_active":"2023-10-09T08:35:04.723730+0000","last_peered":"2023-10-09T08:35:04.723730+0000","last_clean":"2023-10-09T08:35:04.723730+0000","last_became_active":"2023-10-09T08:34:27.734936+0000","last_became_peered":"2023-10-09T08:34:27.734936+0000","last_unstale":"2023-10-09T08:35:04.723730+0000","last_undegraded":"2023-10-09T08:35:04.723730+0000","last_fullsized":"2023-10-09T08:35:04.723730+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.726821+0000","last_change":"2023-10-09T08:34:27.757258+0000","last_active":"2023-10-09T08:35:04.726821+0000","last_peered":"2023-10-09T08:35:04.726821+0000","last_clean":"2023-10-09T08:35:04.726821+0000","last_became_active":"2023-10-09T08:34:27.756970+0000","last_became_peered":"2023-10-09T08:34:27.756970+0000","last_unstale":"2023-10-09T08:35:04.726821+0000","last_undegraded":"2023-10-09T08:35:04.726821+0000","last_fullsized":"2023-10-09T08:35:04.726821+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723642+0000","last_change":"2023-10-09T08:34:27.739113+0000","last_active":"2023-10-09T08:35:04.723642+0000","last_peered":"2023-10-09T08:35:04.723642+0000","last_clean":"2023-10-09T08:35:04.723642+0000","last_became_active":"2023-10-09T08:34:27.738645+0000","last_became_peered":"2023-10-09T08:34:27.738645+0000","last_unstale":"2023-10-09T08:35:04.723642+0000","last_undegraded":"2023-10-09T08:35:04.723642+0000","last_fullsized":"2023-10-09T08:35:04.723642+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727529+0000","last_change":"2023-10-09T08:34:27.757165+0000","last_active":"2023-10-09T08:35:04.727529+0000","last_peered":"2023-10-09T08:35:04.727529+0000","last_clean":"2023-10-09T08:35:04.727529+0000","last_became_active":"2023-10-09T08:34:27.756898+0000","last_became_peered":"2023-10-09T08:34:27.756898+0000","last_unstale":"2023-10-09T08:35:04.727529+0000","last_undegraded":"2023-10-09T08:35:04.727529+0000","last_fullsized":"2023-10-09T08:35:04.727529+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723398+0000","last_change":"2023-10-09T08:34:27.733839+0000","last_active":"2023-10-09T08:35:04.723398+0000","last_peered":"2023-10-09T08:35:04.723398+0000","last_clean":"2023-10-09T08:35:04.723398+0000","last_became_active":"2023-10-09T08:34:27.733406+0000","last_became_peered":"2023-10-09T08:34:27.733406+0000","last_unstale":"2023-10-09T08:35:04.723398+0000","last_undegraded":"2023-10-09T08:35:04.723398+0000","last_fullsized":"2023-10-09T08:35:04.723398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727911+0000","last_change":"2023-10-09T08:34:27.761231+0000","last_active":"2023-10-09T08:35:04.727911+0000","last_peered":"2023-10-09T08:35:04.727911+0000","last_clean":"2023-10-09T08:35:04.727911+0000","last_became_active":"2023-10-09T08:34:27.760876+0000","last_became_peered":"2023-10-09T08:34:27.760876+0000","last_unstale":"2023-10-09T08:35:04.727911+0000","last_undegraded":"2023-10-09T08:35:04.727911+0000","last_fullsized":"2023-10-09T08:35:04.727911+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721556+0000","last_change":"2023-10-09T08:34:27.722989+0000","last_active":"2023-10-09T08:35:04.721556+0000","last_peered":"2023-10-09T08:35:04.721556+0000","last_clean":"2023-10-09T08:35:04.721556+0000","last_became_active":"2023-10-09T08:34:27.722547+0000","last_became_peered":"2023-10-09T08:34:27.722547+0000","last_unstale":"2023-10-09T08:35:04.721556+0000","last_undegraded":"2023-10-09T08:35:04.721556+0000","last_fullsized":"2023-10-09T08:35:04.721556+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727946+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:35:04.727946+0000","last_peered":"2023-10-09T08:35:04.727946+0000","last_clean":"2023-10-09T08:35:04.727946+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:35:04.727946+0000","last_undegraded":"2023-10-09T08:35:04.727946+0000","last_fullsized":"2023-10-09T08:35:04.727946+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721405+0000","last_change":"2023-10-09T08:34:27.724012+0000","last_active":"2023-10-09T08:35:04.721405+0000","last_peered":"2023-10-09T08:35:04.721405+0000","last_clean":"2023-10-09T08:35:04.721405+0000","last_became_active":"2023-10-09T08:34:27.723578+0000","last_became_peered":"2023-10-09T08:34:27.723578+0000","last_unstale":"2023-10-09T08:35:04.721405+0000","last_undegraded":"2023-10-09T08:35:04.721405+0000","last_fullsized":"2023-10-09T08:35:04.721405+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.732190+0000","last_change":"2023-10-09T08:32:37.686368+0000","last_active":"2023-10-09T08:35:04.732190+0000","last_peered":"2023-10-09T08:35:04.732190+0000","last_clean":"2023-10-09T08:35:04.732190+0000","last_became_active":"2023-10-09T08:32:37.685786+0000","last_became_peered":"2023-10-09T08:32:37.685786+0000","last_unstale":"2023-10-09T08:35:04.732190+0000","last_undegraded":"2023-10-09T08:35:04.732190+0000","last_fullsized":"2023-10-09T08:35:04.732190+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723908+0000","last_change":"2023-10-09T08:34:29.489088+0000","last_active":"2023-10-09T08:35:04.723908+0000","last_peered":"2023-10-09T08:35:04.723908+0000","last_clean":"2023-10-09T08:35:04.723908+0000","last_became_active":"2023-10-09T08:34:29.488737+0000","last_became_peered":"2023-10-09T08:34:29.488737+0000","last_unstale":"2023-10-09T08:35:04.723908+0000","last_undegraded":"2023-10-09T08:35:04.723908+0000","last_fullsized":"2023-10-09T08:35:04.723908+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729527+0000","last_change":"2023-10-09T08:32:37.776388+0000","last_active":"2023-10-09T08:35:04.729527+0000","last_peered":"2023-10-09T08:35:04.729527+0000","last_clean":"2023-10-09T08:35:04.729527+0000","last_became_active":"2023-10-09T08:32:37.775915+0000","last_became_peered":"2023-10-09T08:32:37.775915+0000","last_unstale":"2023-10-09T08:35:04.729527+0000","last_undegraded":"2023-10-09T08:35:04.729527+0000","last_fullsized":"2023-10-09T08:35:04.729527+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724424+0000","last_change":"2023-10-09T08:34:27.737193+0000","last_active":"2023-10-09T08:35:04.724424+0000","last_peered":"2023-10-09T08:35:04.724424+0000","last_clean":"2023-10-09T08:35:04.724424+0000","last_became_active":"2023-10-09T08:34:27.736818+0000","last_became_peered":"2023-10-09T08:34:27.736818+0000","last_unstale":"2023-10-09T08:35:04.724424+0000","last_undegraded":"2023-10-09T08:35:04.724424+0000","last_fullsized":"2023-10-09T08:35:04.724424+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721287+0000","last_change":"2023-10-09T08:34:27.859401+0000","last_active":"2023-10-09T08:35:04.721287+0000","last_peered":"2023-10-09T08:35:04.721287+0000","last_clean":"2023-10-09T08:35:04.721287+0000","last_became_active":"2023-10-09T08:34:27.858669+0000","last_became_peered":"2023-10-09T08:34:27.858669+0000","last_unstale":"2023-10-09T08:35:04.721287+0000","last_undegraded":"2023-10-09T08:35:04.721287+0000","last_fullsized":"2023-10-09T08:35:04.721287+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.730112+0000","last_change":"2023-10-09T08:34:27.729156+0000","last_active":"2023-10-09T08:35:04.730112+0000","last_peered":"2023-10-09T08:35:04.730112+0000","last_clean":"2023-10-09T08:35:04.730112+0000","last_became_active":"2023-10-09T08:34:27.728872+0000","last_became_peered":"2023-10-09T08:34:27.728872+0000","last_unstale":"2023-10-09T08:35:04.730112+0000","last_undegraded":"2023-10-09T08:35:04.730112+0000","last_fullsized":"2023-10-09T08:35:04.730112+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733843+0000","last_change":"2023-10-09T08:32:37.777840+0000","last_active":"2023-10-09T08:35:04.733843+0000","last_peered":"2023-10-09T08:35:04.733843+0000","last_clean":"2023-10-09T08:35:04.733843+0000","last_became_active":"2023-10-09T08:32:37.777235+0000","last_became_peered":"2023-10-09T08:32:37.777235+0000","last_unstale":"2023-10-09T08:35:04.733843+0000","last_undegraded":"2023-10-09T08:35:04.733843+0000","last_fullsized":"2023-10-09T08:35:04.733843+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721480+0000","last_change":"2023-10-09T08:34:27.759741+0000","last_active":"2023-10-09T08:35:04.721480+0000","last_peered":"2023-10-09T08:35:04.721480+0000","last_clean":"2023-10-09T08:35:04.721480+0000","last_became_active":"2023-10-09T08:34:27.759413+0000","last_became_peered":"2023-10-09T08:34:27.759413+0000","last_unstale":"2023-10-09T08:35:04.721480+0000","last_undegraded":"2023-10-09T08:35:04.721480+0000","last_fullsized":"2023-10-09T08:35:04.721480+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721594+0000","last_change":"2023-10-09T08:34:27.734843+0000","last_active":"2023-10-09T08:35:04.721594+0000","last_peered":"2023-10-09T08:35:04.721594+0000","last_clean":"2023-10-09T08:35:04.721594+0000","last_became_active":"2023-10-09T08:34:27.734497+0000","last_became_peered":"2023-10-09T08:34:27.734497+0000","last_unstale":"2023-10-09T08:35:04.721594+0000","last_undegraded":"2023-10-09T08:35:04.721594+0000","last_fullsized":"2023-10-09T08:35:04.721594+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733559+0000","last_change":"2023-10-09T08:32:37.689006+0000","last_active":"2023-10-09T08:35:04.733559+0000","last_peered":"2023-10-09T08:35:04.733559+0000","last_clean":"2023-10-09T08:35:04.733559+0000","last_became_active":"2023-10-09T08:32:37.688629+0000","last_became_peered":"2023-10-09T08:32:37.688629+0000","last_unstale":"2023-10-09T08:35:04.733559+0000","last_undegraded":"2023-10-09T08:35:04.733559+0000","last_fullsized":"2023-10-09T08:35:04.733559+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.732105+0000","last_change":"2023-10-09T08:34:27.860287+0000","last_active":"2023-10-09T08:35:04.732105+0000","last_peered":"2023-10-09T08:35:04.732105+0000","last_clean":"2023-10-09T08:35:04.732105+0000","last_became_active":"2023-10-09T08:34:27.859851+0000","last_became_peered":"2023-10-09T08:34:27.859851+0000","last_unstale":"2023-10-09T08:35:04.732105+0000","last_undegraded":"2023-10-09T08:35:04.732105+0000","last_fullsized":"2023-10-09T08:35:04.732105+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.722908+0000","last_change":"2023-10-09T08:34:27.738058+0000","last_active":"2023-10-09T08:35:04.722908+0000","last_peered":"2023-10-09T08:35:04.722908+0000","last_clean":"2023-10-09T08:35:04.722908+0000","last_became_active":"2023-10-09T08:34:27.737006+0000","last_became_peered":"2023-10-09T08:34:27.737006+0000","last_unstale":"2023-10-09T08:35:04.722908+0000","last_undegraded":"2023-10-09T08:35:04.722908+0000","last_fullsized":"2023-10-09T08:35:04.722908+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733498+0000","last_change":"2023-10-09T08:31:31.839687+0000","last_active":"2023-10-09T08:35:04.733498+0000","last_peered":"2023-10-09T08:35:04.733498+0000","last_clean":"2023-10-09T08:35:04.733498+0000","last_became_active":"2023-10-09T08:31:31.838733+0000","last_became_peered":"2023-10-09T08:31:31.838733+0000","last_unstale":"2023-10-09T08:35:04.733498+0000","last_undegraded":"2023-10-09T08:35:04.733498+0000","last_fullsized":"2023-10-09T08:35:04.733498+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723201+0000","last_change":"2023-10-09T08:32:37.686289+0000","last_active":"2023-10-09T08:35:04.723201+0000","last_peered":"2023-10-09T08:35:04.723201+0000","last_clean":"2023-10-09T08:35:04.723201+0000","last_became_active":"2023-10-09T08:32:37.685665+0000","last_became_peered":"2023-10-09T08:32:37.685665+0000","last_unstale":"2023-10-09T08:35:04.723201+0000","last_undegraded":"2023-10-09T08:35:04.723201+0000","last_fullsized":"2023-10-09T08:35:04.723201+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729590+0000","last_change":"2023-10-09T08:34:27.758762+0000","last_active":"2023-10-09T08:35:04.729590+0000","last_peered":"2023-10-09T08:35:04.729590+0000","last_clean":"2023-10-09T08:35:04.729590+0000","last_became_active":"2023-10-09T08:34:27.758358+0000","last_became_peered":"2023-10-09T08:34:27.758358+0000","last_unstale":"2023-10-09T08:35:04.729590+0000","last_undegraded":"2023-10-09T08:35:04.729590+0000","last_fullsized":"2023-10-09T08:35:04.729590+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728562+0000","last_change":"2023-10-09T08:34:27.724082+0000","last_active":"2023-10-09T08:35:04.728562+0000","last_peered":"2023-10-09T08:35:04.728562+0000","last_clean":"2023-10-09T08:35:04.728562+0000","last_became_active":"2023-10-09T08:34:27.723719+0000","last_became_peered":"2023-10-09T08:34:27.723719+0000","last_unstale":"2023-10-09T08:35:04.728562+0000","last_undegraded":"2023-10-09T08:35:04.728562+0000","last_fullsized":"2023-10-09T08:35:04.728562+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727912+0000","last_change":"2023-10-09T08:32:37.688803+0000","last_active":"2023-10-09T08:35:04.727912+0000","last_peered":"2023-10-09T08:35:04.727912+0000","last_clean":"2023-10-09T08:35:04.727912+0000","last_became_active":"2023-10-09T08:32:37.688433+0000","last_became_peered":"2023-10-09T08:32:37.688433+0000","last_unstale":"2023-10-09T08:35:04.727912+0000","last_undegraded":"2023-10-09T08:35:04.727912+0000","last_fullsized":"2023-10-09T08:35:04.727912+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T08:35:05.732328+0000","last_change":"2023-10-09T08:35:05.732328+0000","last_active":"2023-10-09T08:35:05.732328+0000","last_peered":"2023-10-09T08:35:05.732328+0000","last_clean":"2023-10-09T08:35:05.732328+0000","last_became_active":"2023-10-09T08:35:05.731634+0000","last_became_peered":"2023-10-09T08:35:05.731634+0000","last_unstale":"2023-10-09T08:35:05.732328+0000","last_undegraded":"2023-10-09T08:35:05.732328+0000","last_fullsized":"2023-10-09T08:35:05.732328+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727247+0000","last_change":"2023-10-09T08:34:27.730834+0000","last_active":"2023-10-09T08:35:04.727247+0000","last_peered":"2023-10-09T08:35:04.727247+0000","last_clean":"2023-10-09T08:35:04.727247+0000","last_became_active":"2023-10-09T08:34:27.730149+0000","last_became_peered":"2023-10-09T08:34:27.730149+0000","last_unstale":"2023-10-09T08:35:04.727247+0000","last_undegraded":"2023-10-09T08:35:04.727247+0000","last_fullsized":"2023-10-09T08:35:04.727247+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723154+0000","last_change":"2023-10-09T08:32:37.682306+0000","last_active":"2023-10-09T08:35:04.723154+0000","last_peered":"2023-10-09T08:35:04.723154+0000","last_clean":"2023-10-09T08:35:04.723154+0000","last_became_active":"2023-10-09T08:32:37.681879+0000","last_became_peered":"2023-10-09T08:32:37.681879+0000","last_unstale":"2023-10-09T08:35:04.723154+0000","last_undegraded":"2023-10-09T08:35:04.723154+0000","last_fullsized":"2023-10-09T08:35:04.723154+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728676+0000","last_change":"2023-10-09T08:34:27.762067+0000","last_active":"2023-10-09T08:35:04.728676+0000","last_peered":"2023-10-09T08:35:04.728676+0000","last_clean":"2023-10-09T08:35:04.728676+0000","last_became_active":"2023-10-09T08:34:27.761669+0000","last_became_peered":"2023-10-09T08:34:27.761669+0000","last_unstale":"2023-10-09T08:35:04.728676+0000","last_undegraded":"2023-10-09T08:35:04.728676+0000","last_fullsized":"2023-10-09T08:35:04.728676+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723557+0000","last_change":"2023-10-09T08:34:27.735008+0000","last_active":"2023-10-09T08:35:04.723557+0000","last_peered":"2023-10-09T08:35:04.723557+0000","last_clean":"2023-10-09T08:35:04.723557+0000","last_became_active":"2023-10-09T08:34:27.734642+0000","last_became_peered":"2023-10-09T08:34:27.734642+0000","last_unstale":"2023-10-09T08:35:04.723557+0000","last_undegraded":"2023-10-09T08:35:04.723557+0000","last_fullsized":"2023-10-09T08:35:04.723557+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.730035+0000","last_change":"2023-10-09T08:32:37.688617+0000","last_active":"2023-10-09T08:35:04.730035+0000","last_peered":"2023-10-09T08:35:04.730035+0000","last_clean":"2023-10-09T08:35:04.730035+0000","last_became_active":"2023-10-09T08:32:37.688218+0000","last_became_peered":"2023-10-09T08:32:37.688218+0000","last_unstale":"2023-10-09T08:35:04.730035+0000","last_undegraded":"2023-10-09T08:35:04.730035+0000","last_fullsized":"2023-10-09T08:35:04.730035+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729475+0000","last_change":"2023-10-09T08:34:27.763824+0000","last_active":"2023-10-09T08:35:04.729475+0000","last_peered":"2023-10-09T08:35:04.729475+0000","last_clean":"2023-10-09T08:35:04.729475+0000","last_became_active":"2023-10-09T08:34:27.763343+0000","last_became_peered":"2023-10-09T08:34:27.763343+0000","last_unstale":"2023-10-09T08:35:04.729475+0000","last_undegraded":"2023-10-09T08:35:04.729475+0000","last_fullsized":"2023-10-09T08:35:04.729475+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724403+0000","last_change":"2023-10-09T08:34:27.738097+0000","last_active":"2023-10-09T08:35:04.724403+0000","last_peered":"2023-10-09T08:35:04.724403+0000","last_clean":"2023-10-09T08:35:04.724403+0000","last_became_active":"2023-10-09T08:34:27.737587+0000","last_became_peered":"2023-10-09T08:34:27.737587+0000","last_unstale":"2023-10-09T08:35:04.724403+0000","last_undegraded":"2023-10-09T08:35:04.724403+0000","last_fullsized":"2023-10-09T08:35:04.724403+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728202+0000","last_change":"2023-10-09T08:34:27.755514+0000","last_active":"2023-10-09T08:35:04.728202+0000","last_peered":"2023-10-09T08:35:04.728202+0000","last_clean":"2023-10-09T08:35:04.728202+0000","last_became_active":"2023-10-09T08:34:27.754976+0000","last_became_peered":"2023-10-09T08:34:27.754976+0000","last_unstale":"2023-10-09T08:35:04.728202+0000","last_undegraded":"2023-10-09T08:35:04.728202+0000","last_fullsized":"2023-10-09T08:35:04.728202+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727819+0000","last_change":"2023-10-09T08:34:27.727750+0000","last_active":"2023-10-09T08:35:04.727819+0000","last_peered":"2023-10-09T08:35:04.727819+0000","last_clean":"2023-10-09T08:35:04.727819+0000","last_became_active":"2023-10-09T08:34:27.727113+0000","last_became_peered":"2023-10-09T08:34:27.727113+0000","last_unstale":"2023-10-09T08:35:04.727819+0000","last_undegraded":"2023-10-09T08:35:04.727819+0000","last_fullsized":"2023-10-09T08:35:04.727819+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727621+0000","last_change":"2023-10-09T08:34:27.759549+0000","last_active":"2023-10-09T08:35:04.727621+0000","last_peered":"2023-10-09T08:35:04.727621+0000","last_clean":"2023-10-09T08:35:04.727621+0000","last_became_active":"2023-10-09T08:34:27.759253+0000","last_became_peered":"2023-10-09T08:34:27.759253+0000","last_unstale":"2023-10-09T08:35:04.727621+0000","last_undegraded":"2023-10-09T08:35:04.727621+0000","last_fullsized":"2023-10-09T08:35:04.727621+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.732287+0000","last_change":"2023-10-09T08:34:27.722692+0000","last_active":"2023-10-09T08:35:04.732287+0000","last_peered":"2023-10-09T08:35:04.732287+0000","last_clean":"2023-10-09T08:35:04.732287+0000","last_became_active":"2023-10-09T08:34:27.722326+0000","last_became_peered":"2023-10-09T08:34:27.722326+0000","last_unstale":"2023-10-09T08:35:04.732287+0000","last_undegraded":"2023-10-09T08:35:04.732287+0000","last_fullsized":"2023-10-09T08:35:04.732287+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727419+0000","last_change":"2023-10-09T08:34:27.758797+0000","last_active":"2023-10-09T08:35:04.727419+0000","last_peered":"2023-10-09T08:35:04.727419+0000","last_clean":"2023-10-09T08:35:04.727419+0000","last_became_active":"2023-10-09T08:34:27.758390+0000","last_became_peered":"2023-10-09T08:34:27.758390+0000","last_unstale":"2023-10-09T08:35:04.727419+0000","last_undegraded":"2023-10-09T08:35:04.727419+0000","last_fullsized":"2023-10-09T08:35:04.727419+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723336+0000","last_change":"2023-10-09T08:34:27.738199+0000","last_active":"2023-10-09T08:35:04.723336+0000","last_peered":"2023-10-09T08:35:04.723336+0000","last_clean":"2023-10-09T08:35:04.723336+0000","last_became_active":"2023-10-09T08:34:27.737208+0000","last_became_peered":"2023-10-09T08:34:27.737208+0000","last_unstale":"2023-10-09T08:35:04.723336+0000","last_undegraded":"2023-10-09T08:35:04.723336+0000","last_fullsized":"2023-10-09T08:35:04.723336+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721180+0000","last_change":"2023-10-09T08:34:27.752491+0000","last_active":"2023-10-09T08:35:04.721180+0000","last_peered":"2023-10-09T08:35:04.721180+0000","last_clean":"2023-10-09T08:35:04.721180+0000","last_became_active":"2023-10-09T08:34:27.752099+0000","last_became_peered":"2023-10-09T08:34:27.752099+0000","last_unstale":"2023-10-09T08:35:04.721180+0000","last_undegraded":"2023-10-09T08:35:04.721180+0000","last_fullsized":"2023-10-09T08:35:04.721180+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723275+0000","last_change":"2023-10-09T08:34:27.739060+0000","last_active":"2023-10-09T08:35:04.723275+0000","last_peered":"2023-10-09T08:35:04.723275+0000","last_clean":"2023-10-09T08:35:04.723275+0000","last_became_active":"2023-10-09T08:34:27.738704+0000","last_became_peered":"2023-10-09T08:34:27.738704+0000","last_unstale":"2023-10-09T08:35:04.723275+0000","last_undegraded":"2023-10-09T08:35:04.723275+0000","last_fullsized":"2023-10-09T08:35:04.723275+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T08:35:05.733763+0000","last_change":"2023-10-09T08:35:05.733763+0000","last_active":"2023-10-09T08:35:05.733763+0000","last_peered":"2023-10-09T08:35:05.733763+0000","last_clean":"2023-10-09T08:35:05.733763+0000","last_became_active":"2023-10-09T08:35:05.733184+0000","last_became_peered":"2023-10-09T08:35:05.733184+0000","last_unstale":"2023-10-09T08:35:05.733763+0000","last_undegraded":"2023-10-09T08:35:05.733763+0000","last_fullsized":"2023-10-09T08:35:05.733763+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733977+0000","last_change":"2023-10-09T08:34:27.733891+0000","last_active":"2023-10-09T08:35:04.733977+0000","last_peered":"2023-10-09T08:35:04.733977+0000","last_clean":"2023-10-09T08:35:04.733977+0000","last_became_active":"2023-10-09T08:34:27.733379+0000","last_became_peered":"2023-10-09T08:34:27.733379+0000","last_unstale":"2023-10-09T08:35:04.733977+0000","last_undegraded":"2023-10-09T08:35:04.733977+0000","last_fullsized":"2023-10-09T08:35:04.733977+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T08:35:05.734125+0000","last_change":"2023-10-09T08:35:05.734125+0000","last_active":"2023-10-09T08:35:05.734125+0000","last_peered":"2023-10-09T08:35:05.734125+0000","last_clean":"2023-10-09T08:35:05.734125+0000","last_became_active":"2023-10-09T08:35:05.733657+0000","last_became_peered":"2023-10-09T08:35:05.733657+0000","last_unstale":"2023-10-09T08:35:05.734125+0000","last_undegraded":"2023-10-09T08:35:05.734125+0000","last_fullsized":"2023-10-09T08:35:05.734125+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733952+0000","last_change":"2023-10-09T08:34:27.728108+0000","last_active":"2023-10-09T08:35:04.733952+0000","last_peered":"2023-10-09T08:35:04.733952+0000","last_clean":"2023-10-09T08:35:04.733952+0000","last_became_active":"2023-10-09T08:34:27.727139+0000","last_became_peered":"2023-10-09T08:34:27.727139+0000","last_unstale":"2023-10-09T08:35:04.733952+0000","last_undegraded":"2023-10-09T08:35:04.733952+0000","last_fullsized":"2023-10-09T08:35:04.733952+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.722900+0000","last_change":"2023-10-09T08:34:29.490932+0000","last_active":"2023-10-09T08:35:04.722900+0000","last_peered":"2023-10-09T08:35:04.722900+0000","last_clean":"2023-10-09T08:35:04.722900+0000","last_became_active":"2023-10-09T08:34:29.489841+0000","last_became_peered":"2023-10-09T08:34:29.489841+0000","last_unstale":"2023-10-09T08:35:04.722900+0000","last_undegraded":"2023-10-09T08:35:04.722900+0000","last_fullsized":"2023-10-09T08:35:04.722900+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.734081+0000","last_change":"2023-10-09T08:34:27.728079+0000","last_active":"2023-10-09T08:35:04.734081+0000","last_peered":"2023-10-09T08:35:04.734081+0000","last_clean":"2023-10-09T08:35:04.734081+0000","last_became_active":"2023-10-09T08:34:27.727601+0000","last_became_peered":"2023-10-09T08:34:27.727601+0000","last_unstale":"2023-10-09T08:35:04.734081+0000","last_undegraded":"2023-10-09T08:35:04.734081+0000","last_fullsized":"2023-10-09T08:35:04.734081+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724168+0000","last_change":"2023-10-09T08:34:29.490108+0000","last_active":"2023-10-09T08:35:04.724168+0000","last_peered":"2023-10-09T08:35:04.724168+0000","last_clean":"2023-10-09T08:35:04.724168+0000","last_became_active":"2023-10-09T08:34:29.489133+0000","last_became_peered":"2023-10-09T08:34:29.489133+0000","last_unstale":"2023-10-09T08:35:04.724168+0000","last_undegraded":"2023-10-09T08:35:04.724168+0000","last_fullsized":"2023-10-09T08:35:04.724168+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.731477+0000","last_change":"2023-10-09T08:34:27.735071+0000","last_active":"2023-10-09T08:35:04.731477+0000","last_peered":"2023-10-09T08:35:04.731477+0000","last_clean":"2023-10-09T08:35:04.731477+0000","last_became_active":"2023-10-09T08:34:27.734567+0000","last_became_peered":"2023-10-09T08:34:27.734567+0000","last_unstale":"2023-10-09T08:35:04.731477+0000","last_undegraded":"2023-10-09T08:35:04.731477+0000","last_fullsized":"2023-10-09T08:35:04.731477+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721208+0000","last_change":"2023-10-09T08:34:27.859165+0000","last_active":"2023-10-09T08:35:04.721208+0000","last_peered":"2023-10-09T08:35:04.721208+0000","last_clean":"2023-10-09T08:35:04.721208+0000","last_became_active":"2023-10-09T08:34:27.858349+0000","last_became_peered":"2023-10-09T08:34:27.858349+0000","last_unstale":"2023-10-09T08:35:04.721208+0000","last_undegraded":"2023-10-09T08:35:04.721208+0000","last_fullsized":"2023-10-09T08:35:04.721208+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733421+0000","last_change":"2023-10-09T08:34:27.726666+0000","last_active":"2023-10-09T08:35:04.733421+0000","last_peered":"2023-10-09T08:35:04.733421+0000","last_clean":"2023-10-09T08:35:04.733421+0000","last_became_active":"2023-10-09T08:34:27.726189+0000","last_became_peered":"2023-10-09T08:34:27.726189+0000","last_unstale":"2023-10-09T08:35:04.733421+0000","last_undegraded":"2023-10-09T08:35:04.733421+0000","last_fullsized":"2023-10-09T08:35:04.733421+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723442+0000","last_change":"2023-10-09T08:34:29.490096+0000","last_active":"2023-10-09T08:35:04.723442+0000","last_peered":"2023-10-09T08:35:04.723442+0000","last_clean":"2023-10-09T08:35:04.723442+0000","last_became_active":"2023-10-09T08:34:29.489525+0000","last_became_peered":"2023-10-09T08:34:29.489525+0000","last_unstale":"2023-10-09T08:35:04.723442+0000","last_undegraded":"2023-10-09T08:35:04.723442+0000","last_fullsized":"2023-10-09T08:35:04.723442+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721039+0000","last_change":"2023-10-09T08:34:27.732427+0000","last_active":"2023-10-09T08:35:04.721039+0000","last_peered":"2023-10-09T08:35:04.721039+0000","last_clean":"2023-10-09T08:35:04.721039+0000","last_became_active":"2023-10-09T08:34:27.730518+0000","last_became_peered":"2023-10-09T08:34:27.730518+0000","last_unstale":"2023-10-09T08:35:04.721039+0000","last_undegraded":"2023-10-09T08:35:04.721039+0000","last_fullsized":"2023-10-09T08:35:04.721039+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.725322+0000","last_change":"2023-10-09T08:34:29.489335+0000","last_active":"2023-10-09T08:35:04.725322+0000","last_peered":"2023-10-09T08:35:04.725322+0000","last_clean":"2023-10-09T08:35:04.725322+0000","last_became_active":"2023-10-09T08:34:29.488468+0000","last_became_peered":"2023-10-09T08:34:29.488468+0000","last_unstale":"2023-10-09T08:35:04.725322+0000","last_undegraded":"2023-10-09T08:35:04.725322+0000","last_fullsized":"2023-10-09T08:35:04.725322+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733545+0000","last_change":"2023-10-09T08:34:27.731381+0000","last_active":"2023-10-09T08:35:04.733545+0000","last_peered":"2023-10-09T08:35:04.733545+0000","last_clean":"2023-10-09T08:35:04.733545+0000","last_became_active":"2023-10-09T08:34:27.731065+0000","last_became_peered":"2023-10-09T08:34:27.731065+0000","last_unstale":"2023-10-09T08:35:04.733545+0000","last_undegraded":"2023-10-09T08:35:04.733545+0000","last_fullsized":"2023-10-09T08:35:04.733545+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727757+0000","last_change":"2023-10-09T08:34:27.753177+0000","last_active":"2023-10-09T08:35:04.727757+0000","last_peered":"2023-10-09T08:35:04.727757+0000","last_clean":"2023-10-09T08:35:04.727757+0000","last_became_active":"2023-10-09T08:34:27.752856+0000","last_became_peered":"2023-10-09T08:34:27.752856+0000","last_unstale":"2023-10-09T08:35:04.727757+0000","last_undegraded":"2023-10-09T08:35:04.727757+0000","last_fullsized":"2023-10-09T08:35:04.727757+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.727356+0000","last_change":"2023-10-09T08:34:27.730961+0000","last_active":"2023-10-09T08:35:04.727356+0000","last_peered":"2023-10-09T08:35:04.727356+0000","last_clean":"2023-10-09T08:35:04.727356+0000","last_became_active":"2023-10-09T08:34:27.730198+0000","last_became_peered":"2023-10-09T08:34:27.730198+0000","last_unstale":"2023-10-09T08:35:04.727356+0000","last_undegraded":"2023-10-09T08:35:04.727356+0000","last_fullsized":"2023-10-09T08:35:04.727356+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.728084+0000","last_change":"2023-10-09T08:34:27.761929+0000","last_active":"2023-10-09T08:35:04.728084+0000","last_peered":"2023-10-09T08:35:04.728084+0000","last_clean":"2023-10-09T08:35:04.728084+0000","last_became_active":"2023-10-09T08:34:27.761331+0000","last_became_peered":"2023-10-09T08:34:27.761331+0000","last_unstale":"2023-10-09T08:35:04.728084+0000","last_undegraded":"2023-10-09T08:35:04.728084+0000","last_fullsized":"2023-10-09T08:35:04.728084+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.733853+0000","last_change":"2023-10-09T08:34:27.733630+0000","last_active":"2023-10-09T08:35:04.733853+0000","last_peered":"2023-10-09T08:35:04.733853+0000","last_clean":"2023-10-09T08:35:04.733853+0000","last_became_active":"2023-10-09T08:34:27.733125+0000","last_became_peered":"2023-10-09T08:34:27.733125+0000","last_unstale":"2023-10-09T08:35:04.733853+0000","last_undegraded":"2023-10-09T08:35:04.733853+0000","last_fullsized":"2023-10-09T08:35:04.733853+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724448+0000","last_change":"2023-10-09T08:34:29.489279+0000","last_active":"2023-10-09T08:35:04.724448+0000","last_peered":"2023-10-09T08:35:04.724448+0000","last_clean":"2023-10-09T08:35:04.724448+0000","last_became_active":"2023-10-09T08:34:29.488626+0000","last_became_peered":"2023-10-09T08:34:29.488626+0000","last_unstale":"2023-10-09T08:35:04.724448+0000","last_undegraded":"2023-10-09T08:35:04.724448+0000","last_fullsized":"2023-10-09T08:35:04.724448+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723446+0000","last_change":"2023-10-09T08:34:27.733217+0000","last_active":"2023-10-09T08:35:04.723446+0000","last_peered":"2023-10-09T08:35:04.723446+0000","last_clean":"2023-10-09T08:35:04.723446+0000","last_became_active":"2023-10-09T08:34:27.732878+0000","last_became_peered":"2023-10-09T08:34:27.732878+0000","last_unstale":"2023-10-09T08:35:04.723446+0000","last_undegraded":"2023-10-09T08:35:04.723446+0000","last_fullsized":"2023-10-09T08:35:04.723446+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724291+0000","last_change":"2023-10-09T08:34:29.173037+0000","last_active":"2023-10-09T08:35:04.724291+0000","last_peered":"2023-10-09T08:35:04.724291+0000","last_clean":"2023-10-09T08:35:04.724291+0000","last_became_active":"2023-10-09T08:34:29.172274+0000","last_became_peered":"2023-10-09T08:34:29.172274+0000","last_unstale":"2023-10-09T08:35:04.724291+0000","last_undegraded":"2023-10-09T08:35:04.724291+0000","last_fullsized":"2023-10-09T08:35:04.724291+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721232+0000","last_change":"2023-10-09T08:34:27.732056+0000","last_active":"2023-10-09T08:35:04.721232+0000","last_peered":"2023-10-09T08:35:04.721232+0000","last_clean":"2023-10-09T08:35:04.721232+0000","last_became_active":"2023-10-09T08:34:27.731578+0000","last_became_peered":"2023-10-09T08:34:27.731578+0000","last_unstale":"2023-10-09T08:35:04.721232+0000","last_undegraded":"2023-10-09T08:35:04.721232+0000","last_fullsized":"2023-10-09T08:35:04.721232+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.731984+0000","last_change":"2023-10-09T08:34:27.760239+0000","last_active":"2023-10-09T08:35:04.731984+0000","last_peered":"2023-10-09T08:35:04.731984+0000","last_clean":"2023-10-09T08:35:04.731984+0000","last_became_active":"2023-10-09T08:34:27.759819+0000","last_became_peered":"2023-10-09T08:34:27.759819+0000","last_unstale":"2023-10-09T08:35:04.731984+0000","last_undegraded":"2023-10-09T08:35:04.731984+0000","last_fullsized":"2023-10-09T08:35:04.731984+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.724430+0000","last_change":"2023-10-09T08:34:27.737609+0000","last_active":"2023-10-09T08:35:04.724430+0000","last_peered":"2023-10-09T08:35:04.724430+0000","last_clean":"2023-10-09T08:35:04.724430+0000","last_became_active":"2023-10-09T08:34:27.737283+0000","last_became_peered":"2023-10-09T08:34:27.737283+0000","last_unstale":"2023-10-09T08:35:04.724430+0000","last_undegraded":"2023-10-09T08:35:04.724430+0000","last_fullsized":"2023-10-09T08:35:04.724430+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.721392+0000","last_change":"2023-10-09T08:34:27.860387+0000","last_active":"2023-10-09T08:35:04.721392+0000","last_peered":"2023-10-09T08:35:04.721392+0000","last_clean":"2023-10-09T08:35:04.721392+0000","last_became_active":"2023-10-09T08:34:27.859698+0000","last_became_peered":"2023-10-09T08:34:27.859698+0000","last_unstale":"2023-10-09T08:35:04.721392+0000","last_undegraded":"2023-10-09T08:35:04.721392+0000","last_fullsized":"2023-10-09T08:35:04.721392+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.729752+0000","last_change":"2023-10-09T08:34:27.735616+0000","last_active":"2023-10-09T08:35:04.729752+0000","last_peered":"2023-10-09T08:35:04.729752+0000","last_clean":"2023-10-09T08:35:04.729752+0000","last_became_active":"2023-10-09T08:34:27.735286+0000","last_became_peered":"2023-10-09T08:34:27.735286+0000","last_unstale":"2023-10-09T08:35:04.729752+0000","last_undegraded":"2023-10-09T08:35:04.729752+0000","last_fullsized":"2023-10-09T08:35:04.729752+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.723322+0000","last_change":"2023-10-09T08:34:29.234364+0000","last_active":"2023-10-09T08:35:04.723322+0000","last_peered":"2023-10-09T08:35:04.723322+0000","last_clean":"2023-10-09T08:35:04.723322+0000","last_became_active":"2023-10-09T08:34:29.233384+0000","last_became_peered":"2023-10-09T08:34:29.233384+0000","last_unstale":"2023-10-09T08:35:04.723322+0000","last_undegraded":"2023-10-09T08:35:04.723322+0000","last_fullsized":"2023-10-09T08:35:04.723322+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T08:35:04.726835+0000","last_change":"2023-10-09T08:34:27.727597+0000","last_active":"2023-10-09T08:35:04.726835+0000","last_peered":"2023-10-09T08:35:04.726835+0000","last_clean":"2023-10-09T08:35:04.726835+0000","last_became_active":"2023-10-09T08:34:27.727176+0000","last_became_peered":"2023-10-09T08:34:27.727176+0000","last_unstale":"2023-10-09T08:35:04.726835+0000","last_undegraded":"2023-10-09T08:35:04.726835+0000","last_fullsized":"2023-10-09T08:35:04.726835+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430267,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.89000000000000001,"15min":0.85799999999999998},"min":{"1min":0.65100000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.2509999999999999,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.93100000000000005,"15min":0.93700000000000006},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.82399999999999995}]},{"osd":1,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.93500000000000005,"15min":0.93300000000000005},"min":{"1min":0.68200000000000005,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.1499999999999999,"5min":1.786,"15min":1.786},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.92300000000000004,"15min":0.90100000000000002},"min":{"1min":0.71699999999999997,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.216,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.95399999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.93300000000000005,"15min":0.93899999999999995},"min":{"1min":0.76800000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.123,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.95399999999999996,"15min":0.96399999999999997},"min":{"1min":0.65600000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.359,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.73699999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91700000000000004,"15min":0.90200000000000002},"min":{"1min":0.71499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.0860000000000001,"5min":1.276,"15min":1.276},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.038,"5min":1.0049999999999999,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.268,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":0.85399999999999998}]},{"osd":4,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.88500000000000001,"15min":0.86199999999999999},"min":{"1min":0.61699999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.4279999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.92100000000000004,"15min":0.89100000000000001},"min":{"1min":0.68999999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.22,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.68899999999999995}]},{"osd":5,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":0.98499999999999999,"15min":0.94499999999999995},"min":{"1min":0.69599999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.2509999999999999,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.087},{"interface":"front","average":{"1min":1.034,"5min":0.96699999999999997,"15min":0.93300000000000005},"min":{"1min":0.78500000000000003,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.4890000000000001,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.032}]},{"osd":6,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.024,"15min":0.98999999999999999},"min":{"1min":0.72599999999999998,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.5309999999999999,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":1.111},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.0169999999999999,"15min":0.98099999999999998},"min":{"1min":0.81499999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.97999999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626500,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":271239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.85399999999999998,"15min":0.90200000000000002},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.179,"5min":1.385,"15min":1.385},"last":1.2569999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.90800000000000003,"15min":0.91200000000000003},"min":{"1min":0.69299999999999995,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.256,"5min":1.327,"15min":1.327},"last":1.0489999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.873,"15min":0.90200000000000002},"min":{"1min":0.56699999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.1859999999999999,"5min":1.496,"15min":1.496},"last":1.23},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.89400000000000002,"15min":0.91000000000000003},"min":{"1min":0.71699999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.236,"5min":1.329,"15min":1.329},"last":1.0860000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.89000000000000001,"15min":0.93700000000000006},"min":{"1min":0.755,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.1870000000000001,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.93999999999999995,"15min":0.98299999999999998},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.2050000000000001,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.2030000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.93200000000000005,"15min":0.98599999999999999},"min":{"1min":0.73699999999999999,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.1000000000000001,"5min":1.774,"15min":1.774},"last":1.018},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.94899999999999995,"15min":1.0229999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.24,"5min":1.476,"15min":1.476},"last":1.1759999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.86099999999999999,"15min":0.84599999999999997},"min":{"1min":0.63200000000000001,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.123,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.119},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90500000000000003,"15min":0.91200000000000003},"min":{"1min":0.65800000000000003,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.175,"5min":1.619,"15min":1.619},"last":1.141}]},{"osd":5,"last update":"Mon Oct 9 08:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.95699999999999996,"15min":0.96399999999999997},"min":{"1min":0.71199999999999997,"5min":0.504,"15min":0.504},"max":{"1min":1.1830000000000001,"5min":1.544,"15min":1.544},"last":1.2869999999999999},{"interface":"front","average":{"1min":1.0369999999999999,"5min":0.96499999999999997,"15min":0.97699999999999998},"min":{"1min":0.80600000000000005,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.218,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.3140000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.98099999999999998,"15min":0.93200000000000005},"min":{"1min":0.78400000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.232,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.349},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.0089999999999999,"15min":0.95399999999999996},"min":{"1min":0.76000000000000001,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":1.369}]}]},{"osd":1,"up_from":13,"seq":55834574960,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.71599999999999997,"15min":0.60699999999999998},"min":{"1min":0.47799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.91600000000000004,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.78500000000000003,"15min":0.65100000000000002},"min":{"1min":0.58499999999999996,"5min":0.379,"15min":0.379},"max":{"1min":1.0780000000000001,"5min":1.264,"15min":1.264},"last":1.2629999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.76100000000000001,"15min":0.70899999999999996},"min":{"1min":0.45500000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79200000000000004,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.745},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.749,"15min":0.71899999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.96499999999999997,"5min":1.165,"15min":1.165},"last":0.79300000000000004}]},{"osd":3,"last update":"Mon Oct 9 08:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.82399999999999995,"15min":0.81899999999999995},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.0960000000000001,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.83799999999999997,"15min":0.80700000000000005},"min":{"1min":0.47699999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.97999999999999998,"5min":1.6739999999999999,"15min":1.6739999999999999},"last":1.083}]},{"osd":4,"last update":"Mon Oct 9 08:35:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.004,"15min":0.96699999999999997},"min":{"1min":0.78600000000000003,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.1930000000000001,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.94299999999999995},{"interface":"front","average":{"1min":1.022,"5min":1.0369999999999999,"15min":0.97899999999999998},"min":{"1min":0.745,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.405,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":1.014}]},{"osd":5,"last update":"Mon Oct 9 08:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0329999999999999,"15min":1.0780000000000001},"min":{"1min":0.63600000000000001,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4179999999999999,"5min":1.599,"15min":1.599},"last":1.202},{"interface":"front","average":{"1min":1.036,"5min":1.046,"15min":1.04},"min":{"1min":0.65900000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.321,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":1.2250000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.1000000000000001,"15min":1.1559999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.1699999999999999,"5min":1.639,"15min":1.639},"last":1.292},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.0980000000000001,"15min":1.1679999999999999},"min":{"1min":0.70699999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.3879999999999999,"5min":1.724,"15min":1.724},"last":1.3149999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:39 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.127,"15min":1.155},"min":{"1min":0.84199999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.4830000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.371},{"interface":"front","average":{"1min":1.145,"5min":1.171,"15min":1.1910000000000001},"min":{"1min":0.85799999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"max":{"1min":1.4610000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.3560000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738490,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:35:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.61199999999999999},"min":{"1min":0.57899999999999996,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0580000000000001,"5min":1.373,"15min":1.373},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.80100000000000005,"15min":0.66300000000000003},"min":{"1min":0.58399999999999996,"5min":0.41299999999999998,"15min":0.39000000000000001},"max":{"1min":1.1200000000000001,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.97899999999999998}]},{"osd":2,"last update":"Mon Oct 9 08:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.78000000000000003,"15min":0.69299999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.242,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.78000000000000003,"15min":0.68600000000000005},"min":{"1min":0.40799999999999997,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.93400000000000005,"5min":1.335,"15min":1.335},"last":0.74099999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:35:30 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.77600000000000002,"15min":0.76700000000000002},"min":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.002,"5min":1.228,"15min":1.228},"last":1.2450000000000001},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.90400000000000003,"15min":0.85599999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.175,"5min":1.544,"15min":1.544},"last":1.147}]},{"osd":4,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.024,"15min":1.022},"min":{"1min":0.69299999999999995,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.458,"5min":1.458,"15min":1.458},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.115,"5min":1.032,"15min":0.99399999999999999},"min":{"1min":0.78100000000000003,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.5620000000000001,"5min":1.75,"15min":1.75},"last":1.091}]},{"osd":5,"last update":"Mon Oct 9 08:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.0580000000000001,"15min":1.0349999999999999},"min":{"1min":0.90900000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.538,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.044},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.0860000000000001,"15min":1.0640000000000001},"min":{"1min":0.84399999999999997,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":0.90800000000000003}]},{"osd":6,"last update":"Mon Oct 9 08:35:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.119,"15min":1.173},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.5920000000000001,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.093,"5min":1.113,"15min":1.1579999999999999},"min":{"1min":0.71899999999999997,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.6830000000000001,"5min":1.6830000000000001,"15min":1.6830000000000001},"last":1.1719999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:39 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.1559999999999999,"15min":1.2},"min":{"1min":0.69099999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.6459999999999999,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":1.018},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1579999999999999,"15min":1.1890000000000001},"min":{"1min":0.70899999999999996,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.7549999999999999,"5min":1.7549999999999999,"15min":1.7549999999999999},"last":1.1220000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411431,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.72799999999999998,"15min":0.66500000000000004},"min":{"1min":0.48199999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.96799999999999997,"5min":1.595,"15min":1.595},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.71799999999999997,"15min":0.67700000000000005},"min":{"1min":0.39100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.95399999999999996,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.64600000000000002}]},{"osd":1,"last update":"Mon Oct 9 08:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.73999999999999999,"15min":0.69099999999999995},"min":{"1min":0.42399999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.99299999999999999,"5min":1.708,"15min":1.708},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.83499999999999996,"15min":0.73399999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.0920000000000001,"5min":1.3,"15min":1.3},"last":0.95799999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.81899999999999995,"15min":0.81999999999999995},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.042,"5min":1.349,"15min":1.349},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.88300000000000001,"15min":0.86099999999999999},"min":{"1min":0.59199999999999997,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.2569999999999999,"5min":1.464,"15min":1.464},"last":1.0329999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.99299999999999999,"15min":0.98099999999999998},"min":{"1min":0.70599999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.613,"5min":1.613,"15min":1.613},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.052,"5min":1.0469999999999999,"15min":1.0169999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":1.111}]},{"osd":5,"last update":"Mon Oct 9 08:36:05 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.0049999999999999,"15min":1.02},"min":{"1min":0.71399999999999997,"5min":0.63,"15min":0.63},"max":{"1min":1.341,"5min":2.145,"15min":2.145},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.109,"15min":1.069},"min":{"1min":0.69499999999999995,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.3029999999999999,"5min":1.9550000000000001,"15min":1.9550000000000001},"last":1.0720000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:35:48 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.127,"15min":1.133},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.4159999999999999,"5min":2.1120000000000001,"15min":2.1120000000000001},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.1599999999999999,"15min":1.1499999999999999},"min":{"1min":0.87,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.579,"5min":2.1789999999999998,"15min":2.1789999999999998},"last":1.28}]},{"osd":7,"last update":"Mon Oct 9 08:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.117,"15min":1.149},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.371,"5min":2.0310000000000001,"15min":2.0310000000000001},"last":1.393},{"interface":"front","average":{"1min":1.121,"5min":1.206,"15min":1.238},"min":{"1min":0.70399999999999996,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.458,"5min":2.202,"15min":2.202},"last":1.2310000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182495,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.68600000000000005,"15min":0.67000000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83699999999999997,"5min":1.2,"15min":1.2},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78600000000000003,"15min":0.77700000000000002},"min":{"1min":0.46300000000000002,"5min":0.31,"15min":0.31},"max":{"1min":1.0049999999999999,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.623}]},{"osd":1,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.80200000000000005,"15min":0.80900000000000005},"min":{"1min":0.53300000000000003,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0589999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.72899999999999998,"15min":0.745},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.024,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.70399999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.82799999999999996,"15min":0.83899999999999997},"min":{"1min":0.439,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.163,"5min":1.8400000000000001,"15min":1.8400000000000001},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.79200000000000004,"15min":0.77600000000000002},"min":{"1min":0.503,"5min":0.373,"15min":0.373},"max":{"1min":1.2150000000000001,"5min":1.514,"15min":1.514},"last":0.89000000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:35:19 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.98799999999999999,"15min":0.93500000000000005},"min":{"1min":0.755,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2669999999999999,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.995},{"interface":"front","average":{"1min":1.05,"5min":1.0009999999999999,"15min":0.97799999999999998},"min":{"1min":0.73699999999999999,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.4179999999999999,"5min":1.542,"15min":1.542},"last":1.1559999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.0209999999999999,"15min":1.0029999999999999},"min":{"1min":0.68400000000000005,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3540000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.0669999999999999,"15min":1.0289999999999999},"min":{"1min":0.66300000000000003,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.4319999999999999,"5min":1.663,"15min":1.663},"last":1.236}]},{"osd":6,"last update":"Mon Oct 9 08:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.0860000000000001,"15min":1.1220000000000001},"min":{"1min":0.72099999999999997,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.526,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":1.204},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.103,"15min":1.1439999999999999},"min":{"1min":0.747,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.468,"5min":1.905,"15min":1.905},"last":1.373}]},{"osd":7,"last update":"Mon Oct 9 08:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.143,"15min":1.165},"min":{"1min":0.78200000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.605,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":1.3120000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.0880000000000001,"15min":1.1040000000000001},"min":{"1min":0.72899999999999998,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3420000000000001,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":1.173}]}]},{"osd":4,"up_from":30,"seq":128849018966,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.85299999999999998,"15min":0.80100000000000005},"min":{"1min":0.51400000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.163,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.90300000000000002,"15min":0.89500000000000002},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2689999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.88500000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.84299999999999997,"15min":0.80700000000000005},"min":{"1min":0.59399999999999997,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2170000000000001,"5min":1.244,"15min":1.244},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.89000000000000001,"15min":0.878},"min":{"1min":0.57699999999999996,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":1.1339999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.89700000000000002,"15min":0.85899999999999999},"min":{"1min":0.66500000000000004,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.95299999999999996,"15min":0.90900000000000003},"min":{"1min":0.59799999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.1990000000000001,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.121}]},{"osd":3,"last update":"Mon Oct 9 08:35:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.95299999999999996,"15min":0.90300000000000002},"min":{"1min":0.69699999999999995,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.2889999999999999,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.91700000000000004,"15min":0.90500000000000003},"min":{"1min":0.627,"5min":0.625,"15min":0.623},"max":{"1min":1.2809999999999999,"5min":1.302,"15min":1.302},"last":0.91600000000000004}]},{"osd":5,"last update":"Mon Oct 9 08:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.88600000000000001,"15min":0.75800000000000001},"min":{"1min":0.60799999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.05,"5min":0.95599999999999996,"15min":0.81899999999999995},"min":{"1min":0.69399999999999995,"5min":0.439,"15min":0.38700000000000001},"max":{"1min":1.369,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.97599999999999998}]},{"osd":6,"last update":"Mon Oct 9 08:35:47 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.89600000000000002,"15min":0.85199999999999998},"min":{"1min":0.79400000000000004,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.3819999999999999,"5min":1.645,"15min":1.645},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.127,"5min":0.98099999999999998,"15min":0.90500000000000003},"min":{"1min":0.81699999999999995,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.0029999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.0269999999999999,"15min":0.98199999999999998},"min":{"1min":0.88600000000000001,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.014,"15min":0.97599999999999998},"min":{"1min":0.91200000000000003,"5min":0.5,"15min":0.5},"max":{"1min":1.454,"5min":1.724,"15min":1.724},"last":1.099}]}]},{"osd":5,"up_from":36,"seq":154618822734,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.86099999999999999,"15min":0.82399999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.2170000000000001,"5min":1.238,"15min":1.238},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.86399999999999999,"15min":0.88300000000000001},"min":{"1min":0.61599999999999999,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.93700000000000006}]},{"osd":1,"last update":"Mon Oct 9 08:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.89100000000000001,"15min":0.877},"min":{"1min":0.52400000000000002,"5min":0.502,"15min":0.502},"max":{"1min":1.2729999999999999,"5min":1.381,"15min":1.381},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.88100000000000001,"15min":0.84499999999999997},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.0980000000000001,"5min":1.587,"15min":1.587},"last":0.85099999999999998}]},{"osd":2,"last update":"Mon Oct 9 08:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.92900000000000005,"15min":0.88500000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.2270000000000001,"5min":2.9950000000000001,"15min":2.9950000000000001},"last":0.95999999999999996},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.93300000000000005,"15min":0.90300000000000002},"min":{"1min":0.54000000000000004,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.78800000000000003}]},{"osd":3,"last update":"Mon Oct 9 08:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.91800000000000004,"15min":0.92900000000000005},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.244,"5min":1.2569999999999999,"15min":1.276},"last":1.097},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.99199999999999999,"15min":0.93799999999999994},"min":{"1min":0.60499999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2709999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.92000000000000004}]},{"osd":4,"last update":"Mon Oct 9 08:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.82799999999999996,"15min":0.73899999999999999},"min":{"1min":0.46999999999999997,"5min":0.439,"15min":0.439},"max":{"1min":1.0569999999999999,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.90000000000000002,"15min":0.79100000000000004},"min":{"1min":0.58399999999999996,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.242,"5min":1.387,"15min":1.387},"last":0.88200000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.92900000000000005,"15min":0.88300000000000001},"min":{"1min":0.45200000000000001,"5min":0.439,"15min":0.439},"max":{"1min":1.24,"5min":1.349,"15min":1.349},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.91900000000000004,"15min":0.84899999999999998},"min":{"1min":0.68700000000000006,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1859999999999999,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.0029999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.96499999999999997,"15min":0.88900000000000001},"min":{"1min":0.79100000000000004,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.29,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":1.054},{"interface":"front","average":{"1min":1.083,"5min":1.004,"15min":0.92200000000000004},"min":{"1min":0.83399999999999996,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.351,"5min":1.421,"15min":1.421},"last":1.0820000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:36:08.451 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T08:36:08.451 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T08:36:08.451 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T08:36:08.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T08:36:09.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:08 smithi094 bash[27803]: audit 2023-10-09T08:36:07.586393+0000 mgr.y (mgr.24355) 156 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:36:09.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:08 smithi016 bash[27454]: audit 2023-10-09T08:36:07.586393+0000 mgr.y (mgr.24355) 156 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:36:09.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:08 smithi016 bash[33851]: audit 2023-10-09T08:36:07.586393+0000 mgr.y (mgr.24355) 156 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:36:10.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:09 smithi094 bash[27803]: cluster 2023-10-09T08:36:08.678538+0000 mgr.y (mgr.24355) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:10.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:09 smithi016 bash[27454]: cluster 2023-10-09T08:36:08.678538+0000 mgr.y (mgr.24355) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:10.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:09 smithi016 bash[33851]: cluster 2023-10-09T08:36:08.678538+0000 mgr.y (mgr.24355) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:11.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:11 smithi094 bash[27803]: cluster 2023-10-09T08:36:10.679388+0000 mgr.y (mgr.24355) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:11 smithi016 bash[27454]: cluster 2023-10-09T08:36:10.679388+0000 mgr.y (mgr.24355) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:11.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:11 smithi016 bash[33851]: cluster 2023-10-09T08:36:10.679388+0000 mgr.y (mgr.24355) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:12.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:36:12 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:36:13.232 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:36:13.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:13 smithi094 bash[27803]: cluster 2023-10-09T08:36:12.680044+0000 mgr.y (mgr.24355) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:14.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:13 smithi016 bash[27454]: cluster 2023-10-09T08:36:12.680044+0000 mgr.y (mgr.24355) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:14.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:13 smithi016 bash[33851]: cluster 2023-10-09T08:36:12.680044+0000 mgr.y (mgr.24355) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:14.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:14 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:14] "GET /metrics HTTP/1.1" 200 191381 "" "Prometheus/2.33.4" 2023-10-09T08:36:16.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:15 smithi016 bash[33851]: cluster 2023-10-09T08:36:14.680447+0000 mgr.y (mgr.24355) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:16.014 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:15 smithi016 bash[27454]: cluster 2023-10-09T08:36:14.680447+0000 mgr.y (mgr.24355) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:16.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:15 smithi094 bash[27803]: cluster 2023-10-09T08:36:14.680447+0000 mgr.y (mgr.24355) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:16.312 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:36:17.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:16 smithi016 bash[27454]: audit 2023-10-09T08:36:16.309059+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.16:0/1407212634' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:36:17.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:16 smithi016 bash[33851]: audit 2023-10-09T08:36:16.309059+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.16:0/1407212634' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:36:17.132 INFO:teuthology.orchestra.run.smithi016.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T08:36:17.132 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T08:36:17.132 INFO:teuthology.run_tasks:Running task print... 2023-10-09T08:36:17.147 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T08:36:17.147 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T08:36:17.201 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T08:36:17.201 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T08:36:17.202 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T08:36:17.206 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T08:36:17.206 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T08:36:17.206 DEBUG:teuthology.orchestra.run.smithi016:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T08:36:17.207 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T08:36:17.207 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T08:36:17.207 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi016.front.sepia.ceph.com 2023-10-09T08:36:17.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T08:36:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:36:17.215 INFO:teuthology.orchestra.run.smithi016.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T08:36:17.215 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T08:36:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:16 smithi094 bash[27803]: audit 2023-10-09T08:36:16.309059+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.16:0/1407212634' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:36:17.265 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T08:36:17.265 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T08:36:17.313 INFO:tasks.workunit:timeout=3h 2023-10-09T08:36:17.313 INFO:tasks.workunit:cleanup=True 2023-10-09T08:36:17.314 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T08:36:17.363 INFO:tasks.workunit.client.0.smithi016.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T08:36:18.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:17 smithi094 bash[27803]: cluster 2023-10-09T08:36:16.681208+0000 mgr.y (mgr.24355) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:18.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:17 smithi016 bash[33851]: cluster 2023-10-09T08:36:16.681208+0000 mgr.y (mgr.24355) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:18.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:17 smithi016 bash[27454]: cluster 2023-10-09T08:36:16.681208+0000 mgr.y (mgr.24355) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:20.104 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:19 smithi016 bash[27454]: cluster 2023-10-09T08:36:18.681821+0000 mgr.y (mgr.24355) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:20.104 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:19 smithi016 bash[33851]: cluster 2023-10-09T08:36:18.681821+0000 mgr.y (mgr.24355) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:20.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:19 smithi094 bash[27803]: cluster 2023-10-09T08:36:18.681821+0000 mgr.y (mgr.24355) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:21.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:20 smithi016 bash[27723]: [09/Oct/2023:08:36:20] ENGINE Bus STOPPING 2023-10-09T08:36:21.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:20 smithi016 bash[27723]: [09/Oct/2023:08:36:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:36:21.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:20 smithi016 bash[27723]: [09/Oct/2023:08:36:20] ENGINE Bus STOPPED 2023-10-09T08:36:21.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:20 smithi016 bash[27723]: [09/Oct/2023:08:36:20] ENGINE Bus STARTING 2023-10-09T08:36:21.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:20 smithi016 bash[27723]: [09/Oct/2023:08:36:20] ENGINE Serving on http://:::9283 2023-10-09T08:36:21.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:20 smithi016 bash[27723]: [09/Oct/2023:08:36:20] ENGINE Bus STARTED 2023-10-09T08:36:21.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T08:36:21.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:21 smithi094 bash[27803]: audit 2023-10-09T08:36:20.544080+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T08:36:21.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:21 smithi094 bash[27803]: audit 2023-10-09T08:36:20.609486+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:21.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:21 smithi094 bash[27803]: cluster 2023-10-09T08:36:20.682527+0000 mgr.y (mgr.24355) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:22.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:21 smithi016 bash[33851]: audit 2023-10-09T08:36:20.544080+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T08:36:22.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:21 smithi016 bash[33851]: audit 2023-10-09T08:36:20.609486+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:22.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:21 smithi016 bash[33851]: cluster 2023-10-09T08:36:20.682527+0000 mgr.y (mgr.24355) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:22.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:21 smithi016 bash[27454]: audit 2023-10-09T08:36:20.544080+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T08:36:22.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:21 smithi016 bash[27454]: audit 2023-10-09T08:36:20.609486+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:22.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:21 smithi016 bash[27454]: cluster 2023-10-09T08:36:20.682527+0000 mgr.y (mgr.24355) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:22.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:36:22 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:36:23.077 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T08:36:23.146 INFO:tasks.workunit.client.0.smithi016.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T08:36:23.148 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T08:36:23.149 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T08:36:24.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:23 smithi016 bash[27454]: cluster 2023-10-09T08:36:22.683072+0000 mgr.y (mgr.24355) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:24.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:23 smithi016 bash[33851]: cluster 2023-10-09T08:36:22.683072+0000 mgr.y (mgr.24355) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:24.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:23 smithi094 bash[27803]: cluster 2023-10-09T08:36:22.683072+0000 mgr.y (mgr.24355) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:24.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:24 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:24] "GET /metrics HTTP/1.1" 200 191378 "" "Prometheus/2.33.4" 2023-10-09T08:36:26.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:25 smithi016 bash[33851]: cluster 2023-10-09T08:36:24.683675+0000 mgr.y (mgr.24355) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:25 smithi016 bash[27454]: cluster 2023-10-09T08:36:24.683675+0000 mgr.y (mgr.24355) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:26.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:25 smithi094 bash[27803]: cluster 2023-10-09T08:36:24.683675+0000 mgr.y (mgr.24355) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:27.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:27 smithi016 bash[27454]: audit 2023-10-09T08:36:26.505848+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:27.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:27 smithi016 bash[27454]: cluster 2023-10-09T08:36:26.684786+0000 mgr.y (mgr.24355) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:27.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:27 smithi016 bash[33851]: audit 2023-10-09T08:36:26.505848+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:27.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:27 smithi016 bash[33851]: cluster 2023-10-09T08:36:26.684786+0000 mgr.y (mgr.24355) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:27 smithi094 bash[27803]: audit 2023-10-09T08:36:26.505848+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:27 smithi094 bash[27803]: cluster 2023-10-09T08:36:26.684786+0000 mgr.y (mgr.24355) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:28.098 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T08:36:28.197 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T08:36:28.257 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T08:36:28.260 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T08:36:28.260 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T08:36:28.320 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T08:36:28.325 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T08:36:28.325 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T08:36:28.330 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T08:36:28.332 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T08:36:28.332 DEBUG:teuthology.orchestra.run.smithi016:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T08:36:28.393 INFO:tasks.workunit.client.0.smithi016.stdout:Running main() from gmock_main.cc 2023-10-09T08:36:28.393 INFO:tasks.workunit.client.0.smithi016.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T08:36:28.393 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] Global test environment set-up. 2023-10-09T08:36:28.393 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T08:36:28.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:28 smithi016 bash[27723]: [09/Oct/2023:08:36:28] ENGINE Bus STOPPING 2023-10-09T08:36:28.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:28 smithi016 bash[27723]: [09/Oct/2023:08:36:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:36:28.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:28 smithi016 bash[27723]: [09/Oct/2023:08:36:28] ENGINE Bus STOPPED 2023-10-09T08:36:28.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:28 smithi016 bash[27723]: [09/Oct/2023:08:36:28] ENGINE Bus STARTING 2023-10-09T08:36:28.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:28 smithi016 bash[27723]: [09/Oct/2023:08:36:28] ENGINE Serving on http://:::9283 2023-10-09T08:36:28.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:28 smithi016 bash[27723]: [09/Oct/2023:08:36:28] ENGINE Bus STARTED 2023-10-09T08:36:29.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T08:36:29.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.064021+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.066812+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.068177+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.075635+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.341358+0000 mon.a (mon.0) 678 : audit [INF] from='client.? 172.21.15.16:0/2225145838' entity='client.admin' 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.397294+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.401066+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.402587+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: audit 2023-10-09T08:36:28.409881+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[27454]: cluster 2023-10-09T08:36:28.685367+0000 mgr.y (mgr.24355) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.064021+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.066812+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.068177+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.075635+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.341358+0000 mon.a (mon.0) 678 : audit [INF] from='client.? 172.21.15.16:0/2225145838' entity='client.admin' 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.397294+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.401066+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.402587+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: audit 2023-10-09T08:36:28.409881+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:29 smithi016 bash[33851]: cluster 2023-10-09T08:36:28.685367+0000 mgr.y (mgr.24355) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:29.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.064021+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.066812+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.068177+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.075635+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.341358+0000 mon.a (mon.0) 678 : audit [INF] from='client.? 172.21.15.16:0/2225145838' entity='client.admin' 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.397294+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.401066+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.402587+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: audit 2023-10-09T08:36:28.409881+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:29 smithi094 bash[27803]: cluster 2023-10-09T08:36:28.685367+0000 mgr.y (mgr.24355) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:30.446 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T08:36:30.453 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_all_features (7 ms) 2023-10-09T08:36:30.453 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T08:36:30.664 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.copyup (211 ms) 2023-10-09T08:36:30.664 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T08:36:30.688 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.sparse_copyup (24 ms) 2023-10-09T08:36:30.689 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T08:36:30.701 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_and_set_id (14 ms) 2023-10-09T08:36:30.701 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T08:36:30.718 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.add_remove_child (16 ms) 2023-10-09T08:36:30.718 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T08:36:30.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:30 smithi094 bash[27803]: cluster 2023-10-09T08:36:29.419250+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T08:36:30.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:30 smithi094 bash[27803]: audit 2023-10-09T08:36:29.422464+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.16:0/313553704' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:36:30.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:30 smithi094 bash[27803]: audit 2023-10-09T08:36:29.423048+0000 mon.a (mon.0) 681 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:36:30.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:30 smithi016 bash[27454]: cluster 2023-10-09T08:36:29.419250+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T08:36:30.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:30 smithi016 bash[27454]: audit 2023-10-09T08:36:29.422464+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.16:0/313553704' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:36:30.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:30 smithi016 bash[27454]: audit 2023-10-09T08:36:29.423048+0000 mon.a (mon.0) 681 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:36:30.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:30 smithi016 bash[33851]: cluster 2023-10-09T08:36:29.419250+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T08:36:30.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:30 smithi016 bash[33851]: audit 2023-10-09T08:36:29.422464+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.16:0/313553704' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:36:30.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:30 smithi016 bash[33851]: audit 2023-10-09T08:36:29.423048+0000 mon.a (mon.0) 681 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:36:30.769 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.directory_methods (51 ms) 2023-10-09T08:36:30.769 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.create 2023-10-09T08:36:30.795 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.create (26 ms) 2023-10-09T08:36:30.796 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T08:36:30.799 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2023-10-09T08:36:30.799 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T08:36:30.803 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_object_prefix (4 ms) 2023-10-09T08:36:30.803 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T08:36:30.806 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_create_timestamp (3 ms) 2023-10-09T08:36:30.806 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T08:36:30.809 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_access_timestamp (3 ms) 2023-10-09T08:36:30.809 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T08:36:30.811 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T08:36:30.811 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T08:36:30.818 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_data_pool (8 ms) 2023-10-09T08:36:30.819 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T08:36:30.827 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_size (8 ms) 2023-10-09T08:36:30.827 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T08:36:30.838 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.set_size (11 ms) 2023-10-09T08:36:30.838 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T08:36:30.868 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.protection_status (30 ms) 2023-10-09T08:36:30.868 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T08:36:30.880 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapshot_limits (12 ms) 2023-10-09T08:36:30.880 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T08:36:30.947 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.parents_v1 (67 ms) 2023-10-09T08:36:30.947 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T08:36:31.000 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.parents_v2 (53 ms) 2023-10-09T08:36:31.000 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T08:36:31.045 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapshots (45 ms) 2023-10-09T08:36:31.045 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T08:36:31.053 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapid_race (8 ms) 2023-10-09T08:36:31.053 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T08:36:31.072 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.stripingv2 (19 ms) 2023-10-09T08:36:31.073 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T08:36:31.075 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-10-09T08:36:31.075 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T08:36:31.083 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-10-09T08:36:31.083 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T08:36:31.091 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_update (8 ms) 2023-10-09T08:36:31.091 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T08:36:31.091 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T08:36:31.091 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T08:36:31.098 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-10-09T08:36:31.098 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T08:36:31.106 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-10-09T08:36:31.106 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T08:36:31.116 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.flags (9 ms) 2023-10-09T08:36:31.116 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T08:36:31.167 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.metadata (51 ms) 2023-10-09T08:36:31.167 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T08:36:31.182 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.set_features (15 ms) 2023-10-09T08:36:31.182 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T08:36:31.253 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror (71 ms) 2023-10-09T08:36:31.253 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T08:36:31.281 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image (28 ms) 2023-10-09T08:36:31.281 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T08:36:31.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:31 smithi094 bash[27803]: audit 2023-10-09T08:36:30.430293+0000 mon.a (mon.0) 682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T08:36:31.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:31 smithi094 bash[27803]: cluster 2023-10-09T08:36:30.430594+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T08:36:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:31 smithi094 bash[27803]: cluster 2023-10-09T08:36:30.686555+0000 mgr.y (mgr.24355) 168 : cluster [DBG] pgmap v149: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 512 KiB/s rd, 512 KiB/s wr, 0 op/s 2023-10-09T08:36:31.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:31 smithi016 bash[27454]: audit 2023-10-09T08:36:30.430293+0000 mon.a (mon.0) 682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T08:36:31.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:31 smithi016 bash[27454]: cluster 2023-10-09T08:36:30.430594+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T08:36:31.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:31 smithi016 bash[27454]: cluster 2023-10-09T08:36:30.686555+0000 mgr.y (mgr.24355) 168 : cluster [DBG] pgmap v149: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 512 KiB/s rd, 512 KiB/s wr, 0 op/s 2023-10-09T08:36:31.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:31 smithi016 bash[33851]: audit 2023-10-09T08:36:30.430293+0000 mon.a (mon.0) 682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-66116-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T08:36:31.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:31 smithi016 bash[33851]: cluster 2023-10-09T08:36:30.430594+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T08:36:31.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:31 smithi016 bash[33851]: cluster 2023-10-09T08:36:30.686555+0000 mgr.y (mgr.24355) 168 : cluster [DBG] pgmap v149: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 512 KiB/s rd, 512 KiB/s wr, 0 op/s 2023-10-09T08:36:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:32 smithi094 bash[27803]: cluster 2023-10-09T08:36:31.437515+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T08:36:32.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:36:32 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:36:32.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:32 smithi016 bash[27454]: cluster 2023-10-09T08:36:31.437515+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T08:36:32.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:32 smithi016 bash[33851]: cluster 2023-10-09T08:36:31.437515+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T08:36:32.859 INFO:teuthology.orchestra.run.smithi016.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:36:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:33 smithi094 bash[27803]: cluster 2023-10-09T08:36:32.687124+0000 mgr.y (mgr.24355) 169 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 KiB/s rd, 682 KiB/s wr, 1 op/s 2023-10-09T08:36:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:33 smithi094 bash[27803]: audit 2023-10-09T08:36:32.855158+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:33 smithi094 bash[27803]: audit 2023-10-09T08:36:32.856970+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:33 smithi094 bash[27803]: audit 2023-10-09T08:36:32.859299+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:33 smithi094 bash[27803]: audit 2023-10-09T08:36:32.860573+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:33 smithi094 bash[27803]: audit 2023-10-09T08:36:32.868552+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[27454]: cluster 2023-10-09T08:36:32.687124+0000 mgr.y (mgr.24355) 169 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 KiB/s rd, 682 KiB/s wr, 1 op/s 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[27454]: audit 2023-10-09T08:36:32.855158+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[27454]: audit 2023-10-09T08:36:32.856970+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[27454]: audit 2023-10-09T08:36:32.859299+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[27454]: audit 2023-10-09T08:36:32.860573+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[27454]: audit 2023-10-09T08:36:32.868552+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[33851]: cluster 2023-10-09T08:36:32.687124+0000 mgr.y (mgr.24355) 169 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 KiB/s rd, 682 KiB/s wr, 1 op/s 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[33851]: audit 2023-10-09T08:36:32.855158+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[33851]: audit 2023-10-09T08:36:32.856970+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:36:33.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[33851]: audit 2023-10-09T08:36:32.859299+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:36:33.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[33851]: audit 2023-10-09T08:36:32.860573+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:36:33.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:33 smithi016 bash[33851]: audit 2023-10-09T08:36:32.868552+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:36:33.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T08:36:34.447 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:34 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:34] "GET /metrics HTTP/1.1" 200 191378 "" "Prometheus/2.33.4" 2023-10-09T08:36:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:34 smithi016 bash[27454]: audit 2023-10-09T08:36:32.848568+0000 mgr.y (mgr.24355) 170 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:36:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:34 smithi016 bash[27454]: cephadm 2023-10-09T08:36:32.849772+0000 mgr.y (mgr.24355) 171 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:36:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:34 smithi016 bash[27454]: cephadm 2023-10-09T08:36:32.876697+0000 mgr.y (mgr.24355) 172 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:36:34.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:34 smithi094 bash[27803]: audit 2023-10-09T08:36:32.848568+0000 mgr.y (mgr.24355) 170 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:36:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:34 smithi094 bash[27803]: cephadm 2023-10-09T08:36:32.849772+0000 mgr.y (mgr.24355) 171 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:36:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:34 smithi094 bash[27803]: cephadm 2023-10-09T08:36:32.876697+0000 mgr.y (mgr.24355) 172 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:36:34.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:34 smithi016 bash[33851]: audit 2023-10-09T08:36:32.848568+0000 mgr.y (mgr.24355) 170 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:36:34.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:34 smithi016 bash[33851]: cephadm 2023-10-09T08:36:32.849772+0000 mgr.y (mgr.24355) 171 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:36:34.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:34 smithi016 bash[33851]: cephadm 2023-10-09T08:36:32.876697+0000 mgr.y (mgr.24355) 172 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:36:35.510 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image_status (4229 ms) 2023-10-09T08:36:35.510 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T08:36:35.660 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image_map (150 ms) 2023-10-09T08:36:35.660 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T08:36:35.675 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_instances (15 ms) 2023-10-09T08:36:35.675 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T08:36:35.700 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_snapshot (25 ms) 2023-10-09T08:36:35.701 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T08:36:35.704 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_list (4 ms) 2023-10-09T08:36:35.704 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T08:36:35.708 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-10-09T08:36:35.708 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T08:36:35.713 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.dir_add_already_existing (5 ms) 2023-10-09T08:36:35.713 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T08:36:35.724 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_rename (11 ms) 2023-10-09T08:36:35.724 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T08:36:35.729 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_remove (5 ms) 2023-10-09T08:36:35.729 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T08:36:35.737 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (9 ms) 2023-10-09T08:36:35.738 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T08:36:35.742 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_add (4 ms) 2023-10-09T08:36:35.742 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T08:36:35.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:35 smithi094 bash[27803]: cluster 2023-10-09T08:36:34.687649+0000 mgr.y (mgr.24355) 173 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 KiB/s rd, 682 KiB/s wr, 1 op/s 2023-10-09T08:36:35.747 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_remove (5 ms) 2023-10-09T08:36:35.747 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T08:36:35.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:35 smithi016 bash[27454]: cluster 2023-10-09T08:36:34.687649+0000 mgr.y (mgr.24355) 173 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 KiB/s rd, 682 KiB/s wr, 1 op/s 2023-10-09T08:36:35.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:35 smithi016 bash[33851]: cluster 2023-10-09T08:36:34.687649+0000 mgr.y (mgr.24355) 173 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 KiB/s rd, 682 KiB/s wr, 1 op/s 2023-10-09T08:36:35.752 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_list (5 ms) 2023-10-09T08:36:35.752 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T08:36:35.760 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_clean (8 ms) 2023-10-09T08:36:35.760 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T08:36:35.764 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_add (4 ms) 2023-10-09T08:36:35.764 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T08:36:35.769 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_remove (6 ms) 2023-10-09T08:36:35.770 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T08:36:35.774 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_get (4 ms) 2023-10-09T08:36:35.774 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T08:36:35.776 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-10-09T08:36:35.776 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T08:36:35.779 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-10-09T08:36:35.779 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T08:36:35.784 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (5 ms) 2023-10-09T08:36:35.784 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T08:36:35.790 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (5 ms) 2023-10-09T08:36:35.790 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T08:36:35.793 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-09T08:36:35.793 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T08:36:35.799 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-10-09T08:36:35.799 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T08:36:35.827 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (28 ms) 2023-10-09T08:36:35.827 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T08:36:36.156 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (329 ms) 2023-10-09T08:36:36.157 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T08:36:36.163 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_remove (7 ms) 2023-10-09T08:36:36.163 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T08:36:36.167 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (4 ms) 2023-10-09T08:36:36.167 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T08:36:36.181 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.trash_methods (14 ms) 2023-10-09T08:36:36.181 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T08:36:36.193 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.op_features (13 ms) 2023-10-09T08:36:36.194 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T08:36:36.231 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_parent (37 ms) 2023-10-09T08:36:36.231 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T08:36:36.248 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_parent_ns (17 ms) 2023-10-09T08:36:36.248 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T08:36:36.268 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_child (20 ms) 2023-10-09T08:36:36.269 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T08:36:36.281 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.namespace_methods (13 ms) 2023-10-09T08:36:36.281 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T08:36:36.301 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.migration (20 ms) 2023-10-09T08:36:36.301 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T08:36:36.315 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.migration_v1 (14 ms) 2023-10-09T08:36:36.315 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T08:36:37.470 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1154 ms) 2023-10-09T08:36:37.471 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T08:36:37.514 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.sparsify (45 ms) 2023-10-09T08:36:37.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:37 smithi094 bash[27803]: cluster 2023-10-09T08:36:36.456428+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T08:36:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:37 smithi094 bash[27803]: cluster 2023-10-09T08:36:36.688281+0000 mgr.y (mgr.24355) 174 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 2.14k op/s 2023-10-09T08:36:37.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:37 smithi016 bash[33851]: cluster 2023-10-09T08:36:36.456428+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T08:36:37.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:37 smithi016 bash[33851]: cluster 2023-10-09T08:36:36.688281+0000 mgr.y (mgr.24355) 174 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 2.14k op/s 2023-10-09T08:36:37.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:37 smithi016 bash[27454]: cluster 2023-10-09T08:36:36.456428+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T08:36:37.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:37 smithi016 bash[27454]: cluster 2023-10-09T08:36:36.688281+0000 mgr.y (mgr.24355) 174 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 2.14k op/s 2023-10-09T08:36:38.320 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T08:36:38.320 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (3m) 10s ago 3m 13.0M - 0.23.0 ba2b418f427c 2a6a05418a33 2023-10-09T08:36:38.320 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi094 *:3000 running (3m) 11s ago 3m 43.0M - 8.3.5 dad864ee21e9 3c542cc9ca58 2023-10-09T08:36:38.320 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.ccdktz smithi016 running (2m) 10s ago 2m 13.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 17a3e2c7a98b 2023-10-09T08:36:38.320 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi094.zfssci smithi094 running (2m) 11s ago 2m 11.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a917ff5f77b 2023-10-09T08:36:38.321 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi094 *:8443 running (11m) 11s ago 11m 375M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 43160b07b456 2023-10-09T08:36:38.321 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:9283 running (13m) 10s ago 13m 426M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 73c1c168293b 2023-10-09T08:36:38.321 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (13m) 10s ago 13m 51.5M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 6291e72b78ad 2023-10-09T08:36:38.321 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi094 running (11m) 11s ago 11m 45.8M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5e9c5bdb4c92 2023-10-09T08:36:38.321 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (11m) 10s ago 11m 45.0M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8dd3a38c7bb6 2023-10-09T08:36:38.321 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (4m) 10s ago 4m 9152k - 1.3.1 1dbe0e931976 b5f67dbbfded 2023-10-09T08:36:38.321 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi094 *:9100 running (4m) 11s ago 4m 9079k - 1.3.1 1dbe0e931976 d90758554d9d 2023-10-09T08:36:38.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (10m) 10s ago 10m 40.5M 4061M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c2ec86d01448 2023-10-09T08:36:38.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (9m) 10s ago 9m 37.9M 4061M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 302ee697c26f 2023-10-09T08:36:38.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (9m) 10s ago 9m 38.1M 4061M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c358a7abf935 2023-10-09T08:36:38.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (8m) 10s ago 8m 38.7M 4061M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0f6ebb2bd802 2023-10-09T08:36:38.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi094 running (7m) 11s ago 7m 38.6M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1a20cacc6df3 2023-10-09T08:36:38.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi094 running (6m) 11s ago 6m 37.7M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 381d98a4283c 2023-10-09T08:36:38.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi094 running (6m) 11s ago 6m 37.8M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d8307333e14e 2023-10-09T08:36:38.323 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi094 running (5m) 11s ago 5m 40.5M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 04152ef43de4 2023-10-09T08:36:38.323 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi094 *:9095 running (2m) 11s ago 4m 52.9M - 2.33.4 514e6a882f6e caf9b861583e 2023-10-09T08:36:38.482 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] 70 tests from TestClsRbd (7068 ms total) 2023-10-09T08:36:38.483 INFO:tasks.workunit.client.0.smithi016.stdout: 2023-10-09T08:36:38.483 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] Global test environment tear-down 2023-10-09T08:36:38.483 INFO:tasks.workunit.client.0.smithi016.stdout:[==========] 70 tests from 1 test suite ran. (10086 ms total) 2023-10-09T08:36:38.483 INFO:tasks.workunit.client.0.smithi016.stdout:[ PASSED ] 70 tests. 2023-10-09T08:36:38.483 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T08:36:38.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T08:36:38.494 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T08:36:38.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T08:36:38.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:38 smithi094 bash[27803]: cluster 2023-10-09T08:36:37.465408+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T08:36:38.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:38 smithi016 bash[27454]: cluster 2023-10-09T08:36:37.465408+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T08:36:38.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:38 smithi016 bash[33851]: cluster 2023-10-09T08:36:37.465408+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T08:36:38.838 DEBUG:teuthology.parallel:result is None 2023-10-09T08:36:38.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T08:36:38.849 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T08:36:38.850 DEBUG:teuthology.orchestra.run.smithi016:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T08:36:39.126 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T08:36:39.126 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T08:36:39.126 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T08:36:39.127 DEBUG:teuthology.parallel:result is None 2023-10-09T08:36:39.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T08:36:39.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:39 smithi094 bash[27803]: audit 2023-10-09T08:36:38.298049+0000 mgr.y (mgr.24355) 175 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:36:39.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:39 smithi094 bash[27803]: cluster 2023-10-09T08:36:38.471197+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T08:36:39.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:39 smithi094 bash[27803]: cluster 2023-10-09T08:36:38.688694+0000 mgr.y (mgr.24355) 176 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:39.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:39 smithi016 bash[27454]: audit 2023-10-09T08:36:38.298049+0000 mgr.y (mgr.24355) 175 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:36:39.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:39 smithi016 bash[27454]: cluster 2023-10-09T08:36:38.471197+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T08:36:39.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:39 smithi016 bash[27454]: cluster 2023-10-09T08:36:38.688694+0000 mgr.y (mgr.24355) 176 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:39 smithi016 bash[33851]: audit 2023-10-09T08:36:38.298049+0000 mgr.y (mgr.24355) 175 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:36:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:39 smithi016 bash[33851]: cluster 2023-10-09T08:36:38.471197+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T08:36:39.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:39 smithi016 bash[33851]: cluster 2023-10-09T08:36:38.688694+0000 mgr.y (mgr.24355) 176 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:41.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:41 smithi016 bash[27454]: cluster 2023-10-09T08:36:40.689387+0000 mgr.y (mgr.24355) 177 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:41.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:41 smithi016 bash[33851]: cluster 2023-10-09T08:36:40.689387+0000 mgr.y (mgr.24355) 177 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:41.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:41 smithi094 bash[27803]: cluster 2023-10-09T08:36:40.689387+0000 mgr.y (mgr.24355) 177 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:42.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:36:42 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:36:43.039 INFO:teuthology.orchestra.run.smithi016.stdout:wait for minority of mons to upgrade 2023-10-09T08:36:43.957 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:43 smithi016 bash[27454]: cluster 2023-10-09T08:36:42.690030+0000 mgr.y (mgr.24355) 178 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:43.958 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:43 smithi016 bash[33851]: cluster 2023-10-09T08:36:42.690030+0000 mgr.y (mgr.24355) 178 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:43.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T08:36:44.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:43 smithi094 bash[27803]: cluster 2023-10-09T08:36:42.690030+0000 mgr.y (mgr.24355) 178 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:44.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:44 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:44] "GET /metrics HTTP/1.1" 200 193959 "" "Prometheus/2.33.4" 2023-10-09T08:36:46.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:45 smithi016 bash[27454]: cluster 2023-10-09T08:36:44.690581+0000 mgr.y (mgr.24355) 179 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:46.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:45 smithi016 bash[33851]: cluster 2023-10-09T08:36:44.690581+0000 mgr.y (mgr.24355) 179 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:45 smithi094 bash[27803]: cluster 2023-10-09T08:36:44.690581+0000 mgr.y (mgr.24355) 179 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:47 smithi094 bash[27803]: cluster 2023-10-09T08:36:46.692159+0000 mgr.y (mgr.24355) 180 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:47 smithi016 bash[27454]: cluster 2023-10-09T08:36:46.692159+0000 mgr.y (mgr.24355) 180 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:48.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:47 smithi016 bash[33851]: cluster 2023-10-09T08:36:46.692159+0000 mgr.y (mgr.24355) 180 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:49.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:48 smithi094 bash[27803]: audit 2023-10-09T08:36:47.897559+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.16:0/3340716070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:49.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:48 smithi016 bash[27454]: audit 2023-10-09T08:36:47.897559+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.16:0/3340716070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:49.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:48 smithi016 bash[33851]: audit 2023-10-09T08:36:47.897559+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.16:0/3340716070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:49 smithi094 bash[27803]: cluster 2023-10-09T08:36:48.692890+0000 mgr.y (mgr.24355) 181 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:50.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:49 smithi016 bash[27454]: cluster 2023-10-09T08:36:48.692890+0000 mgr.y (mgr.24355) 181 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:50.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:49 smithi016 bash[33851]: cluster 2023-10-09T08:36:48.692890+0000 mgr.y (mgr.24355) 181 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:51 smithi094 bash[27803]: audit 2023-10-09T08:36:50.381735+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.16:0/2173254045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:51 smithi094 bash[27803]: cluster 2023-10-09T08:36:50.693893+0000 mgr.y (mgr.24355) 182 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:51 smithi016 bash[27454]: audit 2023-10-09T08:36:50.381735+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.16:0/2173254045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:51 smithi016 bash[27454]: cluster 2023-10-09T08:36:50.693893+0000 mgr.y (mgr.24355) 182 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:51 smithi016 bash[33851]: audit 2023-10-09T08:36:50.381735+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.16:0/2173254045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:51 smithi016 bash[33851]: cluster 2023-10-09T08:36:50.693893+0000 mgr.y (mgr.24355) 182 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:52.994 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:36:52 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:36:54.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:53 smithi016 bash[27454]: cluster 2023-10-09T08:36:52.694398+0000 mgr.y (mgr.24355) 183 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:54.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:53 smithi016 bash[27454]: audit 2023-10-09T08:36:52.885093+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.16:0/133991219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:54.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:53 smithi016 bash[33851]: cluster 2023-10-09T08:36:52.694398+0000 mgr.y (mgr.24355) 183 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:54.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:53 smithi016 bash[33851]: audit 2023-10-09T08:36:52.885093+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.16:0/133991219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:54.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:53 smithi094 bash[27803]: cluster 2023-10-09T08:36:52.694398+0000 mgr.y (mgr.24355) 183 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:54.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:53 smithi094 bash[27803]: audit 2023-10-09T08:36:52.885093+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.16:0/133991219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:54.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:36:54 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:36:54] "GET /metrics HTTP/1.1" 200 192261 "" "Prometheus/2.33.4" 2023-10-09T08:36:56.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:55 smithi094 bash[27803]: cluster 2023-10-09T08:36:54.694908+0000 mgr.y (mgr.24355) 184 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:56.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:55 smithi094 bash[27803]: audit 2023-10-09T08:36:55.352695+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.16:0/2974034097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:56.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:55 smithi016 bash[27454]: cluster 2023-10-09T08:36:54.694908+0000 mgr.y (mgr.24355) 184 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:56.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:55 smithi016 bash[27454]: audit 2023-10-09T08:36:55.352695+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.16:0/2974034097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:56.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:55 smithi016 bash[33851]: cluster 2023-10-09T08:36:54.694908+0000 mgr.y (mgr.24355) 184 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:56.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:55 smithi016 bash[33851]: audit 2023-10-09T08:36:55.352695+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.16:0/2974034097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:58.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:57 smithi094 bash[27803]: cluster 2023-10-09T08:36:56.695900+0000 mgr.y (mgr.24355) 185 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:58.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:57 smithi016 bash[27454]: cluster 2023-10-09T08:36:56.695900+0000 mgr.y (mgr.24355) 185 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:58.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:57 smithi016 bash[33851]: cluster 2023-10-09T08:36:56.695900+0000 mgr.y (mgr.24355) 185 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:36:59.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:58 smithi094 bash[27803]: audit 2023-10-09T08:36:57.847614+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.16:0/656860687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:59.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:58 smithi016 bash[33851]: audit 2023-10-09T08:36:57.847614+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.16:0/656860687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:36:59.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:58 smithi016 bash[27454]: audit 2023-10-09T08:36:57.847614+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.16:0/656860687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:00.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:36:59 smithi094 bash[27803]: cluster 2023-10-09T08:36:58.696471+0000 mgr.y (mgr.24355) 186 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:00.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:36:59 smithi016 bash[27454]: cluster 2023-10-09T08:36:58.696471+0000 mgr.y (mgr.24355) 186 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:00.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:36:59 smithi016 bash[33851]: cluster 2023-10-09T08:36:58.696471+0000 mgr.y (mgr.24355) 186 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:01.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:01 smithi094 bash[27803]: audit 2023-10-09T08:37:00.322691+0000 mon.a (mon.0) 693 : audit [DBG] from='client.? 172.21.15.16:0/3914748041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:01.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:01 smithi094 bash[27803]: cluster 2023-10-09T08:37:00.697530+0000 mgr.y (mgr.24355) 187 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:01.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:01 smithi016 bash[33851]: audit 2023-10-09T08:37:00.322691+0000 mon.a (mon.0) 693 : audit [DBG] from='client.? 172.21.15.16:0/3914748041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:01.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:01 smithi016 bash[33851]: cluster 2023-10-09T08:37:00.697530+0000 mgr.y (mgr.24355) 187 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:01.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:01 smithi016 bash[27454]: audit 2023-10-09T08:37:00.322691+0000 mon.a (mon.0) 693 : audit [DBG] from='client.? 172.21.15.16:0/3914748041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:01.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:01 smithi016 bash[27454]: cluster 2023-10-09T08:37:00.697530+0000 mgr.y (mgr.24355) 187 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:02.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:02 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:37:04.114 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:03 smithi016 bash[27454]: cluster 2023-10-09T08:37:02.698013+0000 mgr.y (mgr.24355) 188 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:04.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:03 smithi016 bash[27454]: audit 2023-10-09T08:37:02.824491+0000 mon.a (mon.0) 694 : audit [DBG] from='client.? 172.21.15.16:0/3930489358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:04.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:03 smithi016 bash[33851]: cluster 2023-10-09T08:37:02.698013+0000 mgr.y (mgr.24355) 188 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:04.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:03 smithi016 bash[33851]: audit 2023-10-09T08:37:02.824491+0000 mon.a (mon.0) 694 : audit [DBG] from='client.? 172.21.15.16:0/3930489358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:04.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:03 smithi094 bash[27803]: cluster 2023-10-09T08:37:02.698013+0000 mgr.y (mgr.24355) 188 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:04.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:03 smithi094 bash[27803]: audit 2023-10-09T08:37:02.824491+0000 mon.a (mon.0) 694 : audit [DBG] from='client.? 172.21.15.16:0/3930489358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:04.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:37:04 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:04] "GET /metrics HTTP/1.1" 200 192261 "" "Prometheus/2.33.4" 2023-10-09T08:37:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:05 smithi094 bash[27803]: cluster 2023-10-09T08:37:04.698813+0000 mgr.y (mgr.24355) 189 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:05 smithi094 bash[27803]: audit 2023-10-09T08:37:05.232090+0000 mon.a (mon.0) 695 : audit [DBG] from='client.? 172.21.15.16:0/521982781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:06.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:05 smithi016 bash[27454]: cluster 2023-10-09T08:37:04.698813+0000 mgr.y (mgr.24355) 189 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:06.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:05 smithi016 bash[27454]: audit 2023-10-09T08:37:05.232090+0000 mon.a (mon.0) 695 : audit [DBG] from='client.? 172.21.15.16:0/521982781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:06.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:05 smithi016 bash[33851]: cluster 2023-10-09T08:37:04.698813+0000 mgr.y (mgr.24355) 189 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:06.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:05 smithi016 bash[33851]: audit 2023-10-09T08:37:05.232090+0000 mon.a (mon.0) 695 : audit [DBG] from='client.? 172.21.15.16:0/521982781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:07 smithi094 bash[27803]: cluster 2023-10-09T08:37:06.699825+0000 mgr.y (mgr.24355) 190 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:07 smithi094 bash[27803]: audit 2023-10-09T08:37:07.646610+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.16:0/411544089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:08.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:07 smithi016 bash[27454]: cluster 2023-10-09T08:37:06.699825+0000 mgr.y (mgr.24355) 190 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:08.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:07 smithi016 bash[27454]: audit 2023-10-09T08:37:07.646610+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.16:0/411544089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:08.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:07 smithi016 bash[33851]: cluster 2023-10-09T08:37:06.699825+0000 mgr.y (mgr.24355) 190 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:08.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:07 smithi016 bash[33851]: audit 2023-10-09T08:37:07.646610+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.16:0/411544089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:10.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:09 smithi094 bash[27803]: cluster 2023-10-09T08:37:08.700472+0000 mgr.y (mgr.24355) 191 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:10.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:09 smithi016 bash[27454]: cluster 2023-10-09T08:37:08.700472+0000 mgr.y (mgr.24355) 191 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:10.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:09 smithi016 bash[33851]: cluster 2023-10-09T08:37:08.700472+0000 mgr.y (mgr.24355) 191 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:11.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:10 smithi094 bash[27803]: audit 2023-10-09T08:37:10.045848+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.16:0/1953551632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:11.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:10 smithi016 bash[27454]: audit 2023-10-09T08:37:10.045848+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.16:0/1953551632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:11.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:10 smithi016 bash[33851]: audit 2023-10-09T08:37:10.045848+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.16:0/1953551632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:12.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:11 smithi094 bash[27803]: cluster 2023-10-09T08:37:10.701482+0000 mgr.y (mgr.24355) 192 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:11 smithi016 bash[27454]: cluster 2023-10-09T08:37:10.701482+0000 mgr.y (mgr.24355) 192 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:12.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:11 smithi016 bash[33851]: cluster 2023-10-09T08:37:10.701482+0000 mgr.y (mgr.24355) 192 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:12.785 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:12 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:37:13.164 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:12 smithi016 bash[27454]: audit 2023-10-09T08:37:12.435797+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.16:0/4021633427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:13.165 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:12 smithi016 bash[33851]: audit 2023-10-09T08:37:12.435797+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.16:0/4021633427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:12 smithi094 bash[27803]: audit 2023-10-09T08:37:12.435797+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.16:0/4021633427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:14.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:13 smithi016 bash[27454]: cluster 2023-10-09T08:37:12.702070+0000 mgr.y (mgr.24355) 193 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:14.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:13 smithi016 bash[33851]: cluster 2023-10-09T08:37:12.702070+0000 mgr.y (mgr.24355) 193 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:14.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:13 smithi094 bash[27803]: cluster 2023-10-09T08:37:12.702070+0000 mgr.y (mgr.24355) 193 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:14.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:37:14 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:14] "GET /metrics HTTP/1.1" 200 192264 "" "Prometheus/2.33.4" 2023-10-09T08:37:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: audit 2023-10-09T08:37:14.331476+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: cephadm 2023-10-09T08:37:14.332897+0000 mgr.y (mgr.24355) 194 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: cephadm 2023-10-09T08:37:14.333012+0000 mgr.y (mgr.24355) 195 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: audit 2023-10-09T08:37:14.334348+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: audit 2023-10-09T08:37:14.341566+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: cephadm 2023-10-09T08:37:14.343086+0000 mgr.y (mgr.24355) 196 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: cephadm 2023-10-09T08:37:14.564772+0000 mgr.y (mgr.24355) 197 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi094 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: cluster 2023-10-09T08:37:14.702690+0000 mgr.y (mgr.24355) 198 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:15 smithi094 bash[27803]: audit 2023-10-09T08:37:14.851220+0000 mon.a (mon.0) 698 : audit [DBG] from='client.? 172.21.15.16:0/4275664226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: audit 2023-10-09T08:37:14.331476+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: cephadm 2023-10-09T08:37:14.332897+0000 mgr.y (mgr.24355) 194 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: cephadm 2023-10-09T08:37:14.333012+0000 mgr.y (mgr.24355) 195 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: audit 2023-10-09T08:37:14.334348+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: audit 2023-10-09T08:37:14.341566+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: cephadm 2023-10-09T08:37:14.343086+0000 mgr.y (mgr.24355) 196 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: cephadm 2023-10-09T08:37:14.564772+0000 mgr.y (mgr.24355) 197 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi094 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: cluster 2023-10-09T08:37:14.702690+0000 mgr.y (mgr.24355) 198 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[27454]: audit 2023-10-09T08:37:14.851220+0000 mon.a (mon.0) 698 : audit [DBG] from='client.? 172.21.15.16:0/4275664226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: audit 2023-10-09T08:37:14.331476+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: cephadm 2023-10-09T08:37:14.332897+0000 mgr.y (mgr.24355) 194 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: cephadm 2023-10-09T08:37:14.333012+0000 mgr.y (mgr.24355) 195 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: audit 2023-10-09T08:37:14.334348+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: audit 2023-10-09T08:37:14.341566+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: cephadm 2023-10-09T08:37:14.343086+0000 mgr.y (mgr.24355) 196 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: cephadm 2023-10-09T08:37:14.564772+0000 mgr.y (mgr.24355) 197 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi094 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: cluster 2023-10-09T08:37:14.702690+0000 mgr.y (mgr.24355) 198 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:15.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:15 smithi016 bash[33851]: audit 2023-10-09T08:37:14.851220+0000 mon.a (mon.0) 698 : audit [DBG] from='client.? 172.21.15.16:0/4275664226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:18.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:17 smithi094 bash[27803]: cluster 2023-10-09T08:37:16.703745+0000 mgr.y (mgr.24355) 199 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:18.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:17 smithi094 bash[27803]: audit 2023-10-09T08:37:17.275674+0000 mon.a (mon.0) 699 : audit [DBG] from='client.? 172.21.15.16:0/960469786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:18.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:17 smithi016 bash[27454]: cluster 2023-10-09T08:37:16.703745+0000 mgr.y (mgr.24355) 199 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:18.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:17 smithi016 bash[27454]: audit 2023-10-09T08:37:17.275674+0000 mon.a (mon.0) 699 : audit [DBG] from='client.? 172.21.15.16:0/960469786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:18.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:17 smithi016 bash[33851]: cluster 2023-10-09T08:37:16.703745+0000 mgr.y (mgr.24355) 199 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:18.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:17 smithi016 bash[33851]: audit 2023-10-09T08:37:17.275674+0000 mon.a (mon.0) 699 : audit [DBG] from='client.? 172.21.15.16:0/960469786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:20.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:19 smithi094 bash[27803]: cluster 2023-10-09T08:37:18.704402+0000 mgr.y (mgr.24355) 200 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:19 smithi094 bash[27803]: audit 2023-10-09T08:37:19.684486+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.16:0/1300347259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:20.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:19 smithi016 bash[27454]: cluster 2023-10-09T08:37:18.704402+0000 mgr.y (mgr.24355) 200 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:20.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:19 smithi016 bash[27454]: audit 2023-10-09T08:37:19.684486+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.16:0/1300347259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:20.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:19 smithi016 bash[33851]: cluster 2023-10-09T08:37:18.704402+0000 mgr.y (mgr.24355) 200 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:20.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:19 smithi016 bash[33851]: audit 2023-10-09T08:37:19.684486+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.16:0/1300347259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:21 smithi094 bash[27803]: cluster 2023-10-09T08:37:20.705456+0000 mgr.y (mgr.24355) 201 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:21.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:21 smithi016 bash[27454]: cluster 2023-10-09T08:37:20.705456+0000 mgr.y (mgr.24355) 201 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:21.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:21 smithi016 bash[33851]: cluster 2023-10-09T08:37:20.705456+0000 mgr.y (mgr.24355) 201 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:22.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:22 smithi094 bash[27803]: audit 2023-10-09T08:37:22.099794+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.16:0/3389935627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:22.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:22 smithi016 bash[27454]: audit 2023-10-09T08:37:22.099794+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.16:0/3389935627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:22.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:22 smithi016 bash[33851]: audit 2023-10-09T08:37:22.099794+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.16:0/3389935627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:22.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:22 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:37:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:23 smithi094 bash[27803]: cluster 2023-10-09T08:37:22.706083+0000 mgr.y (mgr.24355) 202 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:23.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:23 smithi016 bash[27454]: cluster 2023-10-09T08:37:22.706083+0000 mgr.y (mgr.24355) 202 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:23.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:23 smithi016 bash[33851]: cluster 2023-10-09T08:37:22.706083+0000 mgr.y (mgr.24355) 202 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:24.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:37:24 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:24] "GET /metrics HTTP/1.1" 200 192264 "" "Prometheus/2.33.4" 2023-10-09T08:37:24.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:24 smithi094 bash[27803]: audit 2023-10-09T08:37:24.521158+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.16:0/1881447391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:25.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:24 smithi016 bash[27454]: audit 2023-10-09T08:37:24.521158+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.16:0/1881447391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:25.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:24 smithi016 bash[33851]: audit 2023-10-09T08:37:24.521158+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.16:0/1881447391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:25.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:25 smithi094 bash[27803]: cluster 2023-10-09T08:37:24.706819+0000 mgr.y (mgr.24355) 203 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:25 smithi016 bash[27454]: cluster 2023-10-09T08:37:24.706819+0000 mgr.y (mgr.24355) 203 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:26.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:25 smithi016 bash[33851]: cluster 2023-10-09T08:37:24.706819+0000 mgr.y (mgr.24355) 203 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:28.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:27 smithi094 bash[27803]: cluster 2023-10-09T08:37:26.707890+0000 mgr.y (mgr.24355) 204 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:28.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:27 smithi094 bash[27803]: audit 2023-10-09T08:37:26.935557+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.16:0/2837265626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:28.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:27 smithi016 bash[33851]: cluster 2023-10-09T08:37:26.707890+0000 mgr.y (mgr.24355) 204 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:28.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:27 smithi016 bash[33851]: audit 2023-10-09T08:37:26.935557+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.16:0/2837265626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:28.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:27 smithi016 bash[27454]: cluster 2023-10-09T08:37:26.707890+0000 mgr.y (mgr.24355) 204 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:28.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:27 smithi016 bash[27454]: audit 2023-10-09T08:37:26.935557+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.16:0/2837265626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:30.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:29 smithi094 bash[27803]: cluster 2023-10-09T08:37:28.708519+0000 mgr.y (mgr.24355) 205 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:30.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:29 smithi094 bash[27803]: audit 2023-10-09T08:37:29.348923+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.16:0/2992923991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:30.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:29 smithi016 bash[27454]: cluster 2023-10-09T08:37:28.708519+0000 mgr.y (mgr.24355) 205 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:30.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:29 smithi016 bash[27454]: audit 2023-10-09T08:37:29.348923+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.16:0/2992923991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:30.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:29 smithi016 bash[33851]: cluster 2023-10-09T08:37:28.708519+0000 mgr.y (mgr.24355) 205 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:30.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:29 smithi016 bash[33851]: audit 2023-10-09T08:37:29.348923+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.16:0/2992923991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:31 smithi094 bash[27803]: cluster 2023-10-09T08:37:30.709607+0000 mgr.y (mgr.24355) 206 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:31.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:31 smithi016 bash[27454]: cluster 2023-10-09T08:37:30.709607+0000 mgr.y (mgr.24355) 206 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:31.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:31 smithi016 bash[33851]: cluster 2023-10-09T08:37:30.709607+0000 mgr.y (mgr.24355) 206 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:32.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:32 smithi094 bash[27803]: audit 2023-10-09T08:37:31.769369+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.16:0/4062121242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:32.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:32 smithi016 bash[27454]: audit 2023-10-09T08:37:31.769369+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.16:0/4062121242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:32.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:32 smithi016 bash[33851]: audit 2023-10-09T08:37:31.769369+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.16:0/4062121242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:32.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:32 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:37:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:33 smithi094 bash[27803]: cluster 2023-10-09T08:37:32.710224+0000 mgr.y (mgr.24355) 207 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:33.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:33 smithi016 bash[27454]: cluster 2023-10-09T08:37:32.710224+0000 mgr.y (mgr.24355) 207 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:33.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:33 smithi016 bash[33851]: cluster 2023-10-09T08:37:32.710224+0000 mgr.y (mgr.24355) 207 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:34 smithi094 bash[27803]: audit 2023-10-09T08:37:34.189691+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.16:0/3831855670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:34.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:34 smithi016 bash[27454]: audit 2023-10-09T08:37:34.189691+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.16:0/3831855670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:34.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:37:34 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:34] "GET /metrics HTTP/1.1" 200 192264 "" "Prometheus/2.33.4" 2023-10-09T08:37:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:34 smithi016 bash[33851]: audit 2023-10-09T08:37:34.189691+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.16:0/3831855670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:35 smithi094 bash[27803]: cluster 2023-10-09T08:37:34.710862+0000 mgr.y (mgr.24355) 208 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:35.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:35 smithi016 bash[27454]: cluster 2023-10-09T08:37:34.710862+0000 mgr.y (mgr.24355) 208 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:35.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:35 smithi016 bash[33851]: cluster 2023-10-09T08:37:34.710862+0000 mgr.y (mgr.24355) 208 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:36.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:36 smithi094 bash[27803]: audit 2023-10-09T08:37:36.610812+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.16:0/3075416328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:37.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:36 smithi016 bash[33851]: audit 2023-10-09T08:37:36.610812+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.16:0/3075416328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:37.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:36 smithi016 bash[27454]: audit 2023-10-09T08:37:36.610812+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.16:0/3075416328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:37.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:37 smithi094 bash[27803]: cluster 2023-10-09T08:37:36.711990+0000 mgr.y (mgr.24355) 209 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:38.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:37 smithi016 bash[33851]: cluster 2023-10-09T08:37:36.711990+0000 mgr.y (mgr.24355) 209 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:38.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:37 smithi016 bash[27454]: cluster 2023-10-09T08:37:36.711990+0000 mgr.y (mgr.24355) 209 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:39 smithi094 bash[27803]: cluster 2023-10-09T08:37:38.712604+0000 mgr.y (mgr.24355) 210 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:40.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:39 smithi094 bash[27803]: audit 2023-10-09T08:37:39.022615+0000 mon.a (mon.0) 703 : audit [DBG] from='client.? 172.21.15.16:0/775088018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:40.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:39 smithi016 bash[27454]: cluster 2023-10-09T08:37:38.712604+0000 mgr.y (mgr.24355) 210 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:40.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:39 smithi016 bash[27454]: audit 2023-10-09T08:37:39.022615+0000 mon.a (mon.0) 703 : audit [DBG] from='client.? 172.21.15.16:0/775088018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:40.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:39 smithi016 bash[33851]: cluster 2023-10-09T08:37:38.712604+0000 mgr.y (mgr.24355) 210 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:40.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:39 smithi016 bash[33851]: audit 2023-10-09T08:37:39.022615+0000 mon.a (mon.0) 703 : audit [DBG] from='client.? 172.21.15.16:0/775088018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:41.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:41 smithi094 bash[27803]: cluster 2023-10-09T08:37:40.713639+0000 mgr.y (mgr.24355) 211 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:41 smithi016 bash[27454]: cluster 2023-10-09T08:37:40.713639+0000 mgr.y (mgr.24355) 211 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:41.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:41 smithi016 bash[33851]: cluster 2023-10-09T08:37:40.713639+0000 mgr.y (mgr.24355) 211 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:42.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:42 smithi094 bash[27803]: audit 2023-10-09T08:37:41.450208+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.16:0/429220211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:42.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:42 smithi016 bash[27454]: audit 2023-10-09T08:37:41.450208+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.16:0/429220211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:42.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:42 smithi016 bash[33851]: audit 2023-10-09T08:37:41.450208+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.16:0/429220211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:42.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:42 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:37:43.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:43 smithi094 bash[27803]: cluster 2023-10-09T08:37:42.714306+0000 mgr.y (mgr.24355) 212 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:43.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:43 smithi016 bash[33851]: cluster 2023-10-09T08:37:42.714306+0000 mgr.y (mgr.24355) 212 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:43.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:43 smithi016 bash[27454]: cluster 2023-10-09T08:37:42.714306+0000 mgr.y (mgr.24355) 212 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:44.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:44 smithi094 bash[27803]: audit 2023-10-09T08:37:43.863922+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.16:0/2918754417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:44.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:44 smithi016 bash[27454]: audit 2023-10-09T08:37:43.863922+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.16:0/2918754417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:44.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:37:44 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:44] "GET /metrics HTTP/1.1" 200 192258 "" "Prometheus/2.33.4" 2023-10-09T08:37:44.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:44 smithi016 bash[33851]: audit 2023-10-09T08:37:43.863922+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.16:0/2918754417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:45 smithi094 bash[27803]: cluster 2023-10-09T08:37:44.714953+0000 mgr.y (mgr.24355) 213 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:45.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:45 smithi016 bash[27454]: cluster 2023-10-09T08:37:44.714953+0000 mgr.y (mgr.24355) 213 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:45.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:45 smithi016 bash[33851]: cluster 2023-10-09T08:37:44.714953+0000 mgr.y (mgr.24355) 213 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:46.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:46 smithi094 bash[27803]: audit 2023-10-09T08:37:46.278459+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.16:0/209037536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:46.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:46 smithi016 bash[27454]: audit 2023-10-09T08:37:46.278459+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.16:0/209037536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:46.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:46 smithi016 bash[33851]: audit 2023-10-09T08:37:46.278459+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.16:0/209037536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:47 smithi094 bash[27803]: cluster 2023-10-09T08:37:46.716084+0000 mgr.y (mgr.24355) 214 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:47.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:47 smithi016 bash[27454]: cluster 2023-10-09T08:37:46.716084+0000 mgr.y (mgr.24355) 214 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:47.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:47 smithi016 bash[33851]: cluster 2023-10-09T08:37:46.716084+0000 mgr.y (mgr.24355) 214 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:49.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:48 smithi016 bash[27454]: audit 2023-10-09T08:37:48.694898+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.16:0/3246095466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:49.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:48 smithi016 bash[33851]: audit 2023-10-09T08:37:48.694898+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.16:0/3246095466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:49.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:48 smithi094 bash[27803]: audit 2023-10-09T08:37:48.694898+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.16:0/3246095466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:49 smithi094 bash[27803]: cluster 2023-10-09T08:37:48.716625+0000 mgr.y (mgr.24355) 215 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:50.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:49 smithi016 bash[27454]: cluster 2023-10-09T08:37:48.716625+0000 mgr.y (mgr.24355) 215 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:50.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:49 smithi016 bash[33851]: cluster 2023-10-09T08:37:48.716625+0000 mgr.y (mgr.24355) 215 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:51 smithi094 bash[27803]: cluster 2023-10-09T08:37:50.717740+0000 mgr.y (mgr.24355) 216 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:51 smithi094 bash[27803]: audit 2023-10-09T08:37:51.108434+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.16:0/1734301951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:51 smithi016 bash[27454]: cluster 2023-10-09T08:37:50.717740+0000 mgr.y (mgr.24355) 216 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:51 smithi016 bash[27454]: audit 2023-10-09T08:37:51.108434+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.16:0/1734301951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:51 smithi016 bash[33851]: cluster 2023-10-09T08:37:50.717740+0000 mgr.y (mgr.24355) 216 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:51 smithi016 bash[33851]: audit 2023-10-09T08:37:51.108434+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.16:0/1734301951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:52.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:52 smithi094 bash[28656]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:37:54.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:53 smithi016 bash[27454]: cluster 2023-10-09T08:37:52.718439+0000 mgr.y (mgr.24355) 217 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:54.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:53 smithi016 bash[27454]: audit 2023-10-09T08:37:53.521025+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.16:0/504232993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:54.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:53 smithi016 bash[33851]: cluster 2023-10-09T08:37:52.718439+0000 mgr.y (mgr.24355) 217 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:54.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:53 smithi016 bash[33851]: audit 2023-10-09T08:37:53.521025+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.16:0/504232993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:54.237 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:53 smithi094 bash[27803]: cluster 2023-10-09T08:37:52.718439+0000 mgr.y (mgr.24355) 217 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:54.237 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:53 smithi094 bash[27803]: audit 2023-10-09T08:37:53.521025+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.16:0/504232993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:54.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:37:54 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:37:54] "GET /metrics HTTP/1.1" 200 192255 "" "Prometheus/2.33.4" 2023-10-09T08:37:55.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:55 smithi094 bash[27803]: cluster 2023-10-09T08:37:54.719091+0000 mgr.y (mgr.24355) 218 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:55.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:55 smithi094 bash[27803]: audit 2023-10-09T08:37:55.385414+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:55.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:55 smithi094 bash[27803]: audit 2023-10-09T08:37:55.387390+0000 mon.c (mon.2) 117 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:37:55.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:55 smithi094 bash[27803]: audit 2023-10-09T08:37:55.388101+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:37:55.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:55 smithi094 bash[27803]: audit 2023-10-09T08:37:55.389927+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:37:55.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:55 smithi094 bash[27803]: audit 2023-10-09T08:37:55.391449+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:37:56.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[27454]: cluster 2023-10-09T08:37:54.719091+0000 mgr.y (mgr.24355) 218 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:56.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[27454]: audit 2023-10-09T08:37:55.385414+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:56.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[27454]: audit 2023-10-09T08:37:55.387390+0000 mon.c (mon.2) 117 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:37:56.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[27454]: audit 2023-10-09T08:37:55.388101+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:37:56.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[27454]: audit 2023-10-09T08:37:55.389927+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:37:56.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[27454]: audit 2023-10-09T08:37:55.391449+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:37:56.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[33851]: cluster 2023-10-09T08:37:54.719091+0000 mgr.y (mgr.24355) 218 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:56.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[33851]: audit 2023-10-09T08:37:55.385414+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:37:56.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[33851]: audit 2023-10-09T08:37:55.387390+0000 mon.c (mon.2) 117 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:37:56.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[33851]: audit 2023-10-09T08:37:55.388101+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:37:56.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[33851]: audit 2023-10-09T08:37:55.389927+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:37:56.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:55 smithi016 bash[33851]: audit 2023-10-09T08:37:55.391449+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:37:57.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:56 smithi094 bash[27803]: cephadm 2023-10-09T08:37:55.376383+0000 mgr.y (mgr.24355) 219 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T08:37:57.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:56 smithi094 bash[27803]: cephadm 2023-10-09T08:37:55.392674+0000 mgr.y (mgr.24355) 220 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:37:57.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:56 smithi094 bash[27803]: audit 2023-10-09T08:37:55.947299+0000 mon.a (mon.0) 708 : audit [DBG] from='client.? 172.21.15.16:0/3488333493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:57.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:56 smithi016 bash[27454]: cephadm 2023-10-09T08:37:55.376383+0000 mgr.y (mgr.24355) 219 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T08:37:57.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:56 smithi016 bash[27454]: cephadm 2023-10-09T08:37:55.392674+0000 mgr.y (mgr.24355) 220 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:37:57.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:56 smithi016 bash[27454]: audit 2023-10-09T08:37:55.947299+0000 mon.a (mon.0) 708 : audit [DBG] from='client.? 172.21.15.16:0/3488333493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:57.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:56 smithi016 bash[33851]: cephadm 2023-10-09T08:37:55.376383+0000 mgr.y (mgr.24355) 219 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T08:37:57.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:56 smithi016 bash[33851]: cephadm 2023-10-09T08:37:55.392674+0000 mgr.y (mgr.24355) 220 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:37:57.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:56 smithi016 bash[33851]: audit 2023-10-09T08:37:55.947299+0000 mon.a (mon.0) 708 : audit [DBG] from='client.? 172.21.15.16:0/3488333493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:57.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:57 smithi094 systemd[1]: Stopping Ceph mgr.x for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:37:57.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:57 smithi094 bash[50558]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr.x 2023-10-09T08:37:58.062 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:57 smithi094 bash[27803]: cluster 2023-10-09T08:37:56.720255+0000 mgr.y (mgr.24355) 221 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:58.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:57 smithi016 bash[27454]: cluster 2023-10-09T08:37:56.720255+0000 mgr.y (mgr.24355) 221 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:58.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:57 smithi016 bash[33851]: cluster 2023-10-09T08:37:56.720255+0000 mgr.y (mgr.24355) 221 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:37:58.610 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:58 smithi094 bash[50594]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T08:37:58.611 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:58 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T08:37:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:58 smithi094 bash[27803]: audit 2023-10-09T08:37:58.369579+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.16:0/753775823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:58.865 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:58 smithi094 bash[50657]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T08:37:58.866 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:58 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T08:37:58.866 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:58 smithi094 systemd[1]: Stopped Ceph mgr.x for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:37:58.866 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:37:58 smithi094 systemd[1]: Started Ceph mgr.x for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:37:59.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:58 smithi016 bash[27454]: audit 2023-10-09T08:37:58.369579+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.16:0/753775823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:37:59.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:58 smithi016 bash[33851]: audit 2023-10-09T08:37:58.369579+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.16:0/753775823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:00.075 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:59 smithi094 bash[27803]: cluster 2023-10-09T08:37:58.720913+0000 mgr.y (mgr.24355) 222 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:00.075 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:59 smithi094 bash[27803]: audit 2023-10-09T08:37:58.901054+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:00.075 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:37:59 smithi094 bash[27803]: audit 2023-10-09T08:37:58.903347+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:00.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:59 smithi016 bash[27454]: cluster 2023-10-09T08:37:58.720913+0000 mgr.y (mgr.24355) 222 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:00.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:59 smithi016 bash[27454]: audit 2023-10-09T08:37:58.901054+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:00.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:37:59 smithi016 bash[27454]: audit 2023-10-09T08:37:58.903347+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:00.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:59 smithi016 bash[33851]: cluster 2023-10-09T08:37:58.720913+0000 mgr.y (mgr.24355) 222 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:00.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:59 smithi016 bash[33851]: audit 2023-10-09T08:37:58.901054+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:00.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:37:59 smithi016 bash[33851]: audit 2023-10-09T08:37:58.903347+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:00.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:00 smithi094 bash[50714]: debug 2023-10-09T08:38:00.070+0000 7f1d26e3b700 1 -- 172.21.15.94:0/3540593927 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5568572c91e0 con 0x556858048400 2023-10-09T08:38:00.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:00 smithi094 bash[50714]: debug 2023-10-09T08:38:00.234+0000 7f1d30f89200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:38:01.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:01 smithi094 bash[50714]: debug 2023-10-09T08:38:01.170+0000 7f1d30f89200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T08:38:01.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:01 smithi094 bash[50714]: debug 2023-10-09T08:38:01.282+0000 7f1d30f89200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:38:01.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:01 smithi094 bash[50714]: debug 2023-10-09T08:38:01.390+0000 7f1d30f89200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:38:01.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:01 smithi094 bash[27803]: cluster 2023-10-09T08:38:00.721890+0000 mgr.y (mgr.24355) 223 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:01.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:01 smithi094 bash[27803]: audit 2023-10-09T08:38:00.781378+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.16:0/291112957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:01.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:01 smithi016 bash[27454]: cluster 2023-10-09T08:38:00.721890+0000 mgr.y (mgr.24355) 223 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:01.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:01 smithi016 bash[27454]: audit 2023-10-09T08:38:00.781378+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.16:0/291112957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:01.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:01 smithi016 bash[33851]: cluster 2023-10-09T08:38:00.721890+0000 mgr.y (mgr.24355) 223 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:01.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:01 smithi016 bash[33851]: audit 2023-10-09T08:38:00.781378+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.16:0/291112957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:01.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:01 smithi094 bash[50714]: debug 2023-10-09T08:38:01.686+0000 7f1d30f89200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:38:02.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:02 smithi094 bash[50714]: debug 2023-10-09T08:38:02.078+0000 7f1d30f89200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:38:02.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:02 smithi094 bash[50714]: debug 2023-10-09T08:38:02.254+0000 7f1d30f89200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:38:02.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:02 smithi094 bash[50714]: debug 2023-10-09T08:38:02.486+0000 7f1d30f89200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:38:03.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:03 smithi094 bash[27803]: cluster 2023-10-09T08:38:02.722501+0000 mgr.y (mgr.24355) 224 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:03.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:03 smithi094 bash[27803]: audit 2023-10-09T08:38:03.201715+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.16:0/2794708453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:03.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:03 smithi094 bash[50714]: debug 2023-10-09T08:38:03.538+0000 7f1d30f89200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:38:03.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:03 smithi094 bash[50714]: debug 2023-10-09T08:38:03.654+0000 7f1d30f89200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T08:38:03.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:03 smithi094 bash[50714]: debug 2023-10-09T08:38:03.774+0000 7f1d30f89200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:38:04.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:03 smithi016 bash[33851]: cluster 2023-10-09T08:38:02.722501+0000 mgr.y (mgr.24355) 224 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:04.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:03 smithi016 bash[33851]: audit 2023-10-09T08:38:03.201715+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.16:0/2794708453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:04.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:03 smithi016 bash[27454]: cluster 2023-10-09T08:38:02.722501+0000 mgr.y (mgr.24355) 224 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:04.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:03 smithi016 bash[27454]: audit 2023-10-09T08:38:03.201715+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.16:0/2794708453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:04.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:04 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:04] "GET /metrics HTTP/1.1" 200 192255 "" "Prometheus/2.33.4" 2023-10-09T08:38:04.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:04 smithi094 bash[50714]: debug 2023-10-09T08:38:04.274+0000 7f1d30f89200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T08:38:04.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:04 smithi094 bash[50714]: debug 2023-10-09T08:38:04.510+0000 7f1d30f89200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:38:05.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:04 smithi094 bash[50714]: debug 2023-10-09T08:38:04.986+0000 7f1d30f89200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:38:05.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:05 smithi094 bash[50714]: debug 2023-10-09T08:38:05.530+0000 7f1d30f89200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:38:05.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:05 smithi094 bash[27803]: audit 2023-10-09T08:38:04.657975+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:05.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:05 smithi094 bash[27803]: cluster 2023-10-09T08:38:04.723143+0000 mgr.y (mgr.24355) 225 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:05.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:05 smithi094 bash[27803]: audit 2023-10-09T08:38:05.620431+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.16:0/1803571673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:05 smithi016 bash[27454]: audit 2023-10-09T08:38:04.657975+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:05 smithi016 bash[27454]: cluster 2023-10-09T08:38:04.723143+0000 mgr.y (mgr.24355) 225 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:06.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:05 smithi016 bash[27454]: audit 2023-10-09T08:38:05.620431+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.16:0/1803571673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:05 smithi016 bash[33851]: audit 2023-10-09T08:38:04.657975+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:05 smithi016 bash[33851]: cluster 2023-10-09T08:38:04.723143+0000 mgr.y (mgr.24355) 225 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:05 smithi016 bash[33851]: audit 2023-10-09T08:38:05.620431+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.16:0/1803571673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:06.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[50714]: debug 2023-10-09T08:38:06.178+0000 7f1d30f89200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:38:06.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[50714]: debug 2023-10-09T08:38:06.302+0000 7f1d30f89200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:38:06.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27723]: debug 2023-10-09T08:38:06.218+0000 7f4b4cbf2700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T08:38:06.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27723]: ignoring --setuser ceph since I am not root 2023-10-09T08:38:06.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27723]: ignoring --setgroup ceph since I am not root 2023-10-09T08:38:06.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27723]: debug 2023-10-09T08:38:06.270+0000 7f21b4ab5700 1 -- 172.21.15.16:0/1070365113 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c1267e44e0 con 0x55c1268d8c00 2023-10-09T08:38:06.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27723]: debug 2023-10-09T08:38:06.430+0000 7f21c4c7d680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:38:06.926 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27723]: debug 2023-10-09T08:38:06.542+0000 7f21c4c7d680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:38:06.981 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[50714]: debug 2023-10-09T08:38:06.686+0000 7f1d30f89200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:38:06.982 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[50714]: debug 2023-10-09T08:38:06.814+0000 7f1d30f89200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:38:06.982 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:05.923854+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:06.982 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:06.184983+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:38:06.982 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:06.187080+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:38:06.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:06.196382+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:06.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:06.205353+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:06.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:06.212142+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T08:38:06.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:06.212579+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T08:38:06.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: cluster 2023-10-09T08:38:06.223931+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T08:38:06.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: audit 2023-10-09T08:38:06.224515+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T08:38:06.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[27803]: cluster 2023-10-09T08:38:06.224579+0000 mon.a (mon.0) 719 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0114669s) 2023-10-09T08:38:07.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:06 smithi094 bash[50714]: debug 2023-10-09T08:38:06.974+0000 7f1d30f89200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:38:07.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:07 smithi094 bash[50714]: debug 2023-10-09T08:38:07.102+0000 7f1d30f89200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:38:07.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:07 smithi016 bash[27723]: debug 2023-10-09T08:38:07.022+0000 7f21c4c7d680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:38:07.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:05.923854+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:07.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:06.184983+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:38:07.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:06.187080+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:06.196382+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:06.205353+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:06.212142+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:06.212579+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: cluster 2023-10-09T08:38:06.223931+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: audit 2023-10-09T08:38:06.224515+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[27454]: cluster 2023-10-09T08:38:06.224579+0000 mon.a (mon.0) 719 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0114669s) 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:05.923854+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:06.184983+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:38:07.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:06.187080+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:38:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:06.196382+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24355 ' entity='mgr.y' 2023-10-09T08:38:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:06.205353+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:06.212142+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.24355 172.21.15.16:0/1228969490' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T08:38:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:06.212579+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T08:38:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: cluster 2023-10-09T08:38:06.223931+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T08:38:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: audit 2023-10-09T08:38:06.224515+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.24355 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T08:38:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:06 smithi016 bash[33851]: cluster 2023-10-09T08:38:06.224579+0000 mon.a (mon.0) 719 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0114669s) 2023-10-09T08:38:07.599 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:07 smithi094 bash[50714]: debug 2023-10-09T08:38:07.242+0000 7f1d30f89200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:38:07.599 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:07 smithi094 bash[50714]: debug 2023-10-09T08:38:07.454+0000 7f1d30f89200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:38:07.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:07 smithi094 bash[50714]: debug 2023-10-09T08:38:07.594+0000 7f1d30f89200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:38:08.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:08 smithi016 bash[33851]: audit 2023-10-09T08:38:08.074001+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.16:0/177530063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:08.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:08 smithi016 bash[27454]: audit 2023-10-09T08:38:08.074001+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.16:0/177530063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:08.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:07 smithi016 bash[27723]: debug 2023-10-09T08:38:07.982+0000 7f21c4c7d680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:38:08.253 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:08 smithi016 bash[27723]: debug 2023-10-09T08:38:08.086+0000 7f21c4c7d680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:38:08.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[27803]: audit 2023-10-09T08:38:08.074001+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.16:0/177530063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:08.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:08 smithi016 bash[27723]: debug 2023-10-09T08:38:08.354+0000 7f21c4c7d680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:38:09.127 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: debug 2023-10-09T08:38:08.834+0000 7f1d30f89200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:38:09.127 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: [09/Oct/2023:08:38:08] ENGINE Bus STARTING 2023-10-09T08:38:09.127 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: CherryPy Checker: 2023-10-09T08:38:09.127 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: The Application mounted at '' has an empty config. 2023-10-09T08:38:09.127 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: [09/Oct/2023:08:38:08] ENGINE Serving on http://:::9283 2023-10-09T08:38:09.127 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: [09/Oct/2023:08:38:08] ENGINE Bus STARTED 2023-10-09T08:38:09.128 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: [09/Oct/2023:08:38:08] ENGINE Bus STOPPING 2023-10-09T08:38:09.128 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: [09/Oct/2023:08:38:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:38:09.128 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:08 smithi094 bash[50714]: [09/Oct/2023:08:38:08] ENGINE Bus STOPPED 2023-10-09T08:38:09.128 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: [09/Oct/2023:08:38:09] ENGINE Bus STARTING 2023-10-09T08:38:09.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c9242b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c9242b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c9242b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c9242b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c9242b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c8f425700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.494 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c8f425700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.494 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c8f425700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.494 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c8f425700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.494 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: debug 2023-10-09T08:38:09.210+0000 7f1c8f425700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:38:09.494 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: [09/Oct/2023:08:38:09] ENGINE Serving on http://:::9283 2023-10-09T08:38:09.494 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[50714]: [09/Oct/2023:08:38:09] ENGINE Bus STARTED 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: cluster 2023-10-09T08:38:08.859739+0000 mon.a (mon.0) 720 : cluster [INF] Active manager daemon x restarted 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: cluster 2023-10-09T08:38:08.860414+0000 mon.a (mon.0) 721 : cluster [INF] Activating manager daemon x 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: cluster 2023-10-09T08:38:08.870267+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: cluster 2023-10-09T08:38:08.870664+0000 mon.a (mon.0) 723 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0105121s) 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.871644+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.871788+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.872020+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.872225+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:09.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.872748+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:38:09.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.872953+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:38:09.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.873192+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:38:09.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.873425+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:38:09.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.873671+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:38:09.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.873909+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:38:09.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.874145+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:38:09.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.874369+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:38:09.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.874599+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:38:09.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.874822+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:38:09.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.875050+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:38:09.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.875495+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.875758+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.876241+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.876472+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.877507+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:08.877905+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: cluster 2023-10-09T08:38:09.038991+0000 mon.a (mon.0) 724 : cluster [INF] Manager daemon x is now available 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:09.069940+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:09.085067+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:09.089073+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:09.095068+0000 mon.c (mon.2) 150 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:09.498 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:09.095570+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:09.499 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:09 smithi094 bash[27803]: audit 2023-10-09T08:38:09.119690+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:09.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: cluster 2023-10-09T08:38:08.859739+0000 mon.a (mon.0) 720 : cluster [INF] Active manager daemon x restarted 2023-10-09T08:38:09.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: cluster 2023-10-09T08:38:08.860414+0000 mon.a (mon.0) 721 : cluster [INF] Activating manager daemon x 2023-10-09T08:38:09.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: cluster 2023-10-09T08:38:08.870267+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T08:38:09.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: cluster 2023-10-09T08:38:08.870664+0000 mon.a (mon.0) 723 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0105121s) 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.871644+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.871788+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.872020+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.872225+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.872748+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.872953+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.873192+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.873425+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.873671+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.873909+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:38:09.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.874145+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.874369+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.874599+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.874822+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.875050+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: cluster 2023-10-09T08:38:08.859739+0000 mon.a (mon.0) 720 : cluster [INF] Active manager daemon x restarted 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: cluster 2023-10-09T08:38:08.860414+0000 mon.a (mon.0) 721 : cluster [INF] Activating manager daemon x 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: cluster 2023-10-09T08:38:08.870267+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: cluster 2023-10-09T08:38:08.870664+0000 mon.a (mon.0) 723 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0105121s) 2023-10-09T08:38:09.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.871644+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.871788+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.872020+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.872225+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.872748+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.872953+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.873192+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.873425+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.873671+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:38:09.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.873909+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.874145+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.874369+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.874599+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.874822+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.875050+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.875495+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.875758+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:38:09.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.876241+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.876472+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.877507+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:08.877905+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: cluster 2023-10-09T08:38:09.038991+0000 mon.a (mon.0) 724 : cluster [INF] Manager daemon x is now available 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:09.069940+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:09.085067+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:09.089073+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:09.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:09.095068+0000 mon.c (mon.2) 150 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:09.095570+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[33851]: audit 2023-10-09T08:38:09.119690+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.875495+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.875758+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.876241+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.876472+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.877507+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:08.877905+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:38:09.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: cluster 2023-10-09T08:38:09.038991+0000 mon.a (mon.0) 724 : cluster [INF] Manager daemon x is now available 2023-10-09T08:38:09.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:09.069940+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:09.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:09.085067+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:09.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:09.089073+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:38:09.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:09.095068+0000 mon.c (mon.2) 150 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:09.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:09.095570+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:09.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27454]: audit 2023-10-09T08:38:09.119690+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:09.892 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27723]: debug 2023-10-09T08:38:09.586+0000 7f21c4c7d680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:38:09.893 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27723]: debug 2023-10-09T08:38:09.770+0000 7f21c4c7d680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:38:10.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:09 smithi016 bash[27723]: debug 2023-10-09T08:38:09.886+0000 7f21c4c7d680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:38:10.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27723]: debug 2023-10-09T08:38:10.014+0000 7f21c4c7d680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:09.127650+0000 mon.c (mon.2) 152 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:09.128325+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:09.167565+0000 mon.c (mon.2) 153 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:09.168464+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:09.180948+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: cephadm 2023-10-09T08:38:09.595379+0000 mgr.x (mgr.24790) 1 : cephadm [INF] Deploying cephadm binary to smithi094 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: cephadm 2023-10-09T08:38:09.601946+0000 mgr.x (mgr.24790) 2 : cephadm [INF] Deploying cephadm binary to smithi016 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: cluster 2023-10-09T08:38:09.874628+0000 mon.a (mon.0) 730 : cluster [DBG] mgrmap e22: x(active, since 1.01448s) 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: cluster 2023-10-09T08:38:09.899901+0000 mgr.x (mgr.24790) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:10.070548+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:38:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: cluster 2023-10-09T08:38:10.070633+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:10.081156+0000 mon.c (mon.2) 154 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27454]: audit 2023-10-09T08:38:10.081692+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:09.127650+0000 mon.c (mon.2) 152 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:09.128325+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:09.167565+0000 mon.c (mon.2) 153 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:09.168464+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:09.180948+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: cephadm 2023-10-09T08:38:09.595379+0000 mgr.x (mgr.24790) 1 : cephadm [INF] Deploying cephadm binary to smithi094 2023-10-09T08:38:10.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: cephadm 2023-10-09T08:38:09.601946+0000 mgr.x (mgr.24790) 2 : cephadm [INF] Deploying cephadm binary to smithi016 2023-10-09T08:38:10.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: cluster 2023-10-09T08:38:09.874628+0000 mon.a (mon.0) 730 : cluster [DBG] mgrmap e22: x(active, since 1.01448s) 2023-10-09T08:38:10.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: cluster 2023-10-09T08:38:09.899901+0000 mgr.x (mgr.24790) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:10.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:10.070548+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:38:10.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: cluster 2023-10-09T08:38:10.070633+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T08:38:10.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:10.081156+0000 mon.c (mon.2) 154 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:10.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[33851]: audit 2023-10-09T08:38:10.081692+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:10.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:09.127650+0000 mon.c (mon.2) 152 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:38:10.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:09.128325+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:38:10.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:09.167565+0000 mon.c (mon.2) 153 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T08:38:10.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:09.168464+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T08:38:10.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:09.180948+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:10.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: cephadm 2023-10-09T08:38:09.595379+0000 mgr.x (mgr.24790) 1 : cephadm [INF] Deploying cephadm binary to smithi094 2023-10-09T08:38:10.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: cephadm 2023-10-09T08:38:09.601946+0000 mgr.x (mgr.24790) 2 : cephadm [INF] Deploying cephadm binary to smithi016 2023-10-09T08:38:10.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: cluster 2023-10-09T08:38:09.874628+0000 mon.a (mon.0) 730 : cluster [DBG] mgrmap e22: x(active, since 1.01448s) 2023-10-09T08:38:10.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: cluster 2023-10-09T08:38:09.899901+0000 mgr.x (mgr.24790) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:10.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:10.070548+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:38:10.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: cluster 2023-10-09T08:38:10.070633+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T08:38:10.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:10.081156+0000 mon.c (mon.2) 154 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:10.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:10 smithi094 bash[27803]: audit 2023-10-09T08:38:10.081692+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:38:10.671 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27723]: debug 2023-10-09T08:38:10.270+0000 7f21c4c7d680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:38:11.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27723]: debug 2023-10-09T08:38:10.662+0000 7f21c4c7d680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:38:11.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:10 smithi016 bash[27723]: debug 2023-10-09T08:38:10.794+0000 7f21c4c7d680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:38:11.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27454]: audit 2023-10-09T08:38:10.368898+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:11.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27454]: audit 2023-10-09T08:38:10.386668+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:11.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27454]: audit 2023-10-09T08:38:10.530788+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.16:0/3699261875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:11.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27454]: cluster 2023-10-09T08:38:10.876995+0000 mgr.x (mgr.24790) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:11.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27454]: cephadm 2023-10-09T08:38:10.976241+0000 mgr.x (mgr.24790) 5 : cephadm [INF] [09/Oct/2023:08:38:10] ENGINE Bus STARTING 2023-10-09T08:38:11.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27454]: audit 2023-10-09T08:38:11.072900+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27454]: cluster 2023-10-09T08:38:11.072990+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[33851]: audit 2023-10-09T08:38:10.368898+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[33851]: audit 2023-10-09T08:38:10.386668+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[33851]: audit 2023-10-09T08:38:10.530788+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.16:0/3699261875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[33851]: cluster 2023-10-09T08:38:10.876995+0000 mgr.x (mgr.24790) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[33851]: cephadm 2023-10-09T08:38:10.976241+0000 mgr.x (mgr.24790) 5 : cephadm [INF] [09/Oct/2023:08:38:10] ENGINE Bus STARTING 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[33851]: audit 2023-10-09T08:38:11.072900+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:38:11.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[33851]: cluster 2023-10-09T08:38:11.072990+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T08:38:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:11 smithi094 bash[27803]: audit 2023-10-09T08:38:10.368898+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:11 smithi094 bash[27803]: audit 2023-10-09T08:38:10.386668+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:11 smithi094 bash[27803]: audit 2023-10-09T08:38:10.530788+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.16:0/3699261875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:11 smithi094 bash[27803]: cluster 2023-10-09T08:38:10.876995+0000 mgr.x (mgr.24790) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:11 smithi094 bash[27803]: cephadm 2023-10-09T08:38:10.976241+0000 mgr.x (mgr.24790) 5 : cephadm [INF] [09/Oct/2023:08:38:10] ENGINE Bus STARTING 2023-10-09T08:38:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:11 smithi094 bash[27803]: audit 2023-10-09T08:38:11.072900+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:38:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:11 smithi094 bash[27803]: cluster 2023-10-09T08:38:11.072990+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T08:38:12.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27723]: debug 2023-10-09T08:38:11.726+0000 7f21c4c7d680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:38:12.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27723]: debug 2023-10-09T08:38:11.838+0000 7f21c4c7d680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:38:12.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:11 smithi016 bash[27723]: debug 2023-10-09T08:38:11.970+0000 7f21c4c7d680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:38:12.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27723]: debug 2023-10-09T08:38:12.218+0000 7f21c4c7d680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:38:12.883 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27723]: debug 2023-10-09T08:38:12.522+0000 7f21c4c7d680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:38:12.883 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27723]: debug 2023-10-09T08:38:12.702+0000 7f21c4c7d680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:38:12.884 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:12 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:12] "GET /metrics HTTP/1.1" 200 30884 "" "Prometheus/2.33.4" 2023-10-09T08:38:13.241 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27723]: debug 2023-10-09T08:38:12.886+0000 7f21c4c7d680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:38:13.241 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27723]: debug 2023-10-09T08:38:13.118+0000 7f21c4c7d680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:38:13.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27454]: cephadm 2023-10-09T08:38:11.126827+0000 mgr.x (mgr.24790) 6 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Serving on http://172.21.15.94:8765 2023-10-09T08:38:13.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27454]: cephadm 2023-10-09T08:38:11.353064+0000 mgr.x (mgr.24790) 7 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Serving on https://172.21.15.94:7150 2023-10-09T08:38:13.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27454]: cephadm 2023-10-09T08:38:11.353509+0000 mgr.x (mgr.24790) 8 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Bus STARTED 2023-10-09T08:38:13.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27454]: cluster 2023-10-09T08:38:11.881584+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e23: x(active, since 3s) 2023-10-09T08:38:13.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[27454]: audit 2023-10-09T08:38:12.296356+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:13.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[33851]: cephadm 2023-10-09T08:38:11.126827+0000 mgr.x (mgr.24790) 6 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Serving on http://172.21.15.94:8765 2023-10-09T08:38:13.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[33851]: cephadm 2023-10-09T08:38:11.353064+0000 mgr.x (mgr.24790) 7 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Serving on https://172.21.15.94:7150 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[33851]: cephadm 2023-10-09T08:38:11.353509+0000 mgr.x (mgr.24790) 8 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Bus STARTED 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[33851]: cluster 2023-10-09T08:38:11.881584+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e23: x(active, since 3s) 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:12 smithi016 bash[33851]: audit 2023-10-09T08:38:12.296356+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:12 smithi094 bash[27803]: cephadm 2023-10-09T08:38:11.126827+0000 mgr.x (mgr.24790) 6 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Serving on http://172.21.15.94:8765 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:12 smithi094 bash[27803]: cephadm 2023-10-09T08:38:11.353064+0000 mgr.x (mgr.24790) 7 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Serving on https://172.21.15.94:7150 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:12 smithi094 bash[27803]: cephadm 2023-10-09T08:38:11.353509+0000 mgr.x (mgr.24790) 8 : cephadm [INF] [09/Oct/2023:08:38:11] ENGINE Bus STARTED 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:12 smithi094 bash[27803]: cluster 2023-10-09T08:38:11.881584+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e23: x(active, since 3s) 2023-10-09T08:38:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:12 smithi094 bash[27803]: audit 2023-10-09T08:38:12.296356+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:13.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27723]: debug 2023-10-09T08:38:13.234+0000 7f21c4c7d680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:38:13.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27723]: [09/Oct/2023:08:38:13] ENGINE Bus STARTING 2023-10-09T08:38:13.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27723]: CherryPy Checker: 2023-10-09T08:38:13.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27723]: The Application mounted at '' has an empty config. 2023-10-09T08:38:13.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27723]: [09/Oct/2023:08:38:13] ENGINE Serving on http://:::9283 2023-10-09T08:38:13.503 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27723]: [09/Oct/2023:08:38:13] ENGINE Bus STARTED 2023-10-09T08:38:14.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27454]: cluster 2023-10-09T08:38:12.877690+0000 mgr.x (mgr.24790) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:14.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27454]: audit 2023-10-09T08:38:12.958428+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.16:0/3018919701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27454]: cluster 2023-10-09T08:38:13.262161+0000 mon.a (mon.0) 741 : cluster [DBG] Standby manager daemon y started 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27454]: audit 2023-10-09T08:38:13.263166+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27454]: audit 2023-10-09T08:38:13.265897+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27454]: audit 2023-10-09T08:38:13.269849+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[27454]: audit 2023-10-09T08:38:13.270422+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[33851]: cluster 2023-10-09T08:38:12.877690+0000 mgr.x (mgr.24790) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[33851]: audit 2023-10-09T08:38:12.958428+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.16:0/3018919701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:14.146 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[33851]: cluster 2023-10-09T08:38:13.262161+0000 mon.a (mon.0) 741 : cluster [DBG] Standby manager daemon y started 2023-10-09T08:38:14.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[33851]: audit 2023-10-09T08:38:13.263166+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T08:38:14.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[33851]: audit 2023-10-09T08:38:13.265897+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:38:14.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[33851]: audit 2023-10-09T08:38:13.269849+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T08:38:14.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:13 smithi016 bash[33851]: audit 2023-10-09T08:38:13.270422+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:38:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:13 smithi094 bash[27803]: cluster 2023-10-09T08:38:12.877690+0000 mgr.x (mgr.24790) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:13 smithi094 bash[27803]: audit 2023-10-09T08:38:12.958428+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.16:0/3018919701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:13 smithi094 bash[27803]: cluster 2023-10-09T08:38:13.262161+0000 mon.a (mon.0) 741 : cluster [DBG] Standby manager daemon y started 2023-10-09T08:38:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:13 smithi094 bash[27803]: audit 2023-10-09T08:38:13.263166+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T08:38:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:13 smithi094 bash[27803]: audit 2023-10-09T08:38:13.265897+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:38:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:13 smithi094 bash[27803]: audit 2023-10-09T08:38:13.269849+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T08:38:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:13 smithi094 bash[27803]: audit 2023-10-09T08:38:13.270422+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.? 172.21.15.16:0/2784559461' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:38:14.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:14 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:38:15.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:14 smithi094 bash[27803]: cluster 2023-10-09T08:38:13.894846+0000 mon.a (mon.0) 742 : cluster [DBG] mgrmap e24: x(active, since 5s), standbys: y 2023-10-09T08:38:15.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:14 smithi094 bash[27803]: audit 2023-10-09T08:38:13.900104+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:38:15.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:14 smithi016 bash[27454]: cluster 2023-10-09T08:38:13.894846+0000 mon.a (mon.0) 742 : cluster [DBG] mgrmap e24: x(active, since 5s), standbys: y 2023-10-09T08:38:15.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:14 smithi016 bash[27454]: audit 2023-10-09T08:38:13.900104+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:38:15.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:14 smithi016 bash[33851]: cluster 2023-10-09T08:38:13.894846+0000 mon.a (mon.0) 742 : cluster [DBG] mgrmap e24: x(active, since 5s), standbys: y 2023-10-09T08:38:15.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:14 smithi016 bash[33851]: audit 2023-10-09T08:38:13.900104+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:38:16.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:15 smithi094 bash[27803]: cluster 2023-10-09T08:38:14.878395+0000 mgr.x (mgr.24790) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:16.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:15 smithi094 bash[27803]: audit 2023-10-09T08:38:15.403516+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.16:0/936315987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:16.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:15 smithi016 bash[27454]: cluster 2023-10-09T08:38:14.878395+0000 mgr.x (mgr.24790) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:16.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:15 smithi016 bash[27454]: audit 2023-10-09T08:38:15.403516+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.16:0/936315987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:15 smithi016 bash[33851]: cluster 2023-10-09T08:38:14.878395+0000 mgr.x (mgr.24790) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:16.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:15 smithi016 bash[33851]: audit 2023-10-09T08:38:15.403516+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.16:0/936315987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:17.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:17 smithi094 bash[27803]: audit 2023-10-09T08:38:16.088339+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:17.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:17 smithi094 bash[27803]: cluster 2023-10-09T08:38:16.879361+0000 mgr.x (mgr.24790) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 234 KiB/s wr, 31 op/s 2023-10-09T08:38:17.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:17 smithi016 bash[27454]: audit 2023-10-09T08:38:16.088339+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:17.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:17 smithi016 bash[27454]: cluster 2023-10-09T08:38:16.879361+0000 mgr.x (mgr.24790) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 234 KiB/s wr, 31 op/s 2023-10-09T08:38:17.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:17 smithi016 bash[33851]: audit 2023-10-09T08:38:16.088339+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:17.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:17 smithi016 bash[33851]: cluster 2023-10-09T08:38:16.879361+0000 mgr.x (mgr.24790) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 234 KiB/s wr, 31 op/s 2023-10-09T08:38:19.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:18 smithi094 bash[27803]: audit 2023-10-09T08:38:17.871726+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:18 smithi094 bash[27803]: audit 2023-10-09T08:38:17.880473+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:18 smithi094 bash[27803]: audit 2023-10-09T08:38:17.885484+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.16:0/3483811206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:19.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:18 smithi094 bash[27803]: audit 2023-10-09T08:38:18.143161+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:18 smithi094 bash[27803]: audit 2023-10-09T08:38:18.151782+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:18 smithi094 bash[27803]: audit 2023-10-09T08:38:18.686418+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:18 smithi094 bash[27803]: audit 2023-10-09T08:38:18.695166+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[27454]: audit 2023-10-09T08:38:17.871726+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[27454]: audit 2023-10-09T08:38:17.880473+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[27454]: audit 2023-10-09T08:38:17.885484+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.16:0/3483811206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:19.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[27454]: audit 2023-10-09T08:38:18.143161+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[27454]: audit 2023-10-09T08:38:18.151782+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[27454]: audit 2023-10-09T08:38:18.686418+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[27454]: audit 2023-10-09T08:38:18.695166+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[33851]: audit 2023-10-09T08:38:17.871726+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[33851]: audit 2023-10-09T08:38:17.880473+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[33851]: audit 2023-10-09T08:38:17.885484+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.16:0/3483811206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:19.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[33851]: audit 2023-10-09T08:38:18.143161+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[33851]: audit 2023-10-09T08:38:18.151782+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[33851]: audit 2023-10-09T08:38:18.686418+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:19.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:18 smithi016 bash[33851]: audit 2023-10-09T08:38:18.695166+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: cluster 2023-10-09T08:38:18.879977+0000 mgr.x (mgr.24790) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:18.925793+0000 mon.a (mon.0) 751 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:18.949796+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:18.950131+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:18.950416+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:19.879829+0000 mon.a (mon.0) 752 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:19.890270+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:21.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:19.890273+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:21.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:19.890560+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:21.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:20 smithi094 bash[27803]: audit 2023-10-09T08:38:19.890979+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: cluster 2023-10-09T08:38:18.879977+0000 mgr.x (mgr.24790) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:18.925793+0000 mon.a (mon.0) 751 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:18.949796+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:18.950131+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:18.950416+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:19.879829+0000 mon.a (mon.0) 752 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:19.890270+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:19.890273+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:19.890560+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:21.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[33851]: audit 2023-10-09T08:38:19.890979+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: cluster 2023-10-09T08:38:18.879977+0000 mgr.x (mgr.24790) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:18.925793+0000 mon.a (mon.0) 751 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:18.949796+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:18.950131+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:18.950416+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:19.879829+0000 mon.a (mon.0) 752 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:19.890270+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:19.890273+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:19.890560+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:21.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:20 smithi016 bash[27454]: audit 2023-10-09T08:38:19.890979+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:19.888812+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:20.323054+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.16:0/3882034944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:20.803909+0000 mon.b (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:20.813831+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:20.820526+0000 mon.c (mon.2) 167 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: cluster 2023-10-09T08:38:20.881174+0000 mgr.x (mgr.24790) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:21.776784+0000 mon.c (mon.2) 168 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:22.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:21.778479+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:22.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:21.778962+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:22.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:21 smithi094 bash[27803]: audit 2023-10-09T08:38:21.779314+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:19.888812+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:20.323054+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.16:0/3882034944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:20.803909+0000 mon.b (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:20.813831+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:20.820526+0000 mon.c (mon.2) 167 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: cluster 2023-10-09T08:38:20.881174+0000 mgr.x (mgr.24790) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:21.776784+0000 mon.c (mon.2) 168 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:21.778479+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:21.778962+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[27454]: audit 2023-10-09T08:38:21.779314+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:19.888812+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:20.323054+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.16:0/3882034944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:20.803909+0000 mon.b (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:20.813831+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:22.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:20.820526+0000 mon.c (mon.2) 167 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T08:38:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: cluster 2023-10-09T08:38:20.881174+0000 mgr.x (mgr.24790) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T08:38:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:21.776784+0000 mon.c (mon.2) 168 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T08:38:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:21.778479+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:38:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:21.778962+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:38:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:21 smithi016 bash[33851]: audit 2023-10-09T08:38:21.779314+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:38:22.831 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:22 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:22] "GET /metrics HTTP/1.1" 200 30884 "" "Prometheus/2.33.4" 2023-10-09T08:38:23.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:22 smithi094 bash[27803]: audit 2023-10-09T08:38:22.772635+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.16:0/2876424179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:23.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:22 smithi016 bash[27454]: audit 2023-10-09T08:38:22.772635+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.16:0/2876424179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:23.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:22 smithi016 bash[33851]: audit 2023-10-09T08:38:22.772635+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.16:0/2876424179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:24.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:23 smithi016 bash[27454]: cluster 2023-10-09T08:38:22.881997+0000 mgr.x (mgr.24790) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 182 KiB/s wr, 21 op/s 2023-10-09T08:38:24.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:23 smithi016 bash[33851]: cluster 2023-10-09T08:38:22.881997+0000 mgr.x (mgr.24790) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 182 KiB/s wr, 21 op/s 2023-10-09T08:38:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:23 smithi094 bash[27803]: cluster 2023-10-09T08:38:22.881997+0000 mgr.x (mgr.24790) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 182 KiB/s wr, 21 op/s 2023-10-09T08:38:24.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:24 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:38:24.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:24 smithi016 bash[27454]: audit 2023-10-09T08:38:24.118847+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:25.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:24 smithi094 bash[27803]: audit 2023-10-09T08:38:24.118847+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:25.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:24 smithi016 bash[33851]: audit 2023-10-09T08:38:24.118847+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:25.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:25 smithi094 bash[27803]: cluster 2023-10-09T08:38:24.882844+0000 mgr.x (mgr.24790) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T08:38:25.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:25 smithi094 bash[27803]: audit 2023-10-09T08:38:25.223122+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.16:0/10735282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:26.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:25 smithi016 bash[33851]: cluster 2023-10-09T08:38:24.882844+0000 mgr.x (mgr.24790) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T08:38:26.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:25 smithi016 bash[33851]: audit 2023-10-09T08:38:25.223122+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.16:0/10735282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:26.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:25 smithi016 bash[27454]: cluster 2023-10-09T08:38:24.882844+0000 mgr.x (mgr.24790) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T08:38:26.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:25 smithi016 bash[27454]: audit 2023-10-09T08:38:25.223122+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.16:0/10735282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:28.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:27 smithi094 bash[27803]: cluster 2023-10-09T08:38:26.883968+0000 mgr.x (mgr.24790) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T08:38:28.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:27 smithi094 bash[27803]: audit 2023-10-09T08:38:27.670358+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.16:0/1052654931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:28.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:27 smithi016 bash[27454]: cluster 2023-10-09T08:38:26.883968+0000 mgr.x (mgr.24790) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T08:38:28.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:27 smithi016 bash[27454]: audit 2023-10-09T08:38:27.670358+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.16:0/1052654931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:28.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:27 smithi016 bash[33851]: cluster 2023-10-09T08:38:26.883968+0000 mgr.x (mgr.24790) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T08:38:28.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:27 smithi016 bash[33851]: audit 2023-10-09T08:38:27.670358+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.16:0/1052654931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.047983+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.055742+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.058208+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.058862+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.060652+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.061128+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.062695+0000 mon.c (mon.2) 177 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.063061+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.064404+0000 mon.c (mon.2) 178 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.064819+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: cephadm 2023-10-09T08:38:28.067074+0000 mgr.x (mgr.24790) 17 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 2525M 2023-10-09T08:38:29.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:29 smithi094 bash[27803]: audit 2023-10-09T08:38:28.073184+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.047983+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.055742+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.058208+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.058862+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.060652+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.061128+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.062695+0000 mon.c (mon.2) 177 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.063061+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.064404+0000 mon.c (mon.2) 178 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.064819+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: cephadm 2023-10-09T08:38:28.067074+0000 mgr.x (mgr.24790) 17 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 2525M 2023-10-09T08:38:29.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[27454]: audit 2023-10-09T08:38:28.073184+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.047983+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.055742+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.058208+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.058862+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.060652+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.061128+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.062695+0000 mon.c (mon.2) 177 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.063061+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.064404+0000 mon.c (mon.2) 178 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.064819+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: cephadm 2023-10-09T08:38:28.067074+0000 mgr.x (mgr.24790) 17 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 2525M 2023-10-09T08:38:29.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:29 smithi016 bash[33851]: audit 2023-10-09T08:38:28.073184+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:30 smithi094 bash[27803]: cluster 2023-10-09T08:38:28.884643+0000 mgr.x (mgr.24790) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T08:38:30.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:30 smithi016 bash[27454]: cluster 2023-10-09T08:38:28.884643+0000 mgr.x (mgr.24790) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T08:38:30.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:30 smithi016 bash[33851]: cluster 2023-10-09T08:38:28.884643+0000 mgr.x (mgr.24790) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T08:38:31.397 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.128843+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.16:0/1395354082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:31.397 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.265152+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.274157+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.277390+0000 mon.c (mon.2) 179 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.277954+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.279843+0000 mon.c (mon.2) 180 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.280333+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.282048+0000 mon.c (mon.2) 181 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.282547+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.284207+0000 mon.c (mon.2) 182 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.284683+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.293339+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.295464+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.296928+0000 mon.c (mon.2) 184 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.969518+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.974977+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.982330+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.987489+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[27454]: audit 2023-10-09T08:38:30.994831+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.128843+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.16:0/1395354082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.265152+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.399 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.274157+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.277390+0000 mon.c (mon.2) 179 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.277954+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.279843+0000 mon.c (mon.2) 180 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.280333+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.282048+0000 mon.c (mon.2) 181 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.282547+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.284207+0000 mon.c (mon.2) 182 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.284683+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.293339+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.295464+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.296928+0000 mon.c (mon.2) 184 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:38:31.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.969518+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.401 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.974977+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.401 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.982330+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.401 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.987489+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.401 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[33851]: audit 2023-10-09T08:38:30.994831+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.128843+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.16:0/1395354082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.265152+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.274157+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.277390+0000 mon.c (mon.2) 179 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.277954+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.279843+0000 mon.c (mon.2) 180 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.280333+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.282048+0000 mon.c (mon.2) 181 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.282547+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.284207+0000 mon.c (mon.2) 182 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.284683+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.293339+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.295464+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.296928+0000 mon.c (mon.2) 184 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.969518+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.974977+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.982330+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.987489+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:31 smithi094 bash[27803]: audit 2023-10-09T08:38:30.994831+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:31.741 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:31 smithi016 systemd[1]: Stopping Ceph node-exporter.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:38:31.742 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:31 smithi016 bash[69713]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter.a 2023-10-09T08:38:32.360 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[69749]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.287136+0000 mgr.x (mgr.24790) 19 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 2524M 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.300222+0000 mgr.x (mgr.24790) 20 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.300564+0000 mgr.x (mgr.24790) 21 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.465179+0000 mgr.x (mgr.24790) 22 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.467116+0000 mgr.x (mgr.24790) 23 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.630077+0000 mgr.x (mgr.24790) 24 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.632620+0000 mgr.x (mgr.24790) 25 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:38:32.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.795168+0000 mgr.x (mgr.24790) 26 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cephadm 2023-10-09T08:38:30.798348+0000 mgr.x (mgr.24790) 27 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: cluster 2023-10-09T08:38:30.885405+0000 mgr.x (mgr.24790) 28 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: audit 2023-10-09T08:38:31.072638+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: audit 2023-10-09T08:38:31.084953+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: audit 2023-10-09T08:38:31.089373+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[27454]: audit 2023-10-09T08:38:31.094916+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.287136+0000 mgr.x (mgr.24790) 19 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 2524M 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.300222+0000 mgr.x (mgr.24790) 20 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:38:32.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.300564+0000 mgr.x (mgr.24790) 21 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.465179+0000 mgr.x (mgr.24790) 22 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.467116+0000 mgr.x (mgr.24790) 23 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.630077+0000 mgr.x (mgr.24790) 24 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.632620+0000 mgr.x (mgr.24790) 25 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.795168+0000 mgr.x (mgr.24790) 26 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cephadm 2023-10-09T08:38:30.798348+0000 mgr.x (mgr.24790) 27 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: cluster 2023-10-09T08:38:30.885405+0000 mgr.x (mgr.24790) 28 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: audit 2023-10-09T08:38:31.072638+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: audit 2023-10-09T08:38:31.084953+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: audit 2023-10-09T08:38:31.089373+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[33851]: audit 2023-10-09T08:38:31.094916+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.493 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[48555]: ts=2023-10-09T08:38:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.16:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:38:32.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.287136+0000 mgr.x (mgr.24790) 19 : cephadm [INF] Adjusting osd_memory_target on smithi094 to 2524M 2023-10-09T08:38:32.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.300222+0000 mgr.x (mgr.24790) 20 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:38:32.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.300564+0000 mgr.x (mgr.24790) 21 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:38:32.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.465179+0000 mgr.x (mgr.24790) 22 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.467116+0000 mgr.x (mgr.24790) 23 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.630077+0000 mgr.x (mgr.24790) 24 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.632620+0000 mgr.x (mgr.24790) 25 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.795168+0000 mgr.x (mgr.24790) 26 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cephadm 2023-10-09T08:38:30.798348+0000 mgr.x (mgr.24790) 27 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: cluster 2023-10-09T08:38:30.885405+0000 mgr.x (mgr.24790) 28 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: audit 2023-10-09T08:38:31.072638+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: audit 2023-10-09T08:38:31.084953+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: audit 2023-10-09T08:38:31.089373+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[27803]: audit 2023-10-09T08:38:31.094916+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:32.752 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:32 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T08:38:32.752 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:32 smithi016 bash[69837]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T08:38:32.752 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:32 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T08:38:32.752 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:32 smithi016 systemd[1]: Stopped Ceph node-exporter.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:32.752 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:32 smithi016 systemd[1]: Started Ceph node-exporter.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:32.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:32 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:32] "GET /metrics HTTP/1.1" 200 33714 "" "Prometheus/2.33.4" 2023-10-09T08:38:33.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:33 smithi094 bash[27803]: cephadm 2023-10-09T08:38:31.097858+0000 mgr.x (mgr.24790) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T08:38:33.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:33 smithi094 bash[27803]: cephadm 2023-10-09T08:38:31.098444+0000 mgr.x (mgr.24790) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi016 2023-10-09T08:38:33.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:33 smithi094 bash[27803]: audit 2023-10-09T08:38:32.414535+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:33.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:33 smithi094 bash[27803]: audit 2023-10-09T08:38:32.423631+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:33.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:33 smithi094 bash[27803]: audit 2023-10-09T08:38:32.595622+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.16:0/490700236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:33.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[27454]: cephadm 2023-10-09T08:38:31.097858+0000 mgr.x (mgr.24790) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T08:38:33.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[27454]: cephadm 2023-10-09T08:38:31.098444+0000 mgr.x (mgr.24790) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi016 2023-10-09T08:38:33.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[27454]: audit 2023-10-09T08:38:32.414535+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:33.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[27454]: audit 2023-10-09T08:38:32.423631+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:33.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[27454]: audit 2023-10-09T08:38:32.595622+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.16:0/490700236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:33.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[33851]: cephadm 2023-10-09T08:38:31.097858+0000 mgr.x (mgr.24790) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T08:38:33.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[33851]: cephadm 2023-10-09T08:38:31.098444+0000 mgr.x (mgr.24790) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi016 2023-10-09T08:38:33.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[33851]: audit 2023-10-09T08:38:32.414535+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:33.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[33851]: audit 2023-10-09T08:38:32.423631+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:33.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[33851]: audit 2023-10-09T08:38:32.595622+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.16:0/490700236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:34.002 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.767Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.767Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.769Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.769Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T08:38:34.003 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T08:38:34.004 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T08:38:34.005 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T08:38:34.006 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.770Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.771Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T08:38:34.007 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 08:38:33 smithi016 bash[69867]: ts=2023-10-09T08:38:33.771Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T08:38:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:34 smithi094 bash[27803]: cephadm 2023-10-09T08:38:32.466610+0000 mgr.x (mgr.24790) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T08:38:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:34 smithi094 bash[27803]: cephadm 2023-10-09T08:38:32.489474+0000 mgr.x (mgr.24790) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-10-09T08:38:34.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:34 smithi094 bash[27803]: cluster 2023-10-09T08:38:32.886049+0000 mgr.x (mgr.24790) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:34.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:34 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:38:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:34 smithi016 bash[33851]: cephadm 2023-10-09T08:38:32.466610+0000 mgr.x (mgr.24790) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T08:38:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:34 smithi016 bash[33851]: cephadm 2023-10-09T08:38:32.489474+0000 mgr.x (mgr.24790) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-10-09T08:38:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:34 smithi016 bash[33851]: cluster 2023-10-09T08:38:32.886049+0000 mgr.x (mgr.24790) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:34.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:34 smithi016 bash[27454]: cephadm 2023-10-09T08:38:32.466610+0000 mgr.x (mgr.24790) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T08:38:34.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:34 smithi016 bash[27454]: cephadm 2023-10-09T08:38:32.489474+0000 mgr.x (mgr.24790) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-10-09T08:38:34.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:34 smithi016 bash[27454]: cluster 2023-10-09T08:38:32.886049+0000 mgr.x (mgr.24790) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:35 smithi094 bash[27803]: cluster 2023-10-09T08:38:34.886809+0000 mgr.x (mgr.24790) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:35.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:35 smithi094 bash[27803]: audit 2023-10-09T08:38:35.084651+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.16:0/2511873659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:35.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:35 smithi016 bash[33851]: cluster 2023-10-09T08:38:34.886809+0000 mgr.x (mgr.24790) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:35.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:35 smithi016 bash[33851]: audit 2023-10-09T08:38:35.084651+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.16:0/2511873659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:35.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:35 smithi016 bash[27454]: cluster 2023-10-09T08:38:34.886809+0000 mgr.x (mgr.24790) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:35.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:35 smithi016 bash[27454]: audit 2023-10-09T08:38:35.084651+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.16:0/2511873659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:38.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:37 smithi094 bash[27803]: cluster 2023-10-09T08:38:36.887971+0000 mgr.x (mgr.24790) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:38.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:37 smithi094 bash[27803]: audit 2023-10-09T08:38:37.533515+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.16:0/600938710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:38.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:37 smithi016 bash[27454]: cluster 2023-10-09T08:38:36.887971+0000 mgr.x (mgr.24790) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:38.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:37 smithi016 bash[27454]: audit 2023-10-09T08:38:37.533515+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.16:0/600938710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:38.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:37 smithi016 bash[33851]: cluster 2023-10-09T08:38:36.887971+0000 mgr.x (mgr.24790) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T08:38:38.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:37 smithi016 bash[33851]: audit 2023-10-09T08:38:37.533515+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.16:0/600938710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:40.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:39 smithi094 bash[27803]: cluster 2023-10-09T08:38:38.888699+0000 mgr.x (mgr.24790) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:40.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:39 smithi094 bash[27803]: audit 2023-10-09T08:38:39.119280+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:40.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:39 smithi016 bash[27454]: cluster 2023-10-09T08:38:38.888699+0000 mgr.x (mgr.24790) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:40.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:39 smithi016 bash[27454]: audit 2023-10-09T08:38:39.119280+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:40.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:39 smithi016 bash[33851]: cluster 2023-10-09T08:38:38.888699+0000 mgr.x (mgr.24790) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:40.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:39 smithi016 bash[33851]: audit 2023-10-09T08:38:39.119280+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:40.560 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:40 smithi016 systemd[1]: Stopping Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:38:40.560 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:40 smithi016 bash[70197]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T08:38:40.560 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:40 smithi016 bash[56937]: level=info ts=2023-10-09T08:38:40.502Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T08:38:41.094 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:40 smithi016 bash[27454]: audit 2023-10-09T08:38:39.986585+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.16:0/2972402877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:41.094 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:40 smithi016 bash[33851]: audit 2023-10-09T08:38:39.986585+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.16:0/2972402877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:41.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:40 smithi094 bash[27803]: audit 2023-10-09T08:38:39.986585+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.16:0/2972402877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:41.252 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[70232]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T08:38:41.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[70301]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T08:38:41.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:41 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T08:38:41.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:41 smithi016 systemd[1]: Stopped Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:41.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:41 smithi016 systemd[1]: Started Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:42.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:41 smithi094 bash[27803]: cluster 2023-10-09T08:38:40.889822+0000 mgr.x (mgr.24790) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:42.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:41 smithi094 bash[27803]: audit 2023-10-09T08:38:41.362114+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:42.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:41 smithi094 bash[27803]: audit 2023-10-09T08:38:41.371237+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:42.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[27454]: cluster 2023-10-09T08:38:40.889822+0000 mgr.x (mgr.24790) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:42.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[27454]: audit 2023-10-09T08:38:41.362114+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:42.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[27454]: audit 2023-10-09T08:38:41.371237+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:42.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[33851]: cluster 2023-10-09T08:38:40.889822+0000 mgr.x (mgr.24790) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:42.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[33851]: audit 2023-10-09T08:38:41.362114+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:42.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:41 smithi016 bash[33851]: audit 2023-10-09T08:38:41.371237+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:42.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:42 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:42] "GET /metrics HTTP/1.1" 200 33709 "" "Prometheus/2.33.4" 2023-10-09T08:38:42.743 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:42 smithi094 bash[48555]: ts=2023-10-09T08:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.16:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:38:42.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.395Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T08:38:42.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.395Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T08:38:42.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.397Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.16 port=9094 2023-10-09T08:38:42.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.398Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T08:38:42.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.437Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T08:38:42.753 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.437Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T08:38:42.753 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.441Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T08:38:42.753 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:42.441Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T08:38:43.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:42 smithi094 bash[27803]: cephadm 2023-10-09T08:38:41.381134+0000 mgr.x (mgr.24790) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:38:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:42 smithi094 bash[27803]: cephadm 2023-10-09T08:38:41.624613+0000 mgr.x (mgr.24790) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:38:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:42 smithi094 bash[27803]: audit 2023-10-09T08:38:42.454961+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.16:0/2769459314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[27454]: cephadm 2023-10-09T08:38:41.381134+0000 mgr.x (mgr.24790) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:38:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[27454]: cephadm 2023-10-09T08:38:41.624613+0000 mgr.x (mgr.24790) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:38:43.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[27454]: audit 2023-10-09T08:38:42.454961+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.16:0/2769459314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:43.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[33851]: cephadm 2023-10-09T08:38:41.381134+0000 mgr.x (mgr.24790) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:38:43.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[33851]: cephadm 2023-10-09T08:38:41.624613+0000 mgr.x (mgr.24790) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:38:43.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:42 smithi016 bash[33851]: audit 2023-10-09T08:38:42.454961+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.16:0/2769459314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:44.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:43 smithi094 bash[27803]: cluster 2023-10-09T08:38:42.890599+0000 mgr.x (mgr.24790) 40 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:44.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:43 smithi016 bash[27454]: cluster 2023-10-09T08:38:42.890599+0000 mgr.x (mgr.24790) 40 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:44.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:38:44 smithi016 bash[27723]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:38:44.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:43 smithi016 bash[33851]: cluster 2023-10-09T08:38:42.890599+0000 mgr.x (mgr.24790) 40 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:44.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:44 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:44.399Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000543961s 2023-10-09T08:38:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:44 smithi094 bash[27803]: audit 2023-10-09T08:38:44.902356+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.16:0/3357204123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:44 smithi016 bash[27454]: audit 2023-10-09T08:38:44.902356+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.16:0/3357204123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:45.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:44 smithi016 bash[33851]: audit 2023-10-09T08:38:44.902356+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.16:0/3357204123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:45 smithi094 bash[27803]: cluster 2023-10-09T08:38:44.891142+0000 mgr.x (mgr.24790) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:46.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:45 smithi016 bash[27454]: cluster 2023-10-09T08:38:44.891142+0000 mgr.x (mgr.24790) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:46.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:45 smithi016 bash[33851]: cluster 2023-10-09T08:38:44.891142+0000 mgr.x (mgr.24790) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:48.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:47 smithi094 bash[27803]: cluster 2023-10-09T08:38:46.892127+0000 mgr.x (mgr.24790) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:47 smithi094 bash[27803]: audit 2023-10-09T08:38:47.348560+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.16:0/974554644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:47 smithi016 bash[27454]: cluster 2023-10-09T08:38:46.892127+0000 mgr.x (mgr.24790) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:48.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:47 smithi016 bash[27454]: audit 2023-10-09T08:38:47.348560+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.16:0/974554644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:48.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:47 smithi016 bash[33851]: cluster 2023-10-09T08:38:46.892127+0000 mgr.x (mgr.24790) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:48.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:47 smithi016 bash[33851]: audit 2023-10-09T08:38:47.348560+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.16:0/974554644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:49 smithi094 bash[27803]: cluster 2023-10-09T08:38:48.892880+0000 mgr.x (mgr.24790) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:50.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:49 smithi094 bash[27803]: audit 2023-10-09T08:38:49.798909+0000 mon.a (mon.0) 788 : audit [DBG] from='client.? 172.21.15.16:0/500664582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:50.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:49 smithi016 bash[33851]: cluster 2023-10-09T08:38:48.892880+0000 mgr.x (mgr.24790) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:50.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:49 smithi016 bash[33851]: audit 2023-10-09T08:38:49.798909+0000 mon.a (mon.0) 788 : audit [DBG] from='client.? 172.21.15.16:0/500664582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:50.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:49 smithi016 bash[27454]: cluster 2023-10-09T08:38:48.892880+0000 mgr.x (mgr.24790) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:50.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:49 smithi016 bash[27454]: audit 2023-10-09T08:38:49.798909+0000 mon.a (mon.0) 788 : audit [DBG] from='client.? 172.21.15.16:0/500664582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:51 smithi094 bash[27803]: cluster 2023-10-09T08:38:50.894015+0000 mgr.x (mgr.24790) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:51 smithi016 bash[27454]: cluster 2023-10-09T08:38:50.894015+0000 mgr.x (mgr.24790) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:51 smithi016 bash[33851]: cluster 2023-10-09T08:38:50.894015+0000 mgr.x (mgr.24790) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:52.339 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:52 smithi094 bash[27803]: audit 2023-10-09T08:38:52.256936+0000 mon.a (mon.0) 789 : audit [DBG] from='client.? 172.21.15.16:0/500153056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:52.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:38:52 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:38:52] "GET /metrics HTTP/1.1" 200 33709 "" "Prometheus/2.33.4" 2023-10-09T08:38:52.743 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:52 smithi094 bash[48555]: ts=2023-10-09T08:38:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.16:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:38:52.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:52 smithi016 bash[27454]: audit 2023-10-09T08:38:52.256936+0000 mon.a (mon.0) 789 : audit [DBG] from='client.? 172.21.15.16:0/500153056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:52.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:38:52 smithi016 bash[70330]: level=info ts=2023-10-09T08:38:52.402Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003876834s 2023-10-09T08:38:52.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:52 smithi016 bash[33851]: audit 2023-10-09T08:38:52.256936+0000 mon.a (mon.0) 789 : audit [DBG] from='client.? 172.21.15.16:0/500153056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:53.353 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[27803]: cluster 2023-10-09T08:38:52.894717+0000 mgr.x (mgr.24790) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:53.353 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 systemd[1]: Stopping Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:38:53.353 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[53069]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.269Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.270Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T08:38:53.354 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.270Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T08:38:53.355 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[48555]: ts=2023-10-09T08:38:53.270Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T08:38:53.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:53 smithi016 bash[33851]: cluster 2023-10-09T08:38:52.894717+0000 mgr.x (mgr.24790) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:53 smithi016 bash[27454]: cluster 2023-10-09T08:38:52.894717+0000 mgr.x (mgr.24790) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:54.194 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:53 smithi094 bash[53104]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T08:38:54.194 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:54 smithi094 bash[53168]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T08:38:54.194 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:54 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T08:38:54.194 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:54 smithi094 systemd[1]: Stopped Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:54.195 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:54 smithi094 systemd[1]: Started Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:54 smithi094 bash[27803]: audit 2023-10-09T08:38:54.119229+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:54 smithi094 bash[27803]: audit 2023-10-09T08:38:54.173799+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:54 smithi094 bash[27803]: audit 2023-10-09T08:38:54.183503+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:54.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:54 smithi016 bash[27454]: audit 2023-10-09T08:38:54.119229+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:54.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:54 smithi016 bash[27454]: audit 2023-10-09T08:38:54.173799+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:54.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:54 smithi016 bash[27454]: audit 2023-10-09T08:38:54.183503+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:54.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:54 smithi016 bash[33851]: audit 2023-10-09T08:38:54.119229+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:38:54.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:54 smithi016 bash[33851]: audit 2023-10-09T08:38:54.173799+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:54.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:54 smithi016 bash[33851]: audit 2023-10-09T08:38:54.183503+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:54.886 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:54 smithi094 systemd[1]: Stopping Ceph node-exporter.b for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:38:54.886 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:54 smithi094 bash[53221]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter.b 2023-10-09T08:38:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:55 smithi094 bash[27803]: cephadm 2023-10-09T08:38:54.223270+0000 mgr.x (mgr.24790) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T08:38:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:55 smithi094 bash[27803]: cephadm 2023-10-09T08:38:54.224111+0000 mgr.x (mgr.24790) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi094 2023-10-09T08:38:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:55 smithi094 bash[27803]: audit 2023-10-09T08:38:54.711360+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.16:0/2333035867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:55 smithi094 bash[27803]: cluster 2023-10-09T08:38:54.895228+0000 mgr.x (mgr.24790) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:55.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[27454]: cephadm 2023-10-09T08:38:54.223270+0000 mgr.x (mgr.24790) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T08:38:55.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[27454]: cephadm 2023-10-09T08:38:54.224111+0000 mgr.x (mgr.24790) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi094 2023-10-09T08:38:55.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[27454]: audit 2023-10-09T08:38:54.711360+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.16:0/2333035867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:55.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[27454]: cluster 2023-10-09T08:38:54.895228+0000 mgr.x (mgr.24790) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:55.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[33851]: cephadm 2023-10-09T08:38:54.223270+0000 mgr.x (mgr.24790) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T08:38:55.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[33851]: cephadm 2023-10-09T08:38:54.224111+0000 mgr.x (mgr.24790) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi094 2023-10-09T08:38:55.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[33851]: audit 2023-10-09T08:38:54.711360+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.16:0/2333035867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:55.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:55 smithi016 bash[33851]: cluster 2023-10-09T08:38:54.895228+0000 mgr.x (mgr.24790) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:56.071 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:55 smithi094 bash[53258]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T08:38:56.374 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.199Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T08:38:56.374 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.199Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T08:38:56.374 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.200Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi094 (none))" 2023-10-09T08:38:56.374 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.200Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T08:38:56.374 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.200Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T08:38:56.374 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.201Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T08:38:56.374 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.202Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T08:38:56.375 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.203Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T08:38:56.375 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.208Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T08:38:56.375 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.208Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.819µs 2023-10-09T08:38:56.375 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.208Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T08:38:56.375 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.371Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T08:38:56.375 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:56 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T08:38:56.375 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53341]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T08:38:56.375 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:56 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T08:38:56.376 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:56 smithi094 systemd[1]: Stopped Ceph node-exporter.b for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:56.376 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:56 smithi094 systemd[1]: Started Ceph node-exporter.b for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:38:56.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.641Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T08:38:56.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.642Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T08:38:56.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.642Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=55.143µs wal_replay_duration=434.018594ms total_replay_duration=434.098116ms 2023-10-09T08:38:56.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.647Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T08:38:56.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.647Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T08:38:56.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.647Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T08:38:56.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.672Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.70366ms db_storage=1.321µs remote_storage=1.93µs web_handler=499ns query_engine=1.064µs scrape=437.022µs scrape_sd=139.549µs notify=30.125µs notify_sd=27.943µs rules=23.617509ms 2023-10-09T08:38:56.994 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:38:56 smithi094 bash[53198]: ts=2023-10-09T08:38:56.672Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T08:38:57.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[27803]: audit 2023-10-09T08:38:56.131301+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:57.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[27803]: audit 2023-10-09T08:38:56.140566+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:57.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[27803]: cephadm 2023-10-09T08:38:56.192222+0000 mgr.x (mgr.24790) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T08:38:57.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[27803]: cephadm 2023-10-09T08:38:56.215527+0000 mgr.x (mgr.24790) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi094 2023-10-09T08:38:57.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[27803]: cluster 2023-10-09T08:38:56.896341+0000 mgr.x (mgr.24790) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:57.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[27454]: audit 2023-10-09T08:38:56.131301+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:57.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[27454]: audit 2023-10-09T08:38:56.140566+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:57.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[27454]: cephadm 2023-10-09T08:38:56.192222+0000 mgr.x (mgr.24790) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T08:38:57.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[27454]: cephadm 2023-10-09T08:38:56.215527+0000 mgr.x (mgr.24790) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi094 2023-10-09T08:38:57.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[27454]: cluster 2023-10-09T08:38:56.896341+0000 mgr.x (mgr.24790) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:57.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[33851]: audit 2023-10-09T08:38:56.131301+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:57.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[33851]: audit 2023-10-09T08:38:56.140566+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:38:57.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[33851]: cephadm 2023-10-09T08:38:56.192222+0000 mgr.x (mgr.24790) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T08:38:57.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[33851]: cephadm 2023-10-09T08:38:56.215527+0000 mgr.x (mgr.24790) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi094 2023-10-09T08:38:57.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:57 smithi016 bash[33851]: cluster 2023-10-09T08:38:56.896341+0000 mgr.x (mgr.24790) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:57.993 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.637Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T08:38:57.994 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.637Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T08:38:57.994 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.637Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T08:38:57.994 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.637Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T08:38:57.994 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T08:38:57.994 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T08:38:57.994 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T08:38:57.994 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.638Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T08:38:57.995 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T08:38:57.996 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T08:38:57.997 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T08:38:57.998 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T08:38:57.999 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T08:38:57.999 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T08:38:57.999 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T08:38:57.999 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 08:38:57 smithi094 bash[53389]: ts=2023-10-09T08:38:57.639Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T08:38:58.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:58 smithi094 bash[27803]: audit 2023-10-09T08:38:57.177939+0000 mon.a (mon.0) 795 : audit [DBG] from='client.? 172.21.15.16:0/3517328248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:58.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:58 smithi016 bash[27454]: audit 2023-10-09T08:38:57.177939+0000 mon.a (mon.0) 795 : audit [DBG] from='client.? 172.21.15.16:0/3517328248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:58.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:58 smithi016 bash[33851]: audit 2023-10-09T08:38:57.177939+0000 mon.a (mon.0) 795 : audit [DBG] from='client.? 172.21.15.16:0/3517328248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:38:59.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:38:59 smithi094 bash[27803]: cluster 2023-10-09T08:38:58.896786+0000 mgr.x (mgr.24790) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:38:59 smithi016 bash[27454]: cluster 2023-10-09T08:38:58.896786+0000 mgr.x (mgr.24790) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:38:59.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:38:59 smithi016 bash[33851]: cluster 2023-10-09T08:38:58.896786+0000 mgr.x (mgr.24790) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:00.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:00 smithi094 bash[27803]: audit 2023-10-09T08:38:59.631484+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.16:0/3097475616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:00.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:00 smithi016 bash[27454]: audit 2023-10-09T08:38:59.631484+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.16:0/3097475616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:00.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:00 smithi016 bash[33851]: audit 2023-10-09T08:38:59.631484+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.16:0/3097475616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:01.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:01 smithi094 bash[27803]: cluster 2023-10-09T08:39:00.897757+0000 mgr.x (mgr.24790) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:01.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:01 smithi016 bash[27454]: cluster 2023-10-09T08:39:00.897757+0000 mgr.x (mgr.24790) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:01.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:01 smithi016 bash[33851]: cluster 2023-10-09T08:39:00.897757+0000 mgr.x (mgr.24790) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:02.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:02 smithi094 bash[27803]: audit 2023-10-09T08:39:02.084768+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.16:0/3764209529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:02.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:02 smithi016 bash[27454]: audit 2023-10-09T08:39:02.084768+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.16:0/3764209529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:02.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:02 smithi016 bash[33851]: audit 2023-10-09T08:39:02.084768+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.16:0/3764209529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:03.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:03 smithi016 bash[27454]: cluster 2023-10-09T08:39:02.898436+0000 mgr.x (mgr.24790) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:03.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:03 smithi016 bash[33851]: cluster 2023-10-09T08:39:02.898436+0000 mgr.x (mgr.24790) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:03.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:03 smithi094 bash[27803]: cluster 2023-10-09T08:39:02.898436+0000 mgr.x (mgr.24790) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:04 smithi094 bash[27803]: audit 2023-10-09T08:39:04.539321+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.16:0/511811472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:05.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:04 smithi016 bash[27454]: audit 2023-10-09T08:39:04.539321+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.16:0/511811472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:05.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:04 smithi016 bash[33851]: audit 2023-10-09T08:39:04.539321+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.16:0/511811472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:05.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:05 smithi094 bash[27803]: cluster 2023-10-09T08:39:04.898935+0000 mgr.x (mgr.24790) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:05 smithi016 bash[33851]: cluster 2023-10-09T08:39:04.898935+0000 mgr.x (mgr.24790) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:05 smithi016 bash[27454]: cluster 2023-10-09T08:39:04.898935+0000 mgr.x (mgr.24790) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:07.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:06 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:39:06] "GET /metrics HTTP/1.1" 200 33708 "" "Prometheus/2.33.4" 2023-10-09T08:39:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:07 smithi094 bash[27803]: cluster 2023-10-09T08:39:06.899744+0000 mgr.x (mgr.24790) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:07 smithi094 bash[27803]: audit 2023-10-09T08:39:06.990061+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.16:0/3452368429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:08.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:07 smithi016 bash[27454]: cluster 2023-10-09T08:39:06.899744+0000 mgr.x (mgr.24790) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:08.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:07 smithi016 bash[27454]: audit 2023-10-09T08:39:06.990061+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.16:0/3452368429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:08.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:07 smithi016 bash[33851]: cluster 2023-10-09T08:39:06.899744+0000 mgr.x (mgr.24790) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:08.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:07 smithi016 bash[33851]: audit 2023-10-09T08:39:06.990061+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.16:0/3452368429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:08.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:08 smithi094 bash[53198]: ts=2023-10-09T08:39:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:39:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:09 smithi094 bash[27803]: cluster 2023-10-09T08:39:08.900352+0000 mgr.x (mgr.24790) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:09 smithi094 bash[27803]: audit 2023-10-09T08:39:09.121520+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:09 smithi094 bash[27803]: audit 2023-10-09T08:39:09.444647+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.16:0/937855198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:10.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:09 smithi016 bash[27454]: cluster 2023-10-09T08:39:08.900352+0000 mgr.x (mgr.24790) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:10.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:09 smithi016 bash[27454]: audit 2023-10-09T08:39:09.121520+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:10.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:09 smithi016 bash[27454]: audit 2023-10-09T08:39:09.444647+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.16:0/937855198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:10.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:09 smithi016 bash[33851]: cluster 2023-10-09T08:39:08.900352+0000 mgr.x (mgr.24790) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:10.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:09 smithi016 bash[33851]: audit 2023-10-09T08:39:09.121520+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:10.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:09 smithi016 bash[33851]: audit 2023-10-09T08:39:09.444647+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.16:0/937855198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:11 smithi094 bash[27803]: cluster 2023-10-09T08:39:10.901479+0000 mgr.x (mgr.24790) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:11.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:11 smithi016 bash[27454]: cluster 2023-10-09T08:39:10.901479+0000 mgr.x (mgr.24790) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:11.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:11 smithi016 bash[33851]: cluster 2023-10-09T08:39:10.901479+0000 mgr.x (mgr.24790) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:12.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:12 smithi094 bash[27803]: audit 2023-10-09T08:39:11.905605+0000 mon.a (mon.0) 796 : audit [DBG] from='client.? 172.21.15.16:0/1289786873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:12.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:12 smithi016 bash[27454]: audit 2023-10-09T08:39:11.905605+0000 mon.a (mon.0) 796 : audit [DBG] from='client.? 172.21.15.16:0/1289786873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:12.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:12 smithi016 bash[33851]: audit 2023-10-09T08:39:11.905605+0000 mon.a (mon.0) 796 : audit [DBG] from='client.? 172.21.15.16:0/1289786873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:13.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:13 smithi016 bash[27454]: cluster 2023-10-09T08:39:12.902107+0000 mgr.x (mgr.24790) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:13.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:13 smithi016 bash[33851]: cluster 2023-10-09T08:39:12.902107+0000 mgr.x (mgr.24790) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:13.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:13 smithi094 bash[27803]: cluster 2023-10-09T08:39:12.902107+0000 mgr.x (mgr.24790) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:14 smithi094 bash[27803]: audit 2023-10-09T08:39:14.348772+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.16:0/2407742975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:14.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:14 smithi016 bash[27454]: audit 2023-10-09T08:39:14.348772+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.16:0/2407742975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:14.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:14 smithi016 bash[33851]: audit 2023-10-09T08:39:14.348772+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.16:0/2407742975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:15 smithi094 bash[27803]: cluster 2023-10-09T08:39:14.902639+0000 mgr.x (mgr.24790) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:15.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:15 smithi016 bash[27454]: cluster 2023-10-09T08:39:14.902639+0000 mgr.x (mgr.24790) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:15.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:15 smithi016 bash[33851]: cluster 2023-10-09T08:39:14.902639+0000 mgr.x (mgr.24790) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:17.235 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:16 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:39:16] "GET /metrics HTTP/1.1" 200 33705 "" "Prometheus/2.33.4" 2023-10-09T08:39:17.236 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:16 smithi094 bash[27803]: audit 2023-10-09T08:39:16.805195+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.16:0/26062389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:17.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:16 smithi016 bash[27454]: audit 2023-10-09T08:39:16.805195+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.16:0/26062389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:17.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:16 smithi016 bash[33851]: audit 2023-10-09T08:39:16.805195+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.16:0/26062389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:18.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:17 smithi094 bash[27803]: cluster 2023-10-09T08:39:16.903655+0000 mgr.x (mgr.24790) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:18.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:17 smithi016 bash[27454]: cluster 2023-10-09T08:39:16.903655+0000 mgr.x (mgr.24790) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:18.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:17 smithi016 bash[33851]: cluster 2023-10-09T08:39:16.903655+0000 mgr.x (mgr.24790) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:18.953 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:18 smithi094 bash[53198]: ts=2023-10-09T08:39:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:39:20.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:19 smithi094 systemd[1]: Stopping Ceph grafana.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:39:20.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:19 smithi094 bash[53677]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-grafana.a 2023-10-09T08:39:20.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:19 smithi094 bash[47834]: t=2023-10-09T08:39:19+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T08:39:20.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:19 smithi094 bash[47834]: t=2023-10-09T08:39:19+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2023-10-09T08:39:20.034 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:19 smithi094 bash[27803]: cluster 2023-10-09T08:39:18.904325+0000 mgr.x (mgr.24790) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:20.034 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:19 smithi094 bash[27803]: audit 2023-10-09T08:39:19.252659+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.16:0/1187128714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:20.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:19 smithi016 bash[27454]: cluster 2023-10-09T08:39:18.904325+0000 mgr.x (mgr.24790) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:20.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:19 smithi016 bash[27454]: audit 2023-10-09T08:39:19.252659+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.16:0/1187128714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:20.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:19 smithi016 bash[33851]: cluster 2023-10-09T08:39:18.904325+0000 mgr.x (mgr.24790) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:20.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:19 smithi016 bash[33851]: audit 2023-10-09T08:39:19.252659+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.16:0/1187128714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:20.981 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:20 smithi094 bash[53710]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T08:39:20.981 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:20 smithi094 bash[53786]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T08:39:20.982 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:20 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T08:39:20.982 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:20 smithi094 systemd[1]: Stopped Ceph grafana.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:39:20.982 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:20 smithi094 systemd[1]: Started Ceph grafana.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:39:22.230 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: cluster 2023-10-09T08:39:20.905282+0000 mgr.x (mgr.24790) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:20.953027+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:20.961339+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.000784+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.003207+0000 mgr.x (mgr.24790) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.006897+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.008865+0000 mgr.x (mgr.24790) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.012243+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.014050+0000 mgr.x (mgr.24790) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.021391+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:22.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: cephadm 2023-10-09T08:39:21.027458+0000 mgr.x (mgr.24790) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:39:22.232 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: cephadm 2023-10-09T08:39:21.028145+0000 mgr.x (mgr.24790) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:39:22.232 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:21 smithi094 bash[27803]: audit 2023-10-09T08:39:21.703862+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.16:0/3963112207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:22.232 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T08:39:22.232 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T08:39:22.232 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T08:39:22.232 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T08:39:22.232 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T08:39:22.232 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T08:39:22.232 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T08:39:22.233 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.978679ms 2023-10-09T08:39:22.234 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T08:39:22.234 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T08:39:22.234 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T08:39:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: cluster 2023-10-09T08:39:20.905282+0000 mgr.x (mgr.24790) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:22.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:20.953027+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:20.961339+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.000784+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.003207+0000 mgr.x (mgr.24790) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.006897+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.008865+0000 mgr.x (mgr.24790) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.012243+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.014050+0000 mgr.x (mgr.24790) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.021391+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: cephadm 2023-10-09T08:39:21.027458+0000 mgr.x (mgr.24790) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:39:22.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: cephadm 2023-10-09T08:39:21.028145+0000 mgr.x (mgr.24790) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[27454]: audit 2023-10-09T08:39:21.703862+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.16:0/3963112207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: cluster 2023-10-09T08:39:20.905282+0000 mgr.x (mgr.24790) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:20.953027+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:20.961339+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.000784+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.003207+0000 mgr.x (mgr.24790) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.006897+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.008865+0000 mgr.x (mgr.24790) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:39:22.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.012243+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:39:22.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.014050+0000 mgr.x (mgr.24790) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T08:39:22.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.021391+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:22.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: cephadm 2023-10-09T08:39:21.027458+0000 mgr.x (mgr.24790) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:39:22.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: cephadm 2023-10-09T08:39:21.028145+0000 mgr.x (mgr.24790) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:39:22.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:21 smithi016 bash[33851]: audit 2023-10-09T08:39:21.703862+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.16:0/3963112207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T08:39:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T08:39:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T08:39:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-09T08:39:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T08:39:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-09T08:39:22.493 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T08:39:22.494 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T08:39:22.494 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 08:39:22 smithi094 bash[53813]: t=2023-10-09T08:39:22+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T08:39:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:23 smithi094 bash[27803]: cluster 2023-10-09T08:39:22.905961+0000 mgr.x (mgr.24790) 69 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:23 smithi094 bash[27803]: audit 2023-10-09T08:39:23.256651+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:23 smithi094 bash[27803]: audit 2023-10-09T08:39:23.258855+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:39:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:23 smithi094 bash[27803]: audit 2023-10-09T08:39:23.259482+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:39:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:23 smithi094 bash[27803]: audit 2023-10-09T08:39:23.261725+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:39:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:23 smithi094 bash[27803]: audit 2023-10-09T08:39:23.263432+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:39:24.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[27454]: cluster 2023-10-09T08:39:22.905961+0000 mgr.x (mgr.24790) 69 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:24.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[27454]: audit 2023-10-09T08:39:23.256651+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:24.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[27454]: audit 2023-10-09T08:39:23.258855+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:39:24.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[27454]: audit 2023-10-09T08:39:23.259482+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[27454]: audit 2023-10-09T08:39:23.261725+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[27454]: audit 2023-10-09T08:39:23.263432+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[33851]: cluster 2023-10-09T08:39:22.905961+0000 mgr.x (mgr.24790) 69 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[33851]: audit 2023-10-09T08:39:23.256651+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[33851]: audit 2023-10-09T08:39:23.258855+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[33851]: audit 2023-10-09T08:39:23.259482+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[33851]: audit 2023-10-09T08:39:23.261725+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:39:24.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:23 smithi016 bash[33851]: audit 2023-10-09T08:39:23.263432+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:39:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:24 smithi094 bash[27803]: cephadm 2023-10-09T08:39:23.247533+0000 mgr.x (mgr.24790) 70 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T08:39:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:24 smithi094 bash[27803]: cephadm 2023-10-09T08:39:23.266588+0000 mgr.x (mgr.24790) 71 : cephadm [INF] Deploying daemon mgr.y on smithi016 2023-10-09T08:39:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:24 smithi094 bash[27803]: audit 2023-10-09T08:39:24.122268+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:24 smithi094 bash[27803]: audit 2023-10-09T08:39:24.159783+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.16:0/1574468358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:25.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[33851]: cephadm 2023-10-09T08:39:23.247533+0000 mgr.x (mgr.24790) 70 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T08:39:25.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[33851]: cephadm 2023-10-09T08:39:23.266588+0000 mgr.x (mgr.24790) 71 : cephadm [INF] Deploying daemon mgr.y on smithi016 2023-10-09T08:39:25.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[33851]: audit 2023-10-09T08:39:24.122268+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:25.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[33851]: audit 2023-10-09T08:39:24.159783+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.16:0/1574468358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[27454]: cephadm 2023-10-09T08:39:23.247533+0000 mgr.x (mgr.24790) 70 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T08:39:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[27454]: cephadm 2023-10-09T08:39:23.266588+0000 mgr.x (mgr.24790) 71 : cephadm [INF] Deploying daemon mgr.y on smithi016 2023-10-09T08:39:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[27454]: audit 2023-10-09T08:39:24.122268+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:25.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:24 smithi016 bash[27454]: audit 2023-10-09T08:39:24.159783+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.16:0/1574468358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:25.943 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:25 smithi016 systemd[1]: Stopping Ceph mgr.y for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:39:26.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:25 smithi094 bash[27803]: cluster 2023-10-09T08:39:24.906847+0000 mgr.x (mgr.24790) 72 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:26.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:25 smithi016 bash[27454]: cluster 2023-10-09T08:39:24.906847+0000 mgr.x (mgr.24790) 72 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:26.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:25 smithi016 bash[33851]: cluster 2023-10-09T08:39:24.906847+0000 mgr.x (mgr.24790) 72 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:26.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:26 smithi016 bash[71083]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T08:39:26.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:26 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T08:39:26.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:26 smithi016 bash[71171]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T08:39:26.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:26 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T08:39:26.753 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:26 smithi016 systemd[1]: Stopped Ceph mgr.y for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:39:27.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:26 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:39:26] "GET /metrics HTTP/1.1" 200 33707 "" "Prometheus/2.33.4" 2023-10-09T08:39:27.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:26 smithi094 bash[27803]: audit 2023-10-09T08:39:26.602898+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.16:0/4157810548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:27.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:26 smithi094 bash[27803]: audit 2023-10-09T08:39:26.962144+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:27.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:26 smithi016 bash[33851]: audit 2023-10-09T08:39:26.602898+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.16:0/4157810548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:27.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:26 smithi016 bash[33851]: audit 2023-10-09T08:39:26.962144+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:27.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:26 smithi016 systemd[1]: Started Ceph mgr.y for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:39:27.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:26 smithi016 bash[27454]: audit 2023-10-09T08:39:26.602898+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.16:0/4157810548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:27.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:26 smithi016 bash[27454]: audit 2023-10-09T08:39:26.962144+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:28.214 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:28 smithi016 bash[71225]: debug 2023-10-09T08:39:28.036+0000 7f45827c1700 1 -- 172.21.15.16:0/1308469758 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558a3aa5f1e0 con 0x558a3b7de400 2023-10-09T08:39:28.217 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:27 smithi016 bash[27454]: cluster 2023-10-09T08:39:26.907412+0000 mgr.x (mgr.24790) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:28.217 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:27 smithi016 bash[27454]: audit 2023-10-09T08:39:26.969614+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:28.217 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:27 smithi016 bash[27454]: audit 2023-10-09T08:39:26.990849+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:28.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:27 smithi016 bash[33851]: cluster 2023-10-09T08:39:26.907412+0000 mgr.x (mgr.24790) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:28.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:27 smithi016 bash[33851]: audit 2023-10-09T08:39:26.969614+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:28.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:27 smithi016 bash[33851]: audit 2023-10-09T08:39:26.990849+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:28.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:27 smithi094 bash[27803]: cluster 2023-10-09T08:39:26.907412+0000 mgr.x (mgr.24790) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:28.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:27 smithi094 bash[27803]: audit 2023-10-09T08:39:26.969614+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:28.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:27 smithi094 bash[27803]: audit 2023-10-09T08:39:26.990849+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:28.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:28 smithi016 bash[71225]: debug 2023-10-09T08:39:28.204+0000 7f458c90f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:39:28.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:28 smithi094 bash[53198]: ts=2023-10-09T08:39:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:39:29.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[71225]: debug 2023-10-09T08:39:29.180+0000 7f458c90f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T08:39:29.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[71225]: debug 2023-10-09T08:39:29.288+0000 7f458c90f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:39:29.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[71225]: debug 2023-10-09T08:39:29.396+0000 7f458c90f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:39:29.987 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[71225]: debug 2023-10-09T08:39:29.676+0000 7f458c90f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:39:30.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:29 smithi094 bash[27803]: cluster 2023-10-09T08:39:28.908090+0000 mgr.x (mgr.24790) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:29 smithi094 bash[27803]: audit 2023-10-09T08:39:29.041852+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.16:0/3045002255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:30.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:30 smithi016 bash[71225]: debug 2023-10-09T08:39:30.052+0000 7f458c90f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:39:30.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:30 smithi016 bash[71225]: debug 2023-10-09T08:39:30.220+0000 7f458c90f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:39:30.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[27454]: cluster 2023-10-09T08:39:28.908090+0000 mgr.x (mgr.24790) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:30.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[27454]: audit 2023-10-09T08:39:29.041852+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.16:0/3045002255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:30.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[33851]: cluster 2023-10-09T08:39:28.908090+0000 mgr.x (mgr.24790) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:30.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:29 smithi016 bash[33851]: audit 2023-10-09T08:39:29.041852+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.16:0/3045002255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:30.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:30 smithi016 bash[71225]: debug 2023-10-09T08:39:30.448+0000 7f458c90f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:39:31.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:31 smithi016 bash[27454]: cluster 2023-10-09T08:39:30.909334+0000 mgr.x (mgr.24790) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:31.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:31 smithi016 bash[33851]: cluster 2023-10-09T08:39:30.909334+0000 mgr.x (mgr.24790) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:31 smithi094 bash[27803]: cluster 2023-10-09T08:39:30.909334+0000 mgr.x (mgr.24790) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:31.807 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:31 smithi016 bash[71225]: debug 2023-10-09T08:39:31.540+0000 7f458c90f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:39:31.807 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:31 smithi016 bash[71225]: debug 2023-10-09T08:39:31.668+0000 7f458c90f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T08:39:32.249 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:31 smithi016 bash[71225]: debug 2023-10-09T08:39:31.796+0000 7f458c90f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:39:32.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:32 smithi016 bash[71225]: debug 2023-10-09T08:39:32.348+0000 7f458c90f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T08:39:32.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:32 smithi016 bash[27454]: audit 2023-10-09T08:39:31.544293+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.16:0/413148289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:32.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:32 smithi016 bash[33851]: audit 2023-10-09T08:39:31.544293+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.16:0/413148289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:32 smithi094 bash[27803]: audit 2023-10-09T08:39:31.544293+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.16:0/413148289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:33.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:32 smithi016 bash[71225]: debug 2023-10-09T08:39:32.616+0000 7f458c90f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:39:33.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[71225]: debug 2023-10-09T08:39:33.160+0000 7f458c90f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:39:34.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[71225]: debug 2023-10-09T08:39:33.736+0000 7f458c90f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:39:34.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[27454]: cluster 2023-10-09T08:39:32.910042+0000 mgr.x (mgr.24790) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[27454]: audit 2023-10-09T08:39:32.984068+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[27454]: audit 2023-10-09T08:39:32.993849+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[27454]: audit 2023-10-09T08:39:33.239378+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[27454]: audit 2023-10-09T08:39:33.249291+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[27454]: audit 2023-10-09T08:39:33.797232+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[27454]: audit 2023-10-09T08:39:33.807824+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[33851]: cluster 2023-10-09T08:39:32.910042+0000 mgr.x (mgr.24790) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[33851]: audit 2023-10-09T08:39:32.984068+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[33851]: audit 2023-10-09T08:39:32.993849+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[33851]: audit 2023-10-09T08:39:33.239378+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[33851]: audit 2023-10-09T08:39:33.249291+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[33851]: audit 2023-10-09T08:39:33.797232+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:33 smithi016 bash[33851]: audit 2023-10-09T08:39:33.807824+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:33 smithi094 bash[27803]: cluster 2023-10-09T08:39:32.910042+0000 mgr.x (mgr.24790) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:33 smithi094 bash[27803]: audit 2023-10-09T08:39:32.984068+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:33 smithi094 bash[27803]: audit 2023-10-09T08:39:32.993849+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:33 smithi094 bash[27803]: audit 2023-10-09T08:39:33.239378+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:33 smithi094 bash[27803]: audit 2023-10-09T08:39:33.249291+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:33 smithi094 bash[27803]: audit 2023-10-09T08:39:33.797232+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:33 smithi094 bash[27803]: audit 2023-10-09T08:39:33.807824+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:34.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:34 smithi016 bash[71225]: debug 2023-10-09T08:39:34.396+0000 7f458c90f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:39:34.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:34 smithi016 bash[71225]: debug 2023-10-09T08:39:34.524+0000 7f458c90f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:39:35.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:34 smithi016 bash[33851]: audit 2023-10-09T08:39:33.997556+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.16:0/326830563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:35.197 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:34 smithi016 bash[71225]: debug 2023-10-09T08:39:34.916+0000 7f458c90f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:39:35.197 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:35 smithi016 bash[71225]: debug 2023-10-09T08:39:35.036+0000 7f458c90f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:39:35.198 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:34 smithi016 bash[27454]: audit 2023-10-09T08:39:33.997556+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.16:0/326830563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:35.452 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:35 smithi016 bash[71225]: debug 2023-10-09T08:39:35.188+0000 7f458c90f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:39:35.452 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:35 smithi016 bash[71225]: debug 2023-10-09T08:39:35.312+0000 7f458c90f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:39:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:34 smithi094 bash[27803]: audit 2023-10-09T08:39:33.997556+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.16:0/326830563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:35.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:35 smithi016 bash[71225]: debug 2023-10-09T08:39:35.444+0000 7f458c90f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:39:35.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:35 smithi016 bash[71225]: debug 2023-10-09T08:39:35.648+0000 7f458c90f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:39:36.223 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:35 smithi016 bash[71225]: debug 2023-10-09T08:39:35.784+0000 7f458c90f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:39:36.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:35 smithi016 bash[27454]: cluster 2023-10-09T08:39:34.911022+0000 mgr.x (mgr.24790) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:35 smithi094 bash[27803]: cluster 2023-10-09T08:39:34.911022+0000 mgr.x (mgr.24790) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:36.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:36 smithi016 bash[33851]: cluster 2023-10-09T08:39:34.911022+0000 mgr.x (mgr.24790) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:37.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:36 smithi094 bash[27803]: audit 2023-10-09T08:39:36.425804+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.16:0/431382123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:37.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:36 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:39:36] "GET /metrics HTTP/1.1" 200 33707 "" "Prometheus/2.33.4" 2023-10-09T08:39:37.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:36 smithi016 bash[27454]: audit 2023-10-09T08:39:36.425804+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.16:0/431382123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:37.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:36 smithi016 bash[33851]: audit 2023-10-09T08:39:36.425804+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.16:0/431382123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:37.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:37 smithi016 bash[71225]: debug 2023-10-09T08:39:37.084+0000 7f458c90f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:39:37.253 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:37 smithi016 bash[71225]: [09/Oct/2023:08:39:37] ENGINE Bus STARTING 2023-10-09T08:39:37.253 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:37 smithi016 bash[71225]: CherryPy Checker: 2023-10-09T08:39:37.253 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:37 smithi016 bash[71225]: The Application mounted at '' has an empty config. 2023-10-09T08:39:37.253 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:37 smithi016 bash[71225]: [09/Oct/2023:08:39:37] ENGINE Serving on http://:::9283 2023-10-09T08:39:37.253 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:37 smithi016 bash[71225]: [09/Oct/2023:08:39:37] ENGINE Bus STARTED 2023-10-09T08:39:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[27803]: cluster 2023-10-09T08:39:36.911645+0000 mgr.x (mgr.24790) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[27803]: cluster 2023-10-09T08:39:37.110107+0000 mon.a (mon.0) 814 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T08:39:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[27803]: cluster 2023-10-09T08:39:37.110325+0000 mon.a (mon.0) 815 : cluster [DBG] Standby manager daemon y started 2023-10-09T08:39:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[27803]: audit 2023-10-09T08:39:37.116958+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T08:39:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[27803]: audit 2023-10-09T08:39:37.118123+0000 mon.a (mon.0) 817 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:39:38.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[27803]: audit 2023-10-09T08:39:37.122381+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T08:39:38.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[27803]: audit 2023-10-09T08:39:37.124817+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:39:38.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[27454]: cluster 2023-10-09T08:39:36.911645+0000 mgr.x (mgr.24790) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:38.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[27454]: cluster 2023-10-09T08:39:37.110107+0000 mon.a (mon.0) 814 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T08:39:38.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[27454]: cluster 2023-10-09T08:39:37.110325+0000 mon.a (mon.0) 815 : cluster [DBG] Standby manager daemon y started 2023-10-09T08:39:38.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[27454]: audit 2023-10-09T08:39:37.116958+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T08:39:38.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[27454]: audit 2023-10-09T08:39:37.118123+0000 mon.a (mon.0) 817 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:39:38.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[27454]: audit 2023-10-09T08:39:37.122381+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T08:39:38.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[27454]: audit 2023-10-09T08:39:37.124817+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:39:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[33851]: cluster 2023-10-09T08:39:36.911645+0000 mgr.x (mgr.24790) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[33851]: cluster 2023-10-09T08:39:37.110107+0000 mon.a (mon.0) 814 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T08:39:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[33851]: cluster 2023-10-09T08:39:37.110325+0000 mon.a (mon.0) 815 : cluster [DBG] Standby manager daemon y started 2023-10-09T08:39:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[33851]: audit 2023-10-09T08:39:37.116958+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T08:39:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[33851]: audit 2023-10-09T08:39:37.118123+0000 mon.a (mon.0) 817 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:39:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[33851]: audit 2023-10-09T08:39:37.122381+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T08:39:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:38 smithi016 bash[33851]: audit 2023-10-09T08:39:37.124817+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.? 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:39:38.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:38 smithi094 bash[53198]: ts=2023-10-09T08:39:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:39:39.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:39 smithi094 bash[27803]: cluster 2023-10-09T08:39:38.017771+0000 mon.a (mon.0) 820 : cluster [DBG] mgrmap e25: x(active, since 89s), standbys: y 2023-10-09T08:39:39.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:39 smithi094 bash[27803]: audit 2023-10-09T08:39:38.894989+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.16:0/2428554026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:39.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:39 smithi016 bash[27454]: cluster 2023-10-09T08:39:38.017771+0000 mon.a (mon.0) 820 : cluster [DBG] mgrmap e25: x(active, since 89s), standbys: y 2023-10-09T08:39:39.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:39 smithi016 bash[27454]: audit 2023-10-09T08:39:38.894989+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.16:0/2428554026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:39.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:39 smithi016 bash[33851]: cluster 2023-10-09T08:39:38.017771+0000 mon.a (mon.0) 820 : cluster [DBG] mgrmap e25: x(active, since 89s), standbys: y 2023-10-09T08:39:39.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:39 smithi016 bash[33851]: audit 2023-10-09T08:39:38.894989+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.16:0/2428554026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:40.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:40 smithi016 bash[27454]: cluster 2023-10-09T08:39:38.912281+0000 mgr.x (mgr.24790) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:40.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:40 smithi016 bash[27454]: audit 2023-10-09T08:39:39.122841+0000 mon.c (mon.2) 211 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:40.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:40 smithi016 bash[33851]: cluster 2023-10-09T08:39:38.912281+0000 mgr.x (mgr.24790) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:40.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:40 smithi016 bash[33851]: audit 2023-10-09T08:39:39.122841+0000 mon.c (mon.2) 211 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:40.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:40 smithi094 bash[27803]: cluster 2023-10-09T08:39:38.912281+0000 mgr.x (mgr.24790) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:40 smithi094 bash[27803]: audit 2023-10-09T08:39:39.122841+0000 mon.c (mon.2) 211 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:41 smithi094 bash[27803]: cluster 2023-10-09T08:39:40.047501+0000 mon.a (mon.0) 821 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-10-09T08:39:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:41 smithi016 bash[27454]: cluster 2023-10-09T08:39:40.047501+0000 mon.a (mon.0) 821 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-10-09T08:39:41.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:41 smithi016 bash[33851]: cluster 2023-10-09T08:39:40.047501+0000 mon.a (mon.0) 821 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-10-09T08:39:42.387 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:42 smithi016 bash[33851]: cluster 2023-10-09T08:39:40.913426+0000 mgr.x (mgr.24790) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:39:42.387 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:42 smithi016 bash[33851]: audit 2023-10-09T08:39:41.345763+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.16:0/1547516960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:42.387 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:42 smithi016 bash[27454]: cluster 2023-10-09T08:39:40.913426+0000 mgr.x (mgr.24790) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:39:42.387 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:42 smithi016 bash[27454]: audit 2023-10-09T08:39:41.345763+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.16:0/1547516960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:42.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:42 smithi094 bash[27803]: cluster 2023-10-09T08:39:40.913426+0000 mgr.x (mgr.24790) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:39:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:42 smithi094 bash[27803]: audit 2023-10-09T08:39:41.345763+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.16:0/1547516960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:43.615 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[70330]: level=warn ts=2023-10-09T08:39:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi016.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.16:8443: connect: connection refused" 2023-10-09T08:39:43.615 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[70330]: level=warn ts=2023-10-09T08:39:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi094.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.94:8443: connect: connection refused" 2023-10-09T08:39:43.615 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: [09/Oct/2023:08:39:43] ENGINE Bus STOPPING 2023-10-09T08:39:43.615 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: [09/Oct/2023:08:39:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:39:43.615 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: [09/Oct/2023:08:39:43] ENGINE Bus STOPPED 2023-10-09T08:39:43.615 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: [09/Oct/2023:08:39:43] ENGINE Bus STARTING 2023-10-09T08:39:43.616 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: [09/Oct/2023:08:39:43] ENGINE Serving on http://:::9283 2023-10-09T08:39:43.616 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: [09/Oct/2023:08:39:43] ENGINE Bus STARTED 2023-10-09T08:39:43.637 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:43 smithi094 bash[50714]: debug 2023-10-09T08:39:43.340+0000 7f1d24636700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T08:39:43.637 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:43 smithi094 bash[50714]: ignoring --setuser ceph since I am not root 2023-10-09T08:39:43.638 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:43 smithi094 bash[50714]: ignoring --setgroup ceph since I am not root 2023-10-09T08:39:43.638 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:43 smithi094 bash[50714]: debug 2023-10-09T08:39:43.400+0000 7f1878c45700 1 -- 172.21.15.94:0/244791424 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c4bfbcb1e0 con 0x55c4c094a400 2023-10-09T08:39:43.638 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:43 smithi094 bash[50714]: debug 2023-10-09T08:39:43.568+0000 7f1882975200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:39:44.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.604+0000 7f44e9da9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.604+0000 7f44e9da9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.604+0000 7f44e9da9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.604+0000 7f44e9da9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.604+0000 7f44e9da9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.608+0000 7f44e65a2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.608+0000 7f44e65a2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.608+0000 7f44e65a2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.608+0000 7f44e65a2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:39:43 smithi016 bash[71225]: debug 2023-10-09T08:39:43.608+0000 7f44e65a2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:39:44.375 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: cluster 2023-10-09T08:39:42.914110+0000 mgr.x (mgr.24790) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:39:44.375 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.210678+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.375 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.220294+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.375 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.222639+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.224177+0000 mon.c (mon.2) 214 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.232888+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.319317+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.331025+0000 mon.c (mon.2) 216 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.331726+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: cluster 2023-10-09T08:39:43.344334+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.344761+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T08:39:44.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: cluster 2023-10-09T08:39:43.344865+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e27: y(active, starting, since 0.0123054s) 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.347860+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.348041+0000 mon.a (mon.0) 830 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.348222+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.348629+0000 mon.a (mon.0) 832 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.348884+0000 mon.a (mon.0) 833 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.349522+0000 mon.a (mon.0) 834 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.349751+0000 mon.a (mon.0) 835 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:39:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.349991+0000 mon.a (mon.0) 836 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.350273+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.350602+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.350929+0000 mon.a (mon.0) 839 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.351253+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.351593+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.351932+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.352294+0000 mon.a (mon.0) 843 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:39:44.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.352697+0000 mon.a (mon.0) 844 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.353301+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: cluster 2023-10-09T08:39:43.400695+0000 mon.a (mon.0) 846 : cluster [INF] Manager daemon y is now available 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.441570+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.444340+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.486121+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.499047+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.550555+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: audit 2023-10-09T08:39:43.839608+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.16:0/2563443017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:44.379 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[33851]: cephadm 2023-10-09T08:39:43.996320+0000 mgr.y (mgr.15261) 1 : cephadm [INF] [09/Oct/2023:08:39:43] ENGINE Bus STARTING 2023-10-09T08:39:44.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: cluster 2023-10-09T08:39:42.914110+0000 mgr.x (mgr.24790) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:39:44.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.210678+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.220294+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.222639+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:39:44.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.224177+0000 mon.c (mon.2) 214 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:39:44.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.232888+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.319317+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.331025+0000 mon.c (mon.2) 216 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T08:39:44.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.331726+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T08:39:44.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: cluster 2023-10-09T08:39:43.344334+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T08:39:44.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.344761+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T08:39:44.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: cluster 2023-10-09T08:39:43.344865+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e27: y(active, starting, since 0.0123054s) 2023-10-09T08:39:44.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.347860+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.348041+0000 mon.a (mon.0) 830 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.348222+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.348629+0000 mon.a (mon.0) 832 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.348884+0000 mon.a (mon.0) 833 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.349522+0000 mon.a (mon.0) 834 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.349751+0000 mon.a (mon.0) 835 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.349991+0000 mon.a (mon.0) 836 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.350273+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.350602+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.350929+0000 mon.a (mon.0) 839 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.351253+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:39:44.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.351593+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:39:44.383 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.351932+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:39:44.383 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.352294+0000 mon.a (mon.0) 843 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:39:44.383 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.352697+0000 mon.a (mon.0) 844 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:39:44.383 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.353301+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: cluster 2023-10-09T08:39:43.400695+0000 mon.a (mon.0) 846 : cluster [INF] Manager daemon y is now available 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.441570+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.444340+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.486121+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.499047+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.550555+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: audit 2023-10-09T08:39:43.839608+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.16:0/2563443017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:44.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[27454]: cephadm 2023-10-09T08:39:43.996320+0000 mgr.y (mgr.15261) 1 : cephadm [INF] [09/Oct/2023:08:39:43] ENGINE Bus STARTING 2023-10-09T08:39:44.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: cluster 2023-10-09T08:39:42.914110+0000 mgr.x (mgr.24790) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:39:44.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.210678+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.220294+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.222639+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.224177+0000 mon.c (mon.2) 214 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.232888+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24790 ' entity='mgr.x' 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.319317+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.331025+0000 mon.c (mon.2) 216 : audit [INF] from='mgr.24790 172.21.15.94:0/1992371713' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.331726+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: cluster 2023-10-09T08:39:43.344334+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.344761+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24790 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: cluster 2023-10-09T08:39:43.344865+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e27: y(active, starting, since 0.0123054s) 2023-10-09T08:39:44.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.347860+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.348041+0000 mon.a (mon.0) 830 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.348222+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.348629+0000 mon.a (mon.0) 832 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.348884+0000 mon.a (mon.0) 833 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.349522+0000 mon.a (mon.0) 834 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.349751+0000 mon.a (mon.0) 835 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.349991+0000 mon.a (mon.0) 836 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.350273+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.350602+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.350929+0000 mon.a (mon.0) 839 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.351253+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:39:44.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.351593+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.351932+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.352294+0000 mon.a (mon.0) 843 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.352697+0000 mon.a (mon.0) 844 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.353301+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: cluster 2023-10-09T08:39:43.400695+0000 mon.a (mon.0) 846 : cluster [INF] Manager daemon y is now available 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.441570+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.444340+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.486121+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.499047+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.550555+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:39:44.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: audit 2023-10-09T08:39:43.839608+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.16:0/2563443017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:44.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[27803]: cephadm 2023-10-09T08:39:43.996320+0000 mgr.y (mgr.15261) 1 : cephadm [INF] [09/Oct/2023:08:39:43] ENGINE Bus STARTING 2023-10-09T08:39:44.752 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:39:44 smithi016 bash[70330]: level=info ts=2023-10-09T08:39:44.371Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=2 2023-10-09T08:39:44.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[50714]: debug 2023-10-09T08:39:44.568+0000 7f1882975200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T08:39:44.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[50714]: debug 2023-10-09T08:39:44.676+0000 7f1882975200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:39:44.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:44 smithi094 bash[50714]: debug 2023-10-09T08:39:44.796+0000 7f1882975200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:39:45.354 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[50714]: debug 2023-10-09T08:39:45.088+0000 7f1882975200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:39:45.665 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[50714]: debug 2023-10-09T08:39:45.484+0000 7f1882975200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:39:45.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[27803]: cephadm 2023-10-09T08:39:44.098807+0000 mgr.y (mgr.15261) 2 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:39:45.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[27803]: cephadm 2023-10-09T08:39:44.279552+0000 mgr.y (mgr.15261) 3 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:39:45.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[27803]: cephadm 2023-10-09T08:39:44.290367+0000 mgr.y (mgr.15261) 4 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Bus STARTED 2023-10-09T08:39:45.666 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[27803]: cluster 2023-10-09T08:39:44.347844+0000 mon.a (mon.0) 853 : cluster [DBG] mgrmap e28: y(active, since 1.01528s) 2023-10-09T08:39:45.666 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[27803]: cluster 2023-10-09T08:39:44.356234+0000 mgr.y (mgr.15261) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:45.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[27454]: cephadm 2023-10-09T08:39:44.098807+0000 mgr.y (mgr.15261) 2 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:39:45.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[27454]: cephadm 2023-10-09T08:39:44.279552+0000 mgr.y (mgr.15261) 3 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:39:45.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[27454]: cephadm 2023-10-09T08:39:44.290367+0000 mgr.y (mgr.15261) 4 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Bus STARTED 2023-10-09T08:39:45.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[27454]: cluster 2023-10-09T08:39:44.347844+0000 mon.a (mon.0) 853 : cluster [DBG] mgrmap e28: y(active, since 1.01528s) 2023-10-09T08:39:45.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[27454]: cluster 2023-10-09T08:39:44.356234+0000 mgr.y (mgr.15261) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:45.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[33851]: cephadm 2023-10-09T08:39:44.098807+0000 mgr.y (mgr.15261) 2 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:39:45.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[33851]: cephadm 2023-10-09T08:39:44.279552+0000 mgr.y (mgr.15261) 3 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:39:45.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[33851]: cephadm 2023-10-09T08:39:44.290367+0000 mgr.y (mgr.15261) 4 : cephadm [INF] [09/Oct/2023:08:39:44] ENGINE Bus STARTED 2023-10-09T08:39:45.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[33851]: cluster 2023-10-09T08:39:44.347844+0000 mon.a (mon.0) 853 : cluster [DBG] mgrmap e28: y(active, since 1.01528s) 2023-10-09T08:39:45.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:45 smithi016 bash[33851]: cluster 2023-10-09T08:39:44.356234+0000 mgr.y (mgr.15261) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:45.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[50714]: debug 2023-10-09T08:39:45.656+0000 7f1882975200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:39:45.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:45 smithi094 bash[50714]: debug 2023-10-09T08:39:45.876+0000 7f1882975200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:39:46.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:46 smithi094 bash[27803]: cluster 2023-10-09T08:39:45.351172+0000 mgr.y (mgr.15261) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:46 smithi094 bash[27803]: cluster 2023-10-09T08:39:45.356629+0000 mon.a (mon.0) 854 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-10-09T08:39:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:46 smithi094 bash[27803]: audit 2023-10-09T08:39:46.310799+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.16:0/556498589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:46.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:46 smithi016 bash[27454]: cluster 2023-10-09T08:39:45.351172+0000 mgr.y (mgr.15261) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:46.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:46 smithi016 bash[27454]: cluster 2023-10-09T08:39:45.356629+0000 mon.a (mon.0) 854 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-10-09T08:39:46.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:46 smithi016 bash[27454]: audit 2023-10-09T08:39:46.310799+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.16:0/556498589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:46.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:46 smithi016 bash[33851]: cluster 2023-10-09T08:39:45.351172+0000 mgr.y (mgr.15261) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:46.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:46 smithi016 bash[33851]: cluster 2023-10-09T08:39:45.356629+0000 mon.a (mon.0) 854 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-10-09T08:39:46.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:46 smithi016 bash[33851]: audit 2023-10-09T08:39:46.310799+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.16:0/556498589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:47.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:46 smithi094 bash[50714]: debug 2023-10-09T08:39:46.904+0000 7f1882975200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:39:47.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:47 smithi094 bash[50714]: debug 2023-10-09T08:39:47.020+0000 7f1882975200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T08:39:47.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:47 smithi094 bash[50714]: debug 2023-10-09T08:39:47.140+0000 7f1882975200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:39:47.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:47 smithi094 bash[50714]: debug 2023-10-09T08:39:47.624+0000 7f1882975200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T08:39:47.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:47 smithi094 bash[50714]: debug 2023-10-09T08:39:47.868+0000 7f1882975200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:39:48.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:48 smithi094 bash[27803]: cluster 2023-10-09T08:39:47.351873+0000 mgr.y (mgr.15261) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:48.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:48 smithi094 bash[50714]: debug 2023-10-09T08:39:48.364+0000 7f1882975200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:39:48.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:48 smithi016 bash[27454]: cluster 2023-10-09T08:39:47.351873+0000 mgr.y (mgr.15261) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:48.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:48 smithi016 bash[33851]: cluster 2023-10-09T08:39:47.351873+0000 mgr.y (mgr.15261) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:49.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:48 smithi094 bash[50714]: debug 2023-10-09T08:39:48.864+0000 7f1882975200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:39:49.641 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:49 smithi094 bash[27803]: audit 2023-10-09T08:39:48.762641+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.16:0/1738558987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:49.641 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:49 smithi094 bash[50714]: debug 2023-10-09T08:39:49.484+0000 7f1882975200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:39:49.641 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:49 smithi094 bash[50714]: debug 2023-10-09T08:39:49.632+0000 7f1882975200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:39:49.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:49 smithi016 bash[27454]: audit 2023-10-09T08:39:48.762641+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.16:0/1738558987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:49 smithi016 bash[33851]: audit 2023-10-09T08:39:48.762641+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.16:0/1738558987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:50.368 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:50 smithi094 bash[50714]: debug 2023-10-09T08:39:50.056+0000 7f1882975200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:39:50.368 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:50 smithi094 bash[50714]: debug 2023-10-09T08:39:50.184+0000 7f1882975200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:39:50.645 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:50 smithi094 bash[50714]: debug 2023-10-09T08:39:50.360+0000 7f1882975200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:39:50.645 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:50 smithi094 bash[50714]: debug 2023-10-09T08:39:50.496+0000 7f1882975200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:39:50.645 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:50 smithi094 bash[50714]: debug 2023-10-09T08:39:50.636+0000 7f1882975200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:39:50.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:50 smithi094 bash[50714]: debug 2023-10-09T08:39:50.852+0000 7f1882975200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:39:51.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:50 smithi094 bash[50714]: debug 2023-10-09T08:39:50.992+0000 7f1882975200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:39:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: cluster 2023-10-09T08:39:49.352615+0000 mgr.y (mgr.15261) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.041263+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.048482+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.131118+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.137727+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.937028+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.947662+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.951220+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:51 smithi094 bash[27803]: audit 2023-10-09T08:39:50.956434+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: cluster 2023-10-09T08:39:49.352615+0000 mgr.y (mgr.15261) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.041263+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.048482+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.131118+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.137727+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.937028+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.947662+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.951220+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[27454]: audit 2023-10-09T08:39:50.956434+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: cluster 2023-10-09T08:39:49.352615+0000 mgr.y (mgr.15261) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.041263+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.048482+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.131118+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.137727+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.937028+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.947662+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.951220+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:51.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:51 smithi016 bash[33851]: audit 2023-10-09T08:39:50.956434+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:39:52.307 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:52 smithi094 bash[50714]: debug 2023-10-09T08:39:52.272+0000 7f1882975200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:39:52.307 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:52 smithi094 bash[27803]: audit 2023-10-09T08:39:51.236302+0000 mon.c (mon.2) 219 : audit [DBG] from='client.? 172.21.15.16:0/3878746230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:52.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:52 smithi016 bash[27454]: audit 2023-10-09T08:39:51.236302+0000 mon.c (mon.2) 219 : audit [DBG] from='client.? 172.21.15.16:0/3878746230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:52.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:52 smithi016 bash[33851]: audit 2023-10-09T08:39:51.236302+0000 mon.c (mon.2) 219 : audit [DBG] from='client.? 172.21.15.16:0/3878746230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:52.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:52 smithi094 bash[50714]: [09/Oct/2023:08:39:52] ENGINE Bus STARTING 2023-10-09T08:39:52.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:52 smithi094 bash[50714]: CherryPy Checker: 2023-10-09T08:39:52.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:52 smithi094 bash[50714]: The Application mounted at '' has an empty config. 2023-10-09T08:39:52.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:52 smithi094 bash[50714]: [09/Oct/2023:08:39:52] ENGINE Serving on http://:::9283 2023-10-09T08:39:52.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:52 smithi094 bash[50714]: [09/Oct/2023:08:39:52] ENGINE Bus STARTED 2023-10-09T08:39:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:53 smithi094 bash[27803]: cluster 2023-10-09T08:39:51.353695+0000 mgr.y (mgr.15261) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:53 smithi094 bash[27803]: cluster 2023-10-09T08:39:52.298905+0000 mon.a (mon.0) 863 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:39:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:53 smithi094 bash[27803]: audit 2023-10-09T08:39:52.304268+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:39:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:53 smithi094 bash[27803]: audit 2023-10-09T08:39:52.305587+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:39:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:53 smithi094 bash[27803]: audit 2023-10-09T08:39:52.310624+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:39:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:53 smithi094 bash[27803]: audit 2023-10-09T08:39:52.311180+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:39:53.502 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[70330]: level=info ts=2023-10-09T08:39:53.245Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=7 2023-10-09T08:39:53.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[33851]: cluster 2023-10-09T08:39:51.353695+0000 mgr.y (mgr.15261) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:53.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[33851]: cluster 2023-10-09T08:39:52.298905+0000 mon.a (mon.0) 863 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:39:53.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[33851]: audit 2023-10-09T08:39:52.304268+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:39:53.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[33851]: audit 2023-10-09T08:39:52.305587+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:39:53.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[33851]: audit 2023-10-09T08:39:52.310624+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:39:53.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[33851]: audit 2023-10-09T08:39:52.311180+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:39:53.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[27454]: cluster 2023-10-09T08:39:51.353695+0000 mgr.y (mgr.15261) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:53.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[27454]: cluster 2023-10-09T08:39:52.298905+0000 mon.a (mon.0) 863 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:39:53.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[27454]: audit 2023-10-09T08:39:52.304268+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:39:53.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[27454]: audit 2023-10-09T08:39:52.305587+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:39:53.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[27454]: audit 2023-10-09T08:39:52.310624+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:39:53.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:53 smithi016 bash[27454]: audit 2023-10-09T08:39:52.311180+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.94:0/51699335' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:39:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:54 smithi094 bash[27803]: cluster 2023-10-09T08:39:53.062712+0000 mon.a (mon.0) 864 : cluster [DBG] mgrmap e30: y(active, since 9s), standbys: x 2023-10-09T08:39:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:54 smithi094 bash[27803]: audit 2023-10-09T08:39:53.068267+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:39:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:54 smithi094 bash[27803]: cluster 2023-10-09T08:39:53.354250+0000 mgr.y (mgr.15261) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:54 smithi094 bash[27803]: audit 2023-10-09T08:39:53.692708+0000 mon.c (mon.2) 220 : audit [DBG] from='client.? 172.21.15.16:0/1871508814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:54.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[27454]: cluster 2023-10-09T08:39:53.062712+0000 mon.a (mon.0) 864 : cluster [DBG] mgrmap e30: y(active, since 9s), standbys: x 2023-10-09T08:39:54.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[27454]: audit 2023-10-09T08:39:53.068267+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:39:54.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[27454]: cluster 2023-10-09T08:39:53.354250+0000 mgr.y (mgr.15261) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:54.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[27454]: audit 2023-10-09T08:39:53.692708+0000 mon.c (mon.2) 220 : audit [DBG] from='client.? 172.21.15.16:0/1871508814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:54.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[33851]: cluster 2023-10-09T08:39:53.062712+0000 mon.a (mon.0) 864 : cluster [DBG] mgrmap e30: y(active, since 9s), standbys: x 2023-10-09T08:39:54.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[33851]: audit 2023-10-09T08:39:53.068267+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:39:54.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[33851]: cluster 2023-10-09T08:39:53.354250+0000 mgr.y (mgr.15261) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:54.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:54 smithi016 bash[33851]: audit 2023-10-09T08:39:53.692708+0000 mon.c (mon.2) 220 : audit [DBG] from='client.? 172.21.15.16:0/1871508814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:56.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:56 smithi016 bash[27454]: cluster 2023-10-09T08:39:55.355346+0000 mgr.y (mgr.15261) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:56.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:56 smithi016 bash[27454]: audit 2023-10-09T08:39:56.157397+0000 mon.c (mon.2) 221 : audit [DBG] from='client.? 172.21.15.16:0/1970656292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:56.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:56 smithi016 bash[33851]: cluster 2023-10-09T08:39:55.355346+0000 mgr.y (mgr.15261) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:56.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:56 smithi016 bash[33851]: audit 2023-10-09T08:39:56.157397+0000 mon.c (mon.2) 221 : audit [DBG] from='client.? 172.21.15.16:0/1970656292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:56.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:56 smithi094 bash[27803]: cluster 2023-10-09T08:39:55.355346+0000 mgr.y (mgr.15261) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:56.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:56 smithi094 bash[27803]: audit 2023-10-09T08:39:56.157397+0000 mon.c (mon.2) 221 : audit [DBG] from='client.? 172.21.15.16:0/1970656292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:56.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:39:56 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:39:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:39:56.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:56 smithi094 bash[53198]: ts=2023-10-09T08:39:56.652Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.94:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.94:8765: connect: connection refused" 2023-10-09T08:39:56.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:56 smithi094 bash[53198]: ts=2023-10-09T08:39:56.655Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.94:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.94:8765: connect: connection refused" 2023-10-09T08:39:56.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:56 smithi094 bash[53198]: ts=2023-10-09T08:39:56.656Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.94:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.94:8765: connect: connection refused" 2023-10-09T08:39:56.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:39:56 smithi094 bash[53198]: ts=2023-10-09T08:39:56.658Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.94:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.94:8765: connect: connection refused" 2023-10-09T08:39:58.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:58 smithi094 bash[27803]: cluster 2023-10-09T08:39:57.356062+0000 mgr.y (mgr.15261) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:58 smithi016 bash[33851]: cluster 2023-10-09T08:39:57.356062+0000 mgr.y (mgr.15261) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:58.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:58 smithi016 bash[27454]: cluster 2023-10-09T08:39:57.356062+0000 mgr.y (mgr.15261) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:39:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:59 smithi016 bash[27454]: audit 2023-10-09T08:39:58.486020+0000 mon.a (mon.0) 866 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:39:59 smithi016 bash[27454]: audit 2023-10-09T08:39:58.606882+0000 mon.c (mon.2) 222 : audit [DBG] from='client.? 172.21.15.16:0/4008437850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:59.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:59 smithi016 bash[33851]: audit 2023-10-09T08:39:58.486020+0000 mon.a (mon.0) 866 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:59.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:39:59 smithi016 bash[33851]: audit 2023-10-09T08:39:58.606882+0000 mon.c (mon.2) 222 : audit [DBG] from='client.? 172.21.15.16:0/4008437850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:39:59.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:59 smithi094 bash[27803]: audit 2023-10-09T08:39:58.486020+0000 mon.a (mon.0) 866 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:39:59.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:39:59 smithi094 bash[27803]: audit 2023-10-09T08:39:58.606882+0000 mon.c (mon.2) 222 : audit [DBG] from='client.? 172.21.15.16:0/4008437850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:00.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:00 smithi094 bash[27803]: cluster 2023-10-09T08:39:59.356591+0000 mgr.y (mgr.15261) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:00.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:00 smithi094 bash[27803]: cluster 2023-10-09T08:40:00.000153+0000 mon.a (mon.0) 867 : cluster [INF] overall HEALTH_OK 2023-10-09T08:40:00.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:00 smithi016 bash[33851]: cluster 2023-10-09T08:39:59.356591+0000 mgr.y (mgr.15261) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:00.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:00 smithi016 bash[33851]: cluster 2023-10-09T08:40:00.000153+0000 mon.a (mon.0) 867 : cluster [INF] overall HEALTH_OK 2023-10-09T08:40:00.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:00 smithi016 bash[27454]: cluster 2023-10-09T08:39:59.356591+0000 mgr.y (mgr.15261) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:00.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:00 smithi016 bash[27454]: cluster 2023-10-09T08:40:00.000153+0000 mon.a (mon.0) 867 : cluster [INF] overall HEALTH_OK 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.010125+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.017848+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.020134+0000 mon.a (mon.0) 870 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.021338+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.023103+0000 mgr.y (mgr.15261) 14 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.023394+0000 mgr.y (mgr.15261) 15 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.069264+0000 mon.c (mon.2) 223 : audit [DBG] from='client.? 172.21.15.16:0/4201847824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.690843+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.697780+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.397 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.742475+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.397 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.750120+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.397 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:02 smithi094 bash[27803]: audit 2023-10-09T08:40:01.756873+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.010125+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.017848+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.020134+0000 mon.a (mon.0) 870 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.021338+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.023103+0000 mgr.y (mgr.15261) 14 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.023394+0000 mgr.y (mgr.15261) 15 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.069264+0000 mon.c (mon.2) 223 : audit [DBG] from='client.? 172.21.15.16:0/4201847824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.690843+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.697780+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.742475+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.750120+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[27454]: audit 2023-10-09T08:40:01.756873+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.010125+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.017848+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.020134+0000 mon.a (mon.0) 870 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.021338+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.023103+0000 mgr.y (mgr.15261) 14 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.023394+0000 mgr.y (mgr.15261) 15 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.069264+0000 mon.c (mon.2) 223 : audit [DBG] from='client.? 172.21.15.16:0/4201847824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:02.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.690843+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.697780+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.742475+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.750120+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:02.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:02 smithi016 bash[33851]: audit 2023-10-09T08:40:01.756873+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:03.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.187916+0000 mgr.y (mgr.15261) 16 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:40:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.239122+0000 mgr.y (mgr.15261) 17 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:40:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.352424+0000 mgr.y (mgr.15261) 18 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:40:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cluster 2023-10-09T08:40:01.357118+0000 mgr.y (mgr.15261) 19 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.404481+0000 mgr.y (mgr.15261) 20 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:40:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.517057+0000 mgr.y (mgr.15261) 21 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:40:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.568903+0000 mgr.y (mgr.15261) 22 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:40:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:03 smithi094 bash[27803]: cephadm 2023-10-09T08:40:01.831073+0000 mgr.y (mgr.15261) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:40:03.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.187916+0000 mgr.y (mgr.15261) 16 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:40:03.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.239122+0000 mgr.y (mgr.15261) 17 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.352424+0000 mgr.y (mgr.15261) 18 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cluster 2023-10-09T08:40:01.357118+0000 mgr.y (mgr.15261) 19 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.404481+0000 mgr.y (mgr.15261) 20 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.517057+0000 mgr.y (mgr.15261) 21 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.568903+0000 mgr.y (mgr.15261) 22 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[27454]: cephadm 2023-10-09T08:40:01.831073+0000 mgr.y (mgr.15261) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.187916+0000 mgr.y (mgr.15261) 16 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.239122+0000 mgr.y (mgr.15261) 17 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:40:03.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.352424+0000 mgr.y (mgr.15261) 18 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:40:03.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cluster 2023-10-09T08:40:01.357118+0000 mgr.y (mgr.15261) 19 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:03.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.404481+0000 mgr.y (mgr.15261) 20 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:40:03.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.517057+0000 mgr.y (mgr.15261) 21 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:40:03.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.568903+0000 mgr.y (mgr.15261) 22 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:40:03.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:03 smithi016 bash[33851]: cephadm 2023-10-09T08:40:01.831073+0000 mgr.y (mgr.15261) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:40:04.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:04 smithi094 bash[27803]: cephadm 2023-10-09T08:40:02.099144+0000 mgr.y (mgr.15261) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:40:04.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:04 smithi094 bash[27803]: audit 2023-10-09T08:40:03.521482+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.16:0/1458214145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:04.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:04 smithi016 bash[27454]: cephadm 2023-10-09T08:40:02.099144+0000 mgr.y (mgr.15261) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:40:04.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:04 smithi016 bash[27454]: audit 2023-10-09T08:40:03.521482+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.16:0/1458214145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:04 smithi016 bash[33851]: cephadm 2023-10-09T08:40:02.099144+0000 mgr.y (mgr.15261) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:40:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:04 smithi016 bash[33851]: audit 2023-10-09T08:40:03.521482+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.16:0/1458214145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:05 smithi094 bash[27803]: cluster 2023-10-09T08:40:03.357565+0000 mgr.y (mgr.15261) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:05.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:05 smithi016 bash[27454]: cluster 2023-10-09T08:40:03.357565+0000 mgr.y (mgr.15261) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:05.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:05 smithi016 bash[33851]: cluster 2023-10-09T08:40:03.357565+0000 mgr.y (mgr.15261) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[27803]: cluster 2023-10-09T08:40:05.358591+0000 mgr.y (mgr.15261) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[27803]: audit 2023-10-09T08:40:05.968255+0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.16:0/1111158312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:06.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:06 smithi016 bash[27454]: cluster 2023-10-09T08:40:05.358591+0000 mgr.y (mgr.15261) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:06.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:06 smithi016 bash[27454]: audit 2023-10-09T08:40:05.968255+0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.16:0/1111158312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:06.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:06 smithi016 bash[33851]: cluster 2023-10-09T08:40:05.358591+0000 mgr.y (mgr.15261) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:06.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:06 smithi016 bash[33851]: audit 2023-10-09T08:40:05.968255+0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.16:0/1111158312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:07.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[50714]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:40:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 systemd[1]: Stopping Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[55069]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.973Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.973Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.973Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.973Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.973Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T08:40:07.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.973Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T08:40:07.244 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.973Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T08:40:07.244 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.974Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T08:40:07.244 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.974Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T08:40:07.244 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.974Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T08:40:07.244 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.974Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T08:40:07.244 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:06 smithi094 bash[53198]: ts=2023-10-09T08:40:06.974Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T08:40:07.884 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:07 smithi094 bash[55103]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T08:40:07.884 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:07 smithi094 bash[55167]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T08:40:07.884 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:07 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T08:40:07.885 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:07 smithi094 systemd[1]: Stopped Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:40:07.885 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:07 smithi094 systemd[1]: Started Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:40:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:08 smithi094 bash[27803]: cluster 2023-10-09T08:40:07.359165+0000 mgr.y (mgr.15261) 27 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:08 smithi094 bash[27803]: audit 2023-10-09T08:40:07.859259+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:08 smithi094 bash[27803]: audit 2023-10-09T08:40:07.866730+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:08 smithi094 bash[27803]: audit 2023-10-09T08:40:07.894109+0000 mon.a (mon.0) 881 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:40:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:08 smithi094 bash[27803]: audit 2023-10-09T08:40:07.894775+0000 mgr.y (mgr.15261) 28 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:40:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:08 smithi094 bash[27803]: audit 2023-10-09T08:40:07.902393+0000 mon.a (mon.0) 882 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:08 smithi094 bash[27803]: audit 2023-10-09T08:40:08.428021+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.16:0/1646733659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:09.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[27454]: cluster 2023-10-09T08:40:07.359165+0000 mgr.y (mgr.15261) 27 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[27454]: audit 2023-10-09T08:40:07.859259+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[27454]: audit 2023-10-09T08:40:07.866730+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[27454]: audit 2023-10-09T08:40:07.894109+0000 mon.a (mon.0) 881 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[27454]: audit 2023-10-09T08:40:07.894775+0000 mgr.y (mgr.15261) 28 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[27454]: audit 2023-10-09T08:40:07.902393+0000 mon.a (mon.0) 882 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[27454]: audit 2023-10-09T08:40:08.428021+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.16:0/1646733659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[33851]: cluster 2023-10-09T08:40:07.359165+0000 mgr.y (mgr.15261) 27 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[33851]: audit 2023-10-09T08:40:07.859259+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:09.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[33851]: audit 2023-10-09T08:40:07.866730+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:09.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[33851]: audit 2023-10-09T08:40:07.894109+0000 mon.a (mon.0) 881 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:40:09.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[33851]: audit 2023-10-09T08:40:07.894775+0000 mgr.y (mgr.15261) 28 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T08:40:09.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[33851]: audit 2023-10-09T08:40:07.902393+0000 mon.a (mon.0) 882 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:09.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:08 smithi016 bash[33851]: audit 2023-10-09T08:40:08.428021+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.16:0/1646733659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:09.700 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.530Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T08:40:09.701 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.530Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T08:40:09.701 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.531Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi094 (none))" 2023-10-09T08:40:09.701 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.531Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T08:40:09.701 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.531Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T08:40:09.701 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.533Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T08:40:09.701 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.533Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T08:40:09.702 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.534Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T08:40:09.702 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.538Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T08:40:09.702 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.538Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.893µs 2023-10-09T08:40:09.702 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.538Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T08:40:09.979 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.719Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T08:40:09.979 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.932Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T08:40:10.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.976Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T08:40:10.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.977Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T08:40:10.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.977Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=487.917µs wal_replay_duration=438.183604ms total_replay_duration=438.695284ms 2023-10-09T08:40:10.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.982Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T08:40:10.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.982Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T08:40:10.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:09 smithi094 bash[55195]: ts=2023-10-09T08:40:09.982Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T08:40:10.243 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:10 smithi094 bash[55195]: ts=2023-10-09T08:40:10.005Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.633313ms db_storage=1.611µs remote_storage=2.238µs web_handler=631ns query_engine=1.151µs scrape=440.566µs scrape_sd=183.565µs notify=42.157µs notify_sd=45.679µs rules=21.418934ms 2023-10-09T08:40:10.244 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:10 smithi094 bash[55195]: ts=2023-10-09T08:40:10.005Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T08:40:11.245 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:10 smithi094 bash[27803]: cluster 2023-10-09T08:40:09.360147+0000 mgr.y (mgr.15261) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:11.246 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:10 smithi094 bash[27803]: audit 2023-10-09T08:40:10.528208+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:40:11.246 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:10 smithi094 bash[27803]: audit 2023-10-09T08:40:10.530141+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:40:11.246 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:10 smithi094 bash[27803]: audit 2023-10-09T08:40:10.532018+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:11.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[27454]: cluster 2023-10-09T08:40:09.360147+0000 mgr.y (mgr.15261) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:11.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[27454]: audit 2023-10-09T08:40:10.528208+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:40:11.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[27454]: audit 2023-10-09T08:40:10.530141+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:40:11.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[27454]: audit 2023-10-09T08:40:10.532018+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:11.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[33851]: cluster 2023-10-09T08:40:09.360147+0000 mgr.y (mgr.15261) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:11.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[33851]: audit 2023-10-09T08:40:10.528208+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:40:11.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[33851]: audit 2023-10-09T08:40:10.530141+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:40:11.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:10 smithi016 bash[33851]: audit 2023-10-09T08:40:10.532018+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:12.186 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:11 smithi094 bash[27803]: cephadm 2023-10-09T08:40:10.527339+0000 mgr.y (mgr.15261) 30 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T08:40:12.186 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:11 smithi094 bash[27803]: cephadm 2023-10-09T08:40:10.533809+0000 mgr.y (mgr.15261) 31 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:40:12.186 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:11 smithi094 bash[27803]: audit 2023-10-09T08:40:10.888199+0000 mon.c (mon.2) 224 : audit [DBG] from='client.? 172.21.15.16:0/746720298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:12.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:11 smithi016 bash[27454]: cephadm 2023-10-09T08:40:10.527339+0000 mgr.y (mgr.15261) 30 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T08:40:12.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:11 smithi016 bash[27454]: cephadm 2023-10-09T08:40:10.533809+0000 mgr.y (mgr.15261) 31 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:40:12.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:11 smithi016 bash[27454]: audit 2023-10-09T08:40:10.888199+0000 mon.c (mon.2) 224 : audit [DBG] from='client.? 172.21.15.16:0/746720298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:12.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:11 smithi016 bash[33851]: cephadm 2023-10-09T08:40:10.527339+0000 mgr.y (mgr.15261) 30 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T08:40:12.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:11 smithi016 bash[33851]: cephadm 2023-10-09T08:40:10.533809+0000 mgr.y (mgr.15261) 31 : cephadm [INF] Deploying daemon mgr.x on smithi094 2023-10-09T08:40:12.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:11 smithi016 bash[33851]: audit 2023-10-09T08:40:10.888199+0000 mon.c (mon.2) 224 : audit [DBG] from='client.? 172.21.15.16:0/746720298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:13.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:12 smithi094 bash[27803]: cluster 2023-10-09T08:40:11.360885+0000 mgr.y (mgr.15261) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:13.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:12 smithi016 bash[27454]: cluster 2023-10-09T08:40:11.360885+0000 mgr.y (mgr.15261) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:13.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:12 smithi016 bash[33851]: cluster 2023-10-09T08:40:11.360885+0000 mgr.y (mgr.15261) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:13.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:13 smithi094 systemd[1]: Stopping Ceph mgr.x for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:40:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:13 smithi094 bash[27803]: audit 2023-10-09T08:40:13.339483+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.16:0/1226367305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:14.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:13 smithi094 bash[27803]: audit 2023-10-09T08:40:13.487158+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:14.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:13 smithi094 bash[55495]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T08:40:14.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:14 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T08:40:14.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:14 smithi094 bash[55557]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T08:40:14.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:14 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T08:40:14.244 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:14 smithi094 systemd[1]: Stopped Ceph mgr.x for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:40:14.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:13 smithi016 bash[27454]: audit 2023-10-09T08:40:13.339483+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.16:0/1226367305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:14.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:13 smithi016 bash[27454]: audit 2023-10-09T08:40:13.487158+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:14.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:13 smithi016 bash[33851]: audit 2023-10-09T08:40:13.339483+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.16:0/1226367305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:14.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:13 smithi016 bash[33851]: audit 2023-10-09T08:40:13.487158+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:14.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:14 smithi094 systemd[1]: Started Ceph mgr.x for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:40:15.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:14 smithi094 bash[27803]: cluster 2023-10-09T08:40:13.361569+0000 mgr.y (mgr.15261) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:15.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:14 smithi094 bash[27803]: audit 2023-10-09T08:40:14.355549+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:15.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:14 smithi094 bash[27803]: audit 2023-10-09T08:40:14.362891+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:15.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:14 smithi094 bash[27803]: audit 2023-10-09T08:40:14.411813+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:15.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[27454]: cluster 2023-10-09T08:40:13.361569+0000 mgr.y (mgr.15261) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:15.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[27454]: audit 2023-10-09T08:40:14.355549+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:15.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[27454]: audit 2023-10-09T08:40:14.362891+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:15.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[27454]: audit 2023-10-09T08:40:14.411813+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:15.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[33851]: cluster 2023-10-09T08:40:13.361569+0000 mgr.y (mgr.15261) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:15.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[33851]: audit 2023-10-09T08:40:14.355549+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:15.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[33851]: audit 2023-10-09T08:40:14.362891+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:15.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:14 smithi016 bash[33851]: audit 2023-10-09T08:40:14.411813+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:15.600 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:15 smithi094 bash[55615]: debug 2023-10-09T08:40:15.436+0000 7f2194313700 1 -- 172.21.15.94:0/276347385 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f0beb3f1e0 con 0x55f0bf8bec00 2023-10-09T08:40:15.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:15 smithi094 bash[55615]: debug 2023-10-09T08:40:15.596+0000 7f219e461200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:40:16.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:40:15] "GET /metrics HTTP/1.1" 200 33709 "" "Prometheus/2.33.4" 2023-10-09T08:40:16.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:16 smithi094 bash[27803]: cluster 2023-10-09T08:40:15.362921+0000 mgr.y (mgr.15261) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:16.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:16 smithi094 bash[27803]: audit 2023-10-09T08:40:15.798259+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.16:0/557818160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:16.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:16 smithi016 bash[27454]: cluster 2023-10-09T08:40:15.362921+0000 mgr.y (mgr.15261) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:16.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:16 smithi016 bash[27454]: audit 2023-10-09T08:40:15.798259+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.16:0/557818160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:16.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:16 smithi016 bash[33851]: cluster 2023-10-09T08:40:15.362921+0000 mgr.y (mgr.15261) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:16.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:16 smithi016 bash[33851]: audit 2023-10-09T08:40:15.798259+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.16:0/557818160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:16.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:16 smithi094 bash[55615]: debug 2023-10-09T08:40:16.528+0000 7f219e461200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T08:40:16.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:16 smithi094 bash[55615]: debug 2023-10-09T08:40:16.632+0000 7f219e461200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:40:16.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:16 smithi094 bash[55615]: debug 2023-10-09T08:40:16.736+0000 7f219e461200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:40:17.368 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:17 smithi094 bash[55615]: debug 2023-10-09T08:40:17.000+0000 7f219e461200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:40:17.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:17 smithi094 bash[55615]: debug 2023-10-09T08:40:17.364+0000 7f219e461200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:40:17.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:17 smithi094 bash[55615]: debug 2023-10-09T08:40:17.520+0000 7f219e461200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:40:18.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:17 smithi094 bash[55615]: debug 2023-10-09T08:40:17.740+0000 7f219e461200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:40:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:18 smithi094 bash[27803]: cluster 2023-10-09T08:40:17.363622+0000 mgr.y (mgr.15261) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:18 smithi094 bash[27803]: audit 2023-10-09T08:40:18.248494+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.16:0/1812766983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:18.743 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:18 smithi094 bash[55195]: ts=2023-10-09T08:40:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:40:18.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:18 smithi016 bash[27454]: cluster 2023-10-09T08:40:17.363622+0000 mgr.y (mgr.15261) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:18.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:18 smithi016 bash[27454]: audit 2023-10-09T08:40:18.248494+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.16:0/1812766983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:18.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:18 smithi016 bash[33851]: cluster 2023-10-09T08:40:17.363622+0000 mgr.y (mgr.15261) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:18.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:18 smithi016 bash[33851]: audit 2023-10-09T08:40:18.248494+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.16:0/1812766983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:19.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:18 smithi094 bash[55615]: debug 2023-10-09T08:40:18.760+0000 7f219e461200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:40:19.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:18 smithi094 bash[55615]: debug 2023-10-09T08:40:18.880+0000 7f219e461200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T08:40:19.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:19 smithi094 bash[55615]: debug 2023-10-09T08:40:19.000+0000 7f219e461200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:40:19.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:19 smithi094 bash[55615]: debug 2023-10-09T08:40:19.496+0000 7f219e461200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T08:40:19.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:19 smithi094 bash[55615]: debug 2023-10-09T08:40:19.736+0000 7f219e461200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:40:20.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:20 smithi094 bash[55615]: debug 2023-10-09T08:40:20.232+0000 7f219e461200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:40:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:20 smithi094 bash[27803]: cluster 2023-10-09T08:40:19.364508+0000 mgr.y (mgr.15261) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:20.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:20 smithi016 bash[27454]: cluster 2023-10-09T08:40:19.364508+0000 mgr.y (mgr.15261) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:20.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:20 smithi016 bash[33851]: cluster 2023-10-09T08:40:19.364508+0000 mgr.y (mgr.15261) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:21.243 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:20 smithi094 bash[55615]: debug 2023-10-09T08:40:20.800+0000 7f219e461200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:40:21.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[55615]: debug 2023-10-09T08:40:21.436+0000 7f219e461200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:40:21.743 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[55615]: debug 2023-10-09T08:40:21.568+0000 7f219e461200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:40:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[27803]: audit 2023-10-09T08:40:20.611090+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[27803]: audit 2023-10-09T08:40:20.619585+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[27803]: audit 2023-10-09T08:40:20.707357+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.16:0/3363855063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:21.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[27803]: audit 2023-10-09T08:40:21.417352+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:21.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[27803]: audit 2023-10-09T08:40:21.426023+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[27454]: audit 2023-10-09T08:40:20.611090+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[27454]: audit 2023-10-09T08:40:20.619585+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[27454]: audit 2023-10-09T08:40:20.707357+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.16:0/3363855063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[27454]: audit 2023-10-09T08:40:21.417352+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[27454]: audit 2023-10-09T08:40:21.426023+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[33851]: audit 2023-10-09T08:40:20.611090+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[33851]: audit 2023-10-09T08:40:20.619585+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[33851]: audit 2023-10-09T08:40:20.707357+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.16:0/3363855063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:22.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[33851]: audit 2023-10-09T08:40:21.417352+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:21 smithi016 bash[33851]: audit 2023-10-09T08:40:21.426023+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:22.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:21 smithi094 bash[55615]: debug 2023-10-09T08:40:21.964+0000 7f219e461200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:40:22.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:22 smithi094 bash[55615]: debug 2023-10-09T08:40:22.092+0000 7f219e461200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:40:22.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:22 smithi094 bash[55615]: debug 2023-10-09T08:40:22.248+0000 7f219e461200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:40:22.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:22 smithi094 bash[55615]: debug 2023-10-09T08:40:22.380+0000 7f219e461200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:40:22.874 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:22 smithi094 bash[27803]: cluster 2023-10-09T08:40:21.365150+0000 mgr.y (mgr.15261) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:22.874 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:22 smithi094 bash[55615]: debug 2023-10-09T08:40:22.524+0000 7f219e461200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:40:22.874 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:22 smithi094 bash[55615]: debug 2023-10-09T08:40:22.736+0000 7f219e461200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:40:22.875 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:22 smithi094 bash[55615]: debug 2023-10-09T08:40:22.868+0000 7f219e461200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:40:23.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:22 smithi016 bash[27454]: cluster 2023-10-09T08:40:21.365150+0000 mgr.y (mgr.15261) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:23.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:22 smithi016 bash[33851]: cluster 2023-10-09T08:40:21.365150+0000 mgr.y (mgr.15261) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:23.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:23 smithi094 bash[27803]: audit 2023-10-09T08:40:23.164684+0000 mon.b (mon.1) 38 : audit [DBG] from='client.? 172.21.15.16:0/4193926496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:24.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:23 smithi016 bash[27454]: audit 2023-10-09T08:40:23.164684+0000 mon.b (mon.1) 38 : audit [DBG] from='client.? 172.21.15.16:0/4193926496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:24.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:23 smithi016 bash[33851]: audit 2023-10-09T08:40:23.164684+0000 mon.b (mon.1) 38 : audit [DBG] from='client.? 172.21.15.16:0/4193926496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:24.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[55615]: debug 2023-10-09T08:40:24.132+0000 7f219e461200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:40:24.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[55615]: [09/Oct/2023:08:40:24] ENGINE Bus STARTING 2023-10-09T08:40:24.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[55615]: CherryPy Checker: 2023-10-09T08:40:24.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[55615]: The Application mounted at '' has an empty config. 2023-10-09T08:40:24.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[55615]: [09/Oct/2023:08:40:24] ENGINE Serving on http://:::9283 2023-10-09T08:40:24.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[55615]: [09/Oct/2023:08:40:24] ENGINE Bus STARTED 2023-10-09T08:40:24.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[27803]: cluster 2023-10-09T08:40:23.365780+0000 mgr.y (mgr.15261) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:24.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[27803]: cluster 2023-10-09T08:40:24.154780+0000 mon.a (mon.0) 898 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:40:24.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[27803]: cluster 2023-10-09T08:40:24.155007+0000 mon.a (mon.0) 899 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:40:24.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[27803]: audit 2023-10-09T08:40:24.162643+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:40:24.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[27803]: audit 2023-10-09T08:40:24.164046+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:40:24.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[27803]: audit 2023-10-09T08:40:24.168785+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:40:24.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:24 smithi094 bash[27803]: audit 2023-10-09T08:40:24.169431+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[27454]: cluster 2023-10-09T08:40:23.365780+0000 mgr.y (mgr.15261) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[27454]: cluster 2023-10-09T08:40:24.154780+0000 mon.a (mon.0) 898 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[27454]: cluster 2023-10-09T08:40:24.155007+0000 mon.a (mon.0) 899 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[27454]: audit 2023-10-09T08:40:24.162643+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[27454]: audit 2023-10-09T08:40:24.164046+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[27454]: audit 2023-10-09T08:40:24.168785+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[27454]: audit 2023-10-09T08:40:24.169431+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:40:25.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[33851]: cluster 2023-10-09T08:40:23.365780+0000 mgr.y (mgr.15261) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[33851]: cluster 2023-10-09T08:40:24.154780+0000 mon.a (mon.0) 898 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:40:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[33851]: cluster 2023-10-09T08:40:24.155007+0000 mon.a (mon.0) 899 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:40:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[33851]: audit 2023-10-09T08:40:24.162643+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:40:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[33851]: audit 2023-10-09T08:40:24.164046+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:40:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[33851]: audit 2023-10-09T08:40:24.168785+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:40:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:24 smithi016 bash[33851]: audit 2023-10-09T08:40:24.169431+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.? 172.21.15.94:0/2319780550' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:40:25.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:25 smithi094 bash[27803]: cluster 2023-10-09T08:40:24.643893+0000 mon.a (mon.0) 900 : cluster [DBG] mgrmap e31: y(active, since 41s), standbys: x 2023-10-09T08:40:25.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:25 smithi094 bash[27803]: audit 2023-10-09T08:40:25.618214+0000 mon.c (mon.2) 225 : audit [DBG] from='client.? 172.21.15.16:0/215845644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:26.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:40:25] "GET /metrics HTTP/1.1" 200 33709 "" "Prometheus/2.33.4" 2023-10-09T08:40:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:25 smithi016 bash[27454]: cluster 2023-10-09T08:40:24.643893+0000 mon.a (mon.0) 900 : cluster [DBG] mgrmap e31: y(active, since 41s), standbys: x 2023-10-09T08:40:26.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:25 smithi016 bash[27454]: audit 2023-10-09T08:40:25.618214+0000 mon.c (mon.2) 225 : audit [DBG] from='client.? 172.21.15.16:0/215845644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:26.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:25 smithi016 bash[33851]: cluster 2023-10-09T08:40:24.643893+0000 mon.a (mon.0) 900 : cluster [DBG] mgrmap e31: y(active, since 41s), standbys: x 2023-10-09T08:40:26.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:25 smithi016 bash[33851]: audit 2023-10-09T08:40:25.618214+0000 mon.c (mon.2) 225 : audit [DBG] from='client.? 172.21.15.16:0/215845644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:26.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:26 smithi094 bash[27803]: cluster 2023-10-09T08:40:25.366858+0000 mgr.y (mgr.15261) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:27.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:26 smithi016 bash[27454]: cluster 2023-10-09T08:40:25.366858+0000 mgr.y (mgr.15261) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:27.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:26 smithi016 bash[33851]: cluster 2023-10-09T08:40:25.366858+0000 mgr.y (mgr.15261) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:28.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:28 smithi094 bash[55195]: ts=2023-10-09T08:40:28.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:40:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:28 smithi094 bash[27803]: cluster 2023-10-09T08:40:27.367516+0000 mgr.y (mgr.15261) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:28 smithi094 bash[27803]: audit 2023-10-09T08:40:28.067625+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.16:0/941970830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:28.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:28 smithi094 bash[27803]: audit 2023-10-09T08:40:28.487741+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:29.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:28 smithi016 bash[27454]: cluster 2023-10-09T08:40:27.367516+0000 mgr.y (mgr.15261) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:29.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:28 smithi016 bash[27454]: audit 2023-10-09T08:40:28.067625+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.16:0/941970830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:29.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:28 smithi016 bash[27454]: audit 2023-10-09T08:40:28.487741+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:28 smithi016 bash[33851]: cluster 2023-10-09T08:40:27.367516+0000 mgr.y (mgr.15261) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:28 smithi016 bash[33851]: audit 2023-10-09T08:40:28.067625+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.16:0/941970830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:29.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:28 smithi016 bash[33851]: audit 2023-10-09T08:40:28.487741+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:30.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:30 smithi094 bash[27803]: cluster 2023-10-09T08:40:29.368584+0000 mgr.y (mgr.15261) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:30.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:30 smithi094 bash[27803]: audit 2023-10-09T08:40:30.528246+0000 mon.c (mon.2) 226 : audit [DBG] from='client.? 172.21.15.16:0/4149098228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:31.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:30 smithi016 bash[27454]: cluster 2023-10-09T08:40:29.368584+0000 mgr.y (mgr.15261) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:31.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:30 smithi016 bash[27454]: audit 2023-10-09T08:40:30.528246+0000 mon.c (mon.2) 226 : audit [DBG] from='client.? 172.21.15.16:0/4149098228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:31.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:30 smithi016 bash[33851]: cluster 2023-10-09T08:40:29.368584+0000 mgr.y (mgr.15261) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:31.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:30 smithi016 bash[33851]: audit 2023-10-09T08:40:30.528246+0000 mon.c (mon.2) 226 : audit [DBG] from='client.? 172.21.15.16:0/4149098228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:30.928470+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:30.935917+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:30.939489+0000 mon.a (mon.0) 905 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:30.940920+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:30.949016+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.034841+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.040878+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: cephadm 2023-10-09T08:40:31.042350+0000 mgr.y (mgr.15261) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T08:40:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.049800+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.051564+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T08:40:32.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.056777+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T08:40:32.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.058090+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T08:40:32.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.063810+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T08:40:32.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.065918+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:40:32.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:31 smithi094 bash[27803]: audit 2023-10-09T08:40:31.067119+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T08:40:32.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:30.928470+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:30.935917+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:30.939489+0000 mon.a (mon.0) 905 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:30.940920+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:30.949016+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.034841+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.040878+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: cephadm 2023-10-09T08:40:31.042350+0000 mgr.y (mgr.15261) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.049800+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.051564+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.056777+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.058090+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T08:40:32.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.063810+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.065918+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[33851]: audit 2023-10-09T08:40:31.067119+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:30.928470+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:30.935917+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:30.939489+0000 mon.a (mon.0) 905 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:30.940920+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:30.949016+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.034841+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:32.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.040878+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: cephadm 2023-10-09T08:40:31.042350+0000 mgr.y (mgr.15261) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.049800+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.051564+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.056777+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.058090+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.063810+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.065918+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:40:32.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:31 smithi016 bash[27454]: audit 2023-10-09T08:40:31.067119+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T08:40:33.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:32 smithi016 bash[33851]: cephadm 2023-10-09T08:40:31.068083+0000 mgr.y (mgr.15261) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T08:40:33.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:32 smithi016 bash[33851]: cluster 2023-10-09T08:40:31.369123+0000 mgr.y (mgr.15261) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:32 smithi016 bash[27454]: cephadm 2023-10-09T08:40:31.068083+0000 mgr.y (mgr.15261) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T08:40:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:32 smithi016 bash[27454]: cluster 2023-10-09T08:40:31.369123+0000 mgr.y (mgr.15261) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:32 smithi094 bash[27803]: cephadm 2023-10-09T08:40:31.068083+0000 mgr.y (mgr.15261) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T08:40:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:32 smithi094 bash[27803]: cluster 2023-10-09T08:40:31.369123+0000 mgr.y (mgr.15261) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:34.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:33 smithi094 bash[27803]: audit 2023-10-09T08:40:32.997647+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.16:0/754117513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:33 smithi094 bash[27803]: audit 2023-10-09T08:40:33.598680+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:33 smithi094 bash[27803]: audit 2023-10-09T08:40:33.600337+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:40:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:33 smithi094 bash[27803]: audit 2023-10-09T08:40:33.601500+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:40:34.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:33 smithi094 bash[27803]: audit 2023-10-09T08:40:33.602745+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:34.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[27454]: audit 2023-10-09T08:40:32.997647+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.16:0/754117513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:34.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[27454]: audit 2023-10-09T08:40:33.598680+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:34.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[27454]: audit 2023-10-09T08:40:33.600337+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:40:34.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[27454]: audit 2023-10-09T08:40:33.601500+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:40:34.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[27454]: audit 2023-10-09T08:40:33.602745+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:34.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[33851]: audit 2023-10-09T08:40:32.997647+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.16:0/754117513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:34.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[33851]: audit 2023-10-09T08:40:33.598680+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' 2023-10-09T08:40:34.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[33851]: audit 2023-10-09T08:40:33.600337+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:40:34.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[33851]: audit 2023-10-09T08:40:33.601500+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:40:34.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:33 smithi016 bash[33851]: audit 2023-10-09T08:40:33.602745+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:40:35.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:34 smithi094 bash[27803]: cluster 2023-10-09T08:40:33.369802+0000 mgr.y (mgr.15261) 45 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:34 smithi094 bash[27803]: cephadm 2023-10-09T08:40:33.589082+0000 mgr.y (mgr.15261) 46 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T08:40:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:34 smithi094 bash[27803]: cephadm 2023-10-09T08:40:33.604198+0000 mgr.y (mgr.15261) 47 : cephadm [INF] Deploying daemon mon.a on smithi016 2023-10-09T08:40:35.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:34 smithi016 bash[33851]: cluster 2023-10-09T08:40:33.369802+0000 mgr.y (mgr.15261) 45 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:35.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:34 smithi016 bash[33851]: cephadm 2023-10-09T08:40:33.589082+0000 mgr.y (mgr.15261) 46 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T08:40:35.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:34 smithi016 bash[33851]: cephadm 2023-10-09T08:40:33.604198+0000 mgr.y (mgr.15261) 47 : cephadm [INF] Deploying daemon mon.a on smithi016 2023-10-09T08:40:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:34 smithi016 bash[27454]: cluster 2023-10-09T08:40:33.369802+0000 mgr.y (mgr.15261) 45 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:34 smithi016 bash[27454]: cephadm 2023-10-09T08:40:33.589082+0000 mgr.y (mgr.15261) 46 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T08:40:35.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:34 smithi016 bash[27454]: cephadm 2023-10-09T08:40:33.604198+0000 mgr.y (mgr.15261) 47 : cephadm [INF] Deploying daemon mon.a on smithi016 2023-10-09T08:40:36.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:40:35] "GET /metrics HTTP/1.1" 200 33706 "" "Prometheus/2.33.4" 2023-10-09T08:40:36.423 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[71225]: [09/Oct/2023:08:40:36] ENGINE Bus STOPPING 2023-10-09T08:40:36.423 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[71225]: [09/Oct/2023:08:40:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:40:36.423 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[71225]: [09/Oct/2023:08:40:36] ENGINE Bus STOPPED 2023-10-09T08:40:36.424 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[71225]: [09/Oct/2023:08:40:36] ENGINE Bus STARTING 2023-10-09T08:40:36.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:36 smithi016 systemd[1]: Stopping Ceph mon.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:40:36.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[27454]: cluster 2023-10-09T08:40:35.370418+0000 mgr.y (mgr.15261) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:36.425 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[27454]: audit 2023-10-09T08:40:35.448144+0000 mon.c (mon.2) 227 : audit [DBG] from='client.? 172.21.15.16:0/48701535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:36.425 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[27454]: debug 2023-10-09T08:40:36.303+0000 7f4b5f071700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T08:40:36.425 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[27454]: debug 2023-10-09T08:40:36.303+0000 7f4b5f071700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T08:40:36.425 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[33851]: cluster 2023-10-09T08:40:35.370418+0000 mgr.y (mgr.15261) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:36.426 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[33851]: audit 2023-10-09T08:40:35.448144+0000 mon.c (mon.2) 227 : audit [DBG] from='client.? 172.21.15.16:0/48701535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:36.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:36 smithi094 bash[27803]: cluster 2023-10-09T08:40:35.370418+0000 mgr.y (mgr.15261) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:36 smithi094 bash[27803]: audit 2023-10-09T08:40:35.448144+0000 mon.c (mon.2) 227 : audit [DBG] from='client.? 172.21.15.16:0/48701535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:36.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[71225]: [09/Oct/2023:08:40:36] ENGINE Serving on http://:::9283 2023-10-09T08:40:36.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:36 smithi016 bash[71225]: [09/Oct/2023:08:40:36] ENGINE Bus STARTED 2023-10-09T08:40:37.481 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:37 smithi016 bash[75044]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T08:40:37.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:37 smithi016 bash[75120]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T08:40:37.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:37 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T08:40:37.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:37 smithi016 systemd[1]: Stopped Ceph mon.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:40:37.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:37 smithi016 systemd[1]: Started Ceph mon.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:40:38.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.611+0000 7fa055237c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T08:40:38.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.611+0000 7fa055237c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T08:40:38.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.611+0000 7fa055237c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T08:40:38.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 0 load: jerasure load: lrc 2023-10-09T08:40:38.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T08:40:38.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Git sha 0 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: DB SUMMARY 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: DB Session ID: TM5CNWW4VYV4H9W0IETS 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 966 Bytes 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2023-10-09T08:40:38.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 16322643 ; 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.env: 0x563eb8e23720 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.info_log: 0x563ebc263400 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T08:40:38.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T08:40:38.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T08:40:38.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T08:40:38.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.db_log_dir: 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.wal_dir: 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T08:40:38.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.write_buffer_manager: 0x563ebb4f05a0 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T08:40:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.row_cache: None 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.wal_filter: None 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T08:40:38.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T08:40:38.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T08:40:38.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Compression algorithms supported: 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kZSTD supported: 0 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T08:40:38.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.merge_operator: 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563ebc263520) 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: cache_index_and_filter_blocks: 1 2023-10-09T08:40:38.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: pin_top_level_index_and_filter: 1 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: index_type: 0 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: data_block_index_type: 0 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: index_shortening: 1 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: checksum: 4 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: no_block_cache: 0 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_cache: 0x563ebb5731f0 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_cache_name: BinnedLRUCache 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_cache_options: 2023-10-09T08:40:38.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: capacity : 536870912 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: num_shard_bits : 4 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: strict_capacity_limit : 0 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: high_pri_pool_ratio: 0.000 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_cache_compressed: (nil) 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: persistent_cache: (nil) 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_size: 4096 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_size_deviation: 10 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_restart_interval: 16 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: index_block_restart_interval: 1 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: metadata_block_size: 4096 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: partition_filters: 0 2023-10-09T08:40:38.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: use_delta_encoding: 1 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: filter_policy: bloomfilter 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: whole_key_filtering: 1 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: verify_compression: 0 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: read_amp_bytes_per_bit: 0 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: format_version: 5 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: enable_index_compression: 1 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: block_align: 0 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: max_auto_readahead_size: 262144 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: prepopulate_block_cache: 0 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: initial_auto_readahead_size: 8192 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: num_file_reads_for_auto_readahead: 2 2023-10-09T08:40:38.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.num_levels: 7 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T08:40:38.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:40:38.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T08:40:38.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T08:40:38.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T08:40:38.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T08:40:38.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T08:40:38.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T08:40:38.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T08:40:38.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T08:40:38.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T08:40:38.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T08:40:38.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T08:40:38.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T08:40:38.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T08:40:38.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T08:40:38.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T08:40:38.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T08:40:38.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T08:40:38.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.615+0000 7fa055237c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.619+0000 7fa04aa55700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.619+0000 7fa055237c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.619+0000 7fa055237c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11319, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.619+0000 7fa055237c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.619+0000 7fa055237c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f64c6dab-7467-4555-b2fa-e204acb166a9 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.619+0000 7fa055237c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696840838620777, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.619+0000 7fa055237c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.667+0000 7fa055237c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696840838668746, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 15813191, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11320, "largest_seqno": 15247, "table_properties": {"data_size": 15790546, "index_size": 18103, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3525, "raw_key_size": 26969, "raw_average_key_size": 19, "raw_value_size": 15752864, "raw_average_value_size": 11219, "num_data_blocks": 787, "num_entries": 1404, "num_filter_entries": 1404, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696840838, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f64c6dab-7467-4555-b2fa-e204acb166a9", "db_session_id": "TM5CNWW4VYV4H9W0IETS", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.667+0000 7fa055237c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696840838669026, "job": 1, "event": "recovery_finished"} 2023-10-09T08:40:38.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.667+0000 7fa055237c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.667+0000 7fa055237c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.671+0000 7fa055237c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.671+0000 7fa055237c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563ebb614000 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.671+0000 7fa055237c80 4 rocksdb: DB pointer 0x563ebb602000 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.671+0000 7fa043a47700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.671+0000 7fa043a47700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: ** DB Stats ** 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:40:38.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: ** Compaction Stats [default] ** 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: L0 1/0 15.08 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 700.6 0.02 0.00 1 0.022 0 0 0.0 0.0 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: L6 1/0 22.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Sum 2/0 37.25 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 700.6 0.02 0.00 1 0.022 0 0 0.0 0.0 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 700.6 0.02 0.00 1 0.022 0 0 0.0 0.0 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: ** Compaction Stats [default] ** 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T08:40:38.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 700.6 0.02 0.00 1 0.022 0 0 0.0 0.0 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Flush(GB): cumulative 0.015, interval 0.015 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Cumulative compaction: 0.01 GB write, 273.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Interval compaction: 0.01 GB write, 273.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:40:38.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Block cache BinnedLRUCache@0x563ebb5731f0#7 capacity: 512.00 MB usage: 24.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 3.3e-05 secs_since: 0 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] at bind addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 1 mon.a@-1(???) e3 preinit fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 0 mon.a@-1(???).mds e7 new map 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 0 mon.a@-1(???).mds e7 print_map 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: e7 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: legacy client fscid: 1 2023-10-09T08:40:38.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Filesystem 'foo' (1) 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: fs_name foo 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: epoch 5 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: created 2023-10-09T08:34:27.739170+0000 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: modified 2023-10-09T08:34:35.063882+0000 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: tableserver 0 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: root 0 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: session_timeout 60 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: session_autoclose 300 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: max_file_size 1099511627776 2023-10-09T08:40:38.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: required_client_features {} 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: last_failure 0 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: last_failure_osd_epoch 0 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: max_mds 1 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: in 0 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: up {0=14634} 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: failed 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: damaged 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: stopped 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: data_pools [4] 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: metadata_pool 3 2023-10-09T08:40:38.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: inline_data disabled 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: balancer 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: bal_rank_mask -1 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: standby_count_wanted 1 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: [mds.foo.smithi016.ccdktz{0:14634} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: Standby daemons: 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: [mds.foo.smithi094.zfssci{-1:24523} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.94:6832/1000218818,v1:172.21.15.94:6833/1000218818] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 0 mon.a@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 0 mon.a@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:40:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 0 mon.a@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.675+0000 7fa055237c80 0 mon.a@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T08:40:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T08:40:38.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T08:40:38.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:40:38 smithi016 bash[75179]: debug 2023-10-09T08:40:38.679+0000 7fa055237c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T08:40:38.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:38 smithi094 bash[55195]: ts=2023-10-09T08:40:38.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:40:40.400 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T08:40:40.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[55615]: ignoring --setuser ceph since I am not root 2023-10-09T08:40:40.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[55615]: ignoring --setgroup ceph since I am not root 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[55615]: debug 2023-10-09T08:40:40.232+0000 7fc3df1f6700 1 -- 172.21.15.94:0/2759548686 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555f9a09d1e0 con 0x555f9ae1c400 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[55615]: debug 2023-10-09T08:40:40.388+0000 7fc3e8f26200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: audit 2023-10-09T08:40:36.315910+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: audit 2023-10-09T08:40:36.316999+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: audit 2023-10-09T08:40:36.317556+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:37.370864+0000 mgr.y (mgr.15261) 49 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: audit 2023-10-09T08:40:37.896602+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.16:0/3008769228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:39.371875+0000 mgr.y (mgr.15261) 50 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.097764+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.120475+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.135209+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.136125+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.136180+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.137622+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 56s), standbys: x 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.165692+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 56s), standbys: x 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: cluster 2023-10-09T08:40:40.165746+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T08:40:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:40 smithi094 bash[27803]: audit 2023-10-09T08:40:40.165895+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15261 ' entity='' 2023-10-09T08:40:40.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[71225]: ignoring --setuser ceph since I am not root 2023-10-09T08:40:40.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[71225]: ignoring --setgroup ceph since I am not root 2023-10-09T08:40:40.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[71225]: debug 2023-10-09T08:40:40.231+0000 7f56bf751700 1 -- 172.21.15.16:0/719886971 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55615e5d31e0 con 0x55615f352800 2023-10-09T08:40:40.502 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[71225]: debug 2023-10-09T08:40:40.395+0000 7f56c9481200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:40:40.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: audit 2023-10-09T08:40:36.315910+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:40:40.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: audit 2023-10-09T08:40:36.316999+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:40:40.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: audit 2023-10-09T08:40:36.317556+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.15261 172.21.15.16:0/1212977177' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:40:40.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:37.370864+0000 mgr.y (mgr.15261) 49 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:40.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: audit 2023-10-09T08:40:37.896602+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.16:0/3008769228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:40.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:39.371875+0000 mgr.y (mgr.15261) 50 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:40.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.097764+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.120475+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.135209+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.136125+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.136180+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.137622+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 56s), standbys: x 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.165692+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 56s), standbys: x 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: cluster 2023-10-09T08:40:40.165746+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T08:40:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:40 smithi016 bash[33851]: audit 2023-10-09T08:40:40.165895+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15261 ' entity='' 2023-10-09T08:40:41.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T08:40:41.486 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:41 smithi016 bash[71225]: debug 2023-10-09T08:40:41.367+0000 7f56c9481200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T08:40:41.487 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:41 smithi016 bash[33851]: audit 2023-10-09T08:40:40.375941+0000 mon.c (mon.2) 229 : audit [DBG] from='client.? 172.21.15.16:0/3669945340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:41.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:41 smithi094 bash[27803]: audit 2023-10-09T08:40:40.375941+0000 mon.c (mon.2) 229 : audit [DBG] from='client.? 172.21.15.16:0/3669945340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:40:41.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:41 smithi094 bash[55615]: debug 2023-10-09T08:40:41.304+0000 7fc3e8f26200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T08:40:41.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:41 smithi094 bash[55615]: debug 2023-10-09T08:40:41.404+0000 7fc3e8f26200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:40:41.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:41 smithi016 bash[71225]: debug 2023-10-09T08:40:41.479+0000 7f56c9481200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:40:41.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:41 smithi016 bash[71225]: debug 2023-10-09T08:40:41.595+0000 7f56c9481200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:40:41.783 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:41 smithi094 bash[55615]: debug 2023-10-09T08:40:41.508+0000 7fc3e8f26200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:40:42.147 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:41 smithi094 bash[55615]: debug 2023-10-09T08:40:41.780+0000 7fc3e8f26200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:40:42.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:41 smithi016 bash[71225]: debug 2023-10-09T08:40:41.875+0000 7f56c9481200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:40:42.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:42 smithi094 bash[55615]: debug 2023-10-09T08:40:42.144+0000 7fc3e8f26200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:40:42.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:42 smithi094 bash[55615]: debug 2023-10-09T08:40:42.304+0000 7fc3e8f26200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:40:42.559 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:42 smithi016 bash[71225]: debug 2023-10-09T08:40:42.263+0000 7f56c9481200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:40:42.559 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:42 smithi016 bash[71225]: debug 2023-10-09T08:40:42.443+0000 7f56c9481200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:40:42.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:42 smithi094 bash[55615]: debug 2023-10-09T08:40:42.516+0000 7fc3e8f26200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:40:43.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:42 smithi016 bash[71225]: debug 2023-10-09T08:40:42.691+0000 7f56c9481200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:40:43.992 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:43 smithi094 bash[55615]: debug 2023-10-09T08:40:43.524+0000 7fc3e8f26200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:40:43.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:43 smithi094 bash[55615]: debug 2023-10-09T08:40:43.636+0000 7fc3e8f26200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T08:40:43.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:43 smithi094 bash[55615]: debug 2023-10-09T08:40:43.760+0000 7fc3e8f26200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:40:44.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:43 smithi016 bash[71225]: debug 2023-10-09T08:40:43.835+0000 7f56c9481200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:40:44.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:43 smithi016 bash[71225]: debug 2023-10-09T08:40:43.959+0000 7f56c9481200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T08:40:44.252 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:44 smithi016 bash[71225]: debug 2023-10-09T08:40:44.083+0000 7f56c9481200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:40:44.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:44 smithi094 bash[55615]: debug 2023-10-09T08:40:44.248+0000 7fc3e8f26200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T08:40:44.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:44 smithi094 bash[55615]: debug 2023-10-09T08:40:44.475+0000 7fc3e8f26200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:40:45.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:44 smithi016 bash[71225]: debug 2023-10-09T08:40:44.595+0000 7f56c9481200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T08:40:45.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:44 smithi016 bash[71225]: debug 2023-10-09T08:40:44.831+0000 7f56c9481200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:40:45.242 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:44 smithi094 bash[55615]: debug 2023-10-09T08:40:44.939+0000 7fc3e8f26200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:40:45.742 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:45 smithi094 bash[55615]: debug 2023-10-09T08:40:45.443+0000 7fc3e8f26200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:40:45.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:45 smithi016 bash[71225]: debug 2023-10-09T08:40:45.303+0000 7f56c9481200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:40:46.260 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:45 smithi016 bash[71225]: debug 2023-10-09T08:40:45.815+0000 7f56c9481200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:40:46.464 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:45 smithi094 bash[55615]: debug 2023-10-09T08:40:45.995+0000 7fc3e8f26200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:40:46.464 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:46 smithi094 bash[55615]: debug 2023-10-09T08:40:46.107+0000 7fc3e8f26200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:40:46.464 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:46 smithi094 bash[55615]: debug 2023-10-09T08:40:46.459+0000 7fc3e8f26200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:40:46.726 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:46 smithi094 bash[55615]: debug 2023-10-09T08:40:46.579+0000 7fc3e8f26200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:40:46.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:46 smithi016 bash[71225]: debug 2023-10-09T08:40:46.395+0000 7f56c9481200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:40:46.752 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:46 smithi016 bash[71225]: debug 2023-10-09T08:40:46.511+0000 7f56c9481200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:40:46.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:46 smithi094 bash[55615]: debug 2023-10-09T08:40:46.719+0000 7fc3e8f26200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:40:46.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:46 smithi094 bash[55615]: debug 2023-10-09T08:40:46.839+0000 7fc3e8f26200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:40:46.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:46 smithi094 bash[55615]: debug 2023-10-09T08:40:46.963+0000 7fc3e8f26200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:40:47.152 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:46 smithi016 bash[71225]: debug 2023-10-09T08:40:46.871+0000 7f56c9481200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:40:47.152 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:46 smithi016 bash[71225]: debug 2023-10-09T08:40:46.995+0000 7f56c9481200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:40:47.408 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:47 smithi016 bash[71225]: debug 2023-10-09T08:40:47.143+0000 7f56c9481200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:40:47.408 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:47 smithi016 bash[71225]: debug 2023-10-09T08:40:47.267+0000 7f56c9481200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:40:47.492 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:47 smithi094 bash[55615]: debug 2023-10-09T08:40:47.159+0000 7fc3e8f26200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:40:47.493 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:47 smithi094 bash[55615]: debug 2023-10-09T08:40:47.291+0000 7fc3e8f26200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:40:47.743 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:47 smithi016 bash[71225]: debug 2023-10-09T08:40:47.403+0000 7f56c9481200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:40:47.743 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:47 smithi016 bash[71225]: debug 2023-10-09T08:40:47.599+0000 7f56c9481200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:40:48.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[55615]: debug 2023-10-09T08:40:48.523+0000 7fc3e8f26200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:40:48.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[55615]: [09/Oct/2023:08:40:48] ENGINE Bus STARTING 2023-10-09T08:40:48.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[55615]: CherryPy Checker: 2023-10-09T08:40:48.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[55615]: The Application mounted at '' has an empty config. 2023-10-09T08:40:48.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[55615]: [09/Oct/2023:08:40:48] ENGINE Serving on http://:::9283 2023-10-09T08:40:48.993 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[55615]: [09/Oct/2023:08:40:48] ENGINE Bus STARTED 2023-10-09T08:40:48.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[27803]: cluster 2023-10-09T08:40:48.547019+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:40:48.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[27803]: cluster 2023-10-09T08:40:48.547163+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:40:48.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[27803]: audit 2023-10-09T08:40:48.554077+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:40:48.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[27803]: audit 2023-10-09T08:40:48.555548+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:40:48.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[27803]: audit 2023-10-09T08:40:48.561143+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:40:48.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:48 smithi094 bash[27803]: audit 2023-10-09T08:40:48.561910+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:40:49.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:47 smithi016 bash[71225]: debug 2023-10-09T08:40:47.735+0000 7f56c9481200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:40:49.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:48 smithi016 bash[33851]: cluster 2023-10-09T08:40:48.547019+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T08:40:49.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:48 smithi016 bash[33851]: cluster 2023-10-09T08:40:48.547163+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:40:49.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:48 smithi016 bash[33851]: audit 2023-10-09T08:40:48.554077+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:40:49.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:48 smithi016 bash[33851]: audit 2023-10-09T08:40:48.555548+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:40:49.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:48 smithi016 bash[33851]: audit 2023-10-09T08:40:48.561143+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:40:49.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:48 smithi016 bash[33851]: audit 2023-10-09T08:40:48.561910+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.? 172.21.15.94:0/3621198675' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: debug 2023-10-09T08:40:49.063+0000 7fa9b103a700 -1 mon.c@2(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.035+0000 7f56c9481200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: [09/Oct/2023:08:40:49] ENGINE Bus STARTING 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562685b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562685b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562685b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562685b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.320 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562685b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.321 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562d869700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.321 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562d869700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.321 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562d869700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.321 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562d869700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.321 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: debug 2023-10-09T08:40:49.267+0000 7f562d869700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T08:40:49.349 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: debug 2023-10-09T08:40:49.063+0000 7fb4ee113700 -1 mon.b@1(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T08:40:49.618 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: CherryPy Checker: 2023-10-09T08:40:49.618 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: The Application mounted at '' has an empty config. 2023-10-09T08:40:49.618 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: [09/Oct/2023:08:40:49] ENGINE Serving on http://:::9283 2023-10-09T08:40:49.619 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[71225]: [09/Oct/2023:08:40:49] ENGINE Bus STARTED 2023-10-09T08:40:49.619 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:48.606065+0000 mon.a (mon.0) 12 : cluster [DBG] mgrmap e33: y(active, since 65s), standbys: x 2023-10-09T08:40:49.619 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.059460+0000 mon.a (mon.0) 13 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:40:49.619 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.060329+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.069806+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.070154+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e34: y(active, starting, since 0.0100226s), standbys: x 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.072164+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.076375+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.080277+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.080518+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.080751+0000 mon.c (mon.2) 232 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:40:49.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.082343+0000 mon.c (mon.2) 233 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:40:49.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:48.606065+0000 mon.a (mon.0) 12 : cluster [DBG] mgrmap e33: y(active, since 65s), standbys: x 2023-10-09T08:40:49.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.059460+0000 mon.a (mon.0) 13 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:40:49.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.060329+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-10-09T08:40:49.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.069806+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.070154+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e34: y(active, starting, since 0.0100226s), standbys: x 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.072164+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.076375+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.080277+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.080518+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.080751+0000 mon.c (mon.2) 232 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.082343+0000 mon.c (mon.2) 233 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.ccdktz"}]: dispatch 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.082616+0000 mon.c (mon.2) 234 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.083098+0000 mon.c (mon.2) 235 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.083297+0000 mon.c (mon.2) 236 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:40:49.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.083951+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.084222+0000 mon.c (mon.2) 237 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.084563+0000 mon.c (mon.2) 238 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.085174+0000 mon.c (mon.2) 239 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.085435+0000 mon.c (mon.2) 240 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.085668+0000 mon.c (mon.2) 241 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.085953+0000 mon.c (mon.2) 242 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.086320+0000 mon.c (mon.2) 243 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.086675+0000 mon.c (mon.2) 244 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.087012+0000 mon.c (mon.2) 245 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.087338+0000 mon.c (mon.2) 246 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:40:49.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.087637+0000 mon.c (mon.2) 247 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.114693+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.157119+0000 mon.c (mon.2) 248 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.159790+0000 mon.c (mon.2) 249 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.189400+0000 mon.c (mon.2) 250 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.218950+0000 mon.c (mon.2) 251 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.219477+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.245311+0000 mon.c (mon.2) 252 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:40:49.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:49 smithi094 bash[27803]: audit 2023-10-09T08:40:49.245985+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:40:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.082616+0000 mon.c (mon.2) 234 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi094.zfssci"}]: dispatch 2023-10-09T08:40:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.083098+0000 mon.c (mon.2) 235 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:40:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.083297+0000 mon.c (mon.2) 236 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:40:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.083951+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.084222+0000 mon.c (mon.2) 237 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:40:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.084563+0000 mon.c (mon.2) 238 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:40:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.085174+0000 mon.c (mon.2) 239 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.085435+0000 mon.c (mon.2) 240 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.085668+0000 mon.c (mon.2) 241 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.085953+0000 mon.c (mon.2) 242 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.086320+0000 mon.c (mon.2) 243 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.086675+0000 mon.c (mon.2) 244 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.087012+0000 mon.c (mon.2) 245 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.087338+0000 mon.c (mon.2) 246 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.087637+0000 mon.c (mon.2) 247 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.114693+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.157119+0000 mon.c (mon.2) 248 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.159790+0000 mon.c (mon.2) 249 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:40:50.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.189400+0000 mon.c (mon.2) 250 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:40:50.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.218950+0000 mon.c (mon.2) 251 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:40:50.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.219477+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:40:50.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.245311+0000 mon.c (mon.2) 252 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:40:50.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:49 smithi016 bash[33851]: audit 2023-10-09T08:40:49.245985+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:40:50.104 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (2m) - 8m 14.1M - 0.23.0 ba2b418f427c b9989d9bd43e 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi094 *:3000 running (89s) - 7m 40.1M - 8.3.5 dad864ee21e9 f1a12efb3a56 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.ccdktz smithi016 running (6m) - 6m 15.1M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 17a3e2c7a98b 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi094.zfssci smithi094 running (6m) - 6m 13.1M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a917ff5f77b 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi094 *:8443,9283,8765 running (35s) - 15m 78.3M - 18.2.0-598-gb4fa663f 3060897fdf97 f0fcc951b93f 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (83s) - 17m 461M - 18.2.0-598-gb4fa663f 3060897fdf97 cdf79c37f30d 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (17m) - 17m 60.9M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 6291e72b78ad 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi094 running (15m) - 15m 61.4M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5e9c5bdb4c92 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (15m) - 15m 57.7M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8dd3a38c7bb6 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (2m) - 8m 8531k - 1.3.1 1dbe0e931976 6c4a45c1d04a 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi094 *:9100 running (113s) - 8m 8523k - 1.3.1 1dbe0e931976 59bcf67d7eb5 2023-10-09T08:40:50.105 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (14m) - 14m 45.5M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c2ec86d01448 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (14m) - 14m 43.1M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 302ee697c26f 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (13m) - 13m 51.2M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c358a7abf935 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (12m) - 12m 42.4M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0f6ebb2bd802 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi094 running (11m) - 11m 44.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1a20cacc6df3 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi094 running (11m) - 11m 49.2M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 381d98a4283c 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi094 running (10m) - 10m 41.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d8307333e14e 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi094 running (9m) - 9m 52.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 04152ef43de4 2023-10-09T08:40:50.106 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi094 *:9095 running (42s) - 8m 45.5M - 2.33.4 514e6a882f6e 4aee8e4f263b 2023-10-09T08:40:50.340 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:50 smithi016 bash[71225]: debug 2023-10-09T08:40:50.079+0000 7f5681439700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T08:40:50.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T08:40:51.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: cluster 2023-10-09T08:40:49.236346+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:51.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: cephadm 2023-10-09T08:40:49.942190+0000 mgr.y (mgr.25421) 1 : cephadm [INF] [09/Oct/2023:08:40:49] ENGINE Bus STARTING 2023-10-09T08:40:51.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: cephadm 2023-10-09T08:40:50.063452+0000 mgr.y (mgr.25421) 2 : cephadm [INF] [09/Oct/2023:08:40:50] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:40:51.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: cluster 2023-10-09T08:40:50.080148+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e35: y(active, since 1.02s), standbys: x 2023-10-09T08:40:51.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: audit 2023-10-09T08:40:50.084545+0000 mgr.y (mgr.25421) 3 : audit [DBG] from='client.34113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:40:51.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: cluster 2023-10-09T08:40:50.089243+0000 mgr.y (mgr.25421) 4 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:51.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: cephadm 2023-10-09T08:40:50.165832+0000 mgr.y (mgr.25421) 5 : cephadm [INF] [09/Oct/2023:08:40:50] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:40:51.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:51 smithi016 bash[33851]: cephadm 2023-10-09T08:40:50.166180+0000 mgr.y (mgr.25421) 6 : cephadm [INF] [09/Oct/2023:08:40:50] ENGINE Bus STARTED 2023-10-09T08:40:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: cluster 2023-10-09T08:40:49.236346+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-09T08:40:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: cephadm 2023-10-09T08:40:49.942190+0000 mgr.y (mgr.25421) 1 : cephadm [INF] [09/Oct/2023:08:40:49] ENGINE Bus STARTING 2023-10-09T08:40:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: cephadm 2023-10-09T08:40:50.063452+0000 mgr.y (mgr.25421) 2 : cephadm [INF] [09/Oct/2023:08:40:50] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:40:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: cluster 2023-10-09T08:40:50.080148+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e35: y(active, since 1.02s), standbys: x 2023-10-09T08:40:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: audit 2023-10-09T08:40:50.084545+0000 mgr.y (mgr.25421) 3 : audit [DBG] from='client.34113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:40:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: cluster 2023-10-09T08:40:50.089243+0000 mgr.y (mgr.25421) 4 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:51.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: cephadm 2023-10-09T08:40:50.165832+0000 mgr.y (mgr.25421) 5 : cephadm [INF] [09/Oct/2023:08:40:50] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:40:51.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:51 smithi094 bash[27803]: cephadm 2023-10-09T08:40:50.166180+0000 mgr.y (mgr.25421) 6 : cephadm [INF] [09/Oct/2023:08:40:50] ENGINE Bus STARTED 2023-10-09T08:40:52.482 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:52 smithi016 bash[33851]: cluster 2023-10-09T08:40:51.085470+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T08:40:52.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:52 smithi016 bash[33851]: cluster 2023-10-09T08:40:51.087805+0000 mgr.y (mgr.25421) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:52.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:52 smithi094 bash[27803]: cluster 2023-10-09T08:40:51.085470+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T08:40:52.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:52 smithi094 bash[27803]: cluster 2023-10-09T08:40:51.087805+0000 mgr.y (mgr.25421) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:54.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:54 smithi094 bash[27803]: cluster 2023-10-09T08:40:53.088239+0000 mgr.y (mgr.25421) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:54.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:54 smithi016 bash[33851]: cluster 2023-10-09T08:40:53.088239+0000 mgr.y (mgr.25421) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:54.772 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:40:55.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T08:40:56.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:40:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:40:55] "GET /metrics HTTP/1.1" 200 31077 "" "Prometheus/2.33.4" 2023-10-09T08:40:56.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:55 smithi016 bash[33851]: audit 2023-10-09T08:40:54.759397+0000 mgr.y (mgr.25421) 9 : audit [DBG] from='client.25466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:40:56.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:55 smithi016 bash[33851]: cephadm 2023-10-09T08:40:54.760144+0000 mgr.y (mgr.25421) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:40:56.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:55 smithi016 bash[33851]: audit 2023-10-09T08:40:54.767632+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:56.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:55 smithi016 bash[33851]: cluster 2023-10-09T08:40:55.088716+0000 mgr.y (mgr.25421) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:56.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:55 smithi016 bash[33851]: audit 2023-10-09T08:40:55.628542+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:56.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:55 smithi016 bash[33851]: audit 2023-10-09T08:40:55.639224+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:56.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:55 smithi094 bash[27803]: audit 2023-10-09T08:40:54.759397+0000 mgr.y (mgr.25421) 9 : audit [DBG] from='client.25466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:40:56.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:55 smithi094 bash[27803]: cephadm 2023-10-09T08:40:54.760144+0000 mgr.y (mgr.25421) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:40:56.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:55 smithi094 bash[27803]: audit 2023-10-09T08:40:54.767632+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:56.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:55 smithi094 bash[27803]: cluster 2023-10-09T08:40:55.088716+0000 mgr.y (mgr.25421) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:56.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:55 smithi094 bash[27803]: audit 2023-10-09T08:40:55.628542+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:56.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:55 smithi094 bash[27803]: audit 2023-10-09T08:40:55.639224+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:56 smithi016 bash[33851]: audit 2023-10-09T08:40:55.909342+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:56 smithi016 bash[33851]: audit 2023-10-09T08:40:55.918458+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:56 smithi016 bash[33851]: audit 2023-10-09T08:40:56.438953+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:56 smithi016 bash[33851]: audit 2023-10-09T08:40:56.449839+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:56 smithi016 bash[33851]: audit 2023-10-09T08:40:56.709908+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.172 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:56 smithi016 bash[33851]: audit 2023-10-09T08:40:56.718628+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:56 smithi094 bash[27803]: audit 2023-10-09T08:40:55.909342+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:56 smithi094 bash[27803]: audit 2023-10-09T08:40:55.918458+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:56 smithi094 bash[27803]: audit 2023-10-09T08:40:56.438953+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:56 smithi094 bash[27803]: audit 2023-10-09T08:40:56.449839+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:56 smithi094 bash[27803]: audit 2023-10-09T08:40:56.709908+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:57.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:56 smithi094 bash[27803]: audit 2023-10-09T08:40:56.718628+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:40:58.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:57 smithi094 bash[27803]: cluster 2023-10-09T08:40:57.089305+0000 mgr.y (mgr.25421) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:58.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:57 smithi016 bash[33851]: cluster 2023-10-09T08:40:57.089305+0000 mgr.y (mgr.25421) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:40:58.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:40:58 smithi094 bash[55195]: ts=2023-10-09T08:40:58.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:40:59.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:40:59 smithi094 bash[27803]: cluster 2023-10-09T08:40:59.089932+0000 mgr.y (mgr.25421) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:00.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:40:59 smithi016 bash[33851]: cluster 2023-10-09T08:40:59.089932+0000 mgr.y (mgr.25421) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:02.463 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:02 smithi094 bash[27803]: cluster 2023-10-09T08:41:01.090913+0000 mgr.y (mgr.25421) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:02.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:02 smithi016 bash[33851]: cluster 2023-10-09T08:41:01.090913+0000 mgr.y (mgr.25421) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:04.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:04 smithi094 bash[27803]: cluster 2023-10-09T08:41:03.091554+0000 mgr.y (mgr.25421) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:04 smithi016 bash[33851]: cluster 2023-10-09T08:41:03.091554+0000 mgr.y (mgr.25421) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:05 smithi094 bash[27803]: audit 2023-10-09T08:41:04.190242+0000 mon.c (mon.2) 253 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:05.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:05 smithi016 bash[33851]: audit 2023-10-09T08:41:04.190242+0000 mon.c (mon.2) 253 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:06.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:41:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:41:05] "GET /metrics HTTP/1.1" 200 33706 "" "Prometheus/2.33.4" 2023-10-09T08:41:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:06 smithi094 bash[27803]: cluster 2023-10-09T08:41:05.092384+0000 mgr.y (mgr.25421) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:06.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:06 smithi094 bash[27803]: audit 2023-10-09T08:41:05.971127+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:06.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:06 smithi094 bash[27803]: audit 2023-10-09T08:41:05.981479+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:06.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:06 smithi094 bash[27803]: audit 2023-10-09T08:41:05.985119+0000 mon.c (mon.2) 254 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:41:06.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:06 smithi094 bash[27803]: audit 2023-10-09T08:41:05.986705+0000 mon.c (mon.2) 255 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:41:06.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:06 smithi016 bash[33851]: cluster 2023-10-09T08:41:05.092384+0000 mgr.y (mgr.25421) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:06.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:06 smithi016 bash[33851]: audit 2023-10-09T08:41:05.971127+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:06.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:06 smithi016 bash[33851]: audit 2023-10-09T08:41:05.981479+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:06.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:06 smithi016 bash[33851]: audit 2023-10-09T08:41:05.985119+0000 mon.c (mon.2) 254 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:41:06.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:06 smithi016 bash[33851]: audit 2023-10-09T08:41:05.986705+0000 mon.c (mon.2) 255 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: cephadm 2023-10-09T08:41:05.988785+0000 mgr.y (mgr.25421) 17 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: cephadm 2023-10-09T08:41:05.989266+0000 mgr.y (mgr.25421) 18 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: cephadm 2023-10-09T08:41:06.154257+0000 mgr.y (mgr.25421) 19 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: cephadm 2023-10-09T08:41:06.204964+0000 mgr.y (mgr.25421) 20 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.708857+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.717298+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.760327+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.771275+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.778811+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.862986+0000 mon.c (mon.2) 256 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:41:07.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.865816+0000 mon.c (mon.2) 257 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:41:07.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.867441+0000 mon.c (mon.2) 258 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:41:07.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:07 smithi094 bash[27803]: audit 2023-10-09T08:41:06.877571+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: cephadm 2023-10-09T08:41:05.988785+0000 mgr.y (mgr.25421) 17 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: cephadm 2023-10-09T08:41:05.989266+0000 mgr.y (mgr.25421) 18 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: cephadm 2023-10-09T08:41:06.154257+0000 mgr.y (mgr.25421) 19 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: cephadm 2023-10-09T08:41:06.204964+0000 mgr.y (mgr.25421) 20 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.708857+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.717298+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.760327+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.771275+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.778811+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:07.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.862986+0000 mon.c (mon.2) 256 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:41:07.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.865816+0000 mon.c (mon.2) 257 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:41:07.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.867441+0000 mon.c (mon.2) 258 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:41:07.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:07 smithi016 bash[33851]: audit 2023-10-09T08:41:06.877571+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:41:08.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:08 smithi094 bash[27803]: cephadm 2023-10-09T08:41:06.319115+0000 mgr.y (mgr.25421) 21 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:41:08.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:08 smithi094 bash[27803]: cephadm 2023-10-09T08:41:06.369781+0000 mgr.y (mgr.25421) 22 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:41:08.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:08 smithi094 bash[27803]: cephadm 2023-10-09T08:41:06.534574+0000 mgr.y (mgr.25421) 23 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:41:08.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:08 smithi094 bash[27803]: cephadm 2023-10-09T08:41:06.584631+0000 mgr.y (mgr.25421) 24 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:41:08.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:08 smithi094 bash[27803]: cluster 2023-10-09T08:41:07.093018+0000 mgr.y (mgr.25421) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:08.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:08 smithi016 bash[33851]: cephadm 2023-10-09T08:41:06.319115+0000 mgr.y (mgr.25421) 21 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:41:08.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:08 smithi016 bash[33851]: cephadm 2023-10-09T08:41:06.369781+0000 mgr.y (mgr.25421) 22 : cephadm [INF] Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:41:08.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:08 smithi016 bash[33851]: cephadm 2023-10-09T08:41:06.534574+0000 mgr.y (mgr.25421) 23 : cephadm [INF] Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:41:08.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:08 smithi016 bash[33851]: cephadm 2023-10-09T08:41:06.584631+0000 mgr.y (mgr.25421) 24 : cephadm [INF] Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:41:08.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:08 smithi016 bash[33851]: cluster 2023-10-09T08:41:07.093018+0000 mgr.y (mgr.25421) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:08.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:41:08 smithi094 bash[55195]: ts=2023-10-09T08:41:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:41:09.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:09 smithi094 bash[27803]: cluster 2023-10-09T08:41:09.093746+0000 mgr.y (mgr.25421) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:10.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:09 smithi016 bash[33851]: cluster 2023-10-09T08:41:09.093746+0000 mgr.y (mgr.25421) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:12.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:12 smithi094 bash[27803]: cluster 2023-10-09T08:41:11.094874+0000 mgr.y (mgr.25421) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:12.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:12 smithi016 bash[33851]: cluster 2023-10-09T08:41:11.094874+0000 mgr.y (mgr.25421) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:14 smithi094 bash[27803]: cluster 2023-10-09T08:41:13.095532+0000 mgr.y (mgr.25421) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:14.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:14 smithi016 bash[33851]: cluster 2023-10-09T08:41:13.095532+0000 mgr.y (mgr.25421) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:16.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:41:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:41:15] "GET /metrics HTTP/1.1" 200 33706 "" "Prometheus/2.33.4" 2023-10-09T08:41:16.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:16 smithi094 bash[27803]: cluster 2023-10-09T08:41:15.096496+0000 mgr.y (mgr.25421) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:16.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:16 smithi016 bash[33851]: cluster 2023-10-09T08:41:15.096496+0000 mgr.y (mgr.25421) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:18.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:18 smithi094 bash[27803]: cluster 2023-10-09T08:41:17.097153+0000 mgr.y (mgr.25421) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:18.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:18 smithi016 bash[33851]: cluster 2023-10-09T08:41:17.097153+0000 mgr.y (mgr.25421) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:18.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:41:18 smithi094 bash[55195]: ts=2023-10-09T08:41:18.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:41:19.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:19 smithi094 bash[27803]: cluster 2023-10-09T08:41:19.097823+0000 mgr.y (mgr.25421) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:19.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:19 smithi094 bash[27803]: audit 2023-10-09T08:41:19.190973+0000 mon.c (mon.2) 259 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:20.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:19 smithi016 bash[33851]: cluster 2023-10-09T08:41:19.097823+0000 mgr.y (mgr.25421) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:20.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:19 smithi016 bash[33851]: audit 2023-10-09T08:41:19.190973+0000 mon.c (mon.2) 259 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:22.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:22 smithi094 bash[27803]: cluster 2023-10-09T08:41:21.098925+0000 mgr.y (mgr.25421) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:22.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:22 smithi016 bash[33851]: cluster 2023-10-09T08:41:21.098925+0000 mgr.y (mgr.25421) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:24.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:24 smithi094 bash[27803]: cluster 2023-10-09T08:41:23.099594+0000 mgr.y (mgr.25421) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:24.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:24 smithi016 bash[33851]: cluster 2023-10-09T08:41:23.099594+0000 mgr.y (mgr.25421) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:26.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:41:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:41:25] "GET /metrics HTTP/1.1" 200 33705 "" "Prometheus/2.33.4" 2023-10-09T08:41:26.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:26 smithi094 bash[27803]: cluster 2023-10-09T08:41:25.100556+0000 mgr.y (mgr.25421) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:26.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:26 smithi016 bash[33851]: cluster 2023-10-09T08:41:25.100556+0000 mgr.y (mgr.25421) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:28.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:28 smithi094 bash[27803]: cluster 2023-10-09T08:41:27.101198+0000 mgr.y (mgr.25421) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:28.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:28 smithi016 bash[33851]: cluster 2023-10-09T08:41:27.101198+0000 mgr.y (mgr.25421) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:28.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:41:28 smithi094 bash[55195]: ts=2023-10-09T08:41:28.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:41:29.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:29 smithi094 bash[27803]: cluster 2023-10-09T08:41:29.101821+0000 mgr.y (mgr.25421) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:30.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:29 smithi016 bash[33851]: cluster 2023-10-09T08:41:29.101821+0000 mgr.y (mgr.25421) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:32.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:32 smithi094 bash[27803]: cluster 2023-10-09T08:41:31.102914+0000 mgr.y (mgr.25421) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:32.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:32 smithi016 bash[33851]: cluster 2023-10-09T08:41:31.102914+0000 mgr.y (mgr.25421) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:34 smithi094 bash[27803]: cluster 2023-10-09T08:41:33.103564+0000 mgr.y (mgr.25421) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:34 smithi016 bash[33851]: cluster 2023-10-09T08:41:33.103564+0000 mgr.y (mgr.25421) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:35 smithi094 bash[27803]: audit 2023-10-09T08:41:34.191196+0000 mon.c (mon.2) 260 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:35.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:35 smithi016 bash[33851]: audit 2023-10-09T08:41:34.191196+0000 mon.c (mon.2) 260 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:36.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:41:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:41:35] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-10-09T08:41:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:36 smithi094 bash[27803]: cluster 2023-10-09T08:41:35.104434+0000 mgr.y (mgr.25421) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:36.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:36 smithi016 bash[33851]: cluster 2023-10-09T08:41:35.104434+0000 mgr.y (mgr.25421) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:38.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:38 smithi094 bash[27803]: cluster 2023-10-09T08:41:37.105035+0000 mgr.y (mgr.25421) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:38.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:38 smithi016 bash[33851]: cluster 2023-10-09T08:41:37.105035+0000 mgr.y (mgr.25421) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:38.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:41:38 smithi094 bash[55195]: ts=2023-10-09T08:41:38.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:41:39.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:39 smithi094 bash[27803]: cluster 2023-10-09T08:41:39.105674+0000 mgr.y (mgr.25421) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:40.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:39 smithi016 bash[33851]: cluster 2023-10-09T08:41:39.105674+0000 mgr.y (mgr.25421) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:42.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:42 smithi094 bash[27803]: cluster 2023-10-09T08:41:41.106817+0000 mgr.y (mgr.25421) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:42.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:42 smithi016 bash[33851]: cluster 2023-10-09T08:41:41.106817+0000 mgr.y (mgr.25421) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:44.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:44 smithi094 bash[27803]: cluster 2023-10-09T08:41:43.107494+0000 mgr.y (mgr.25421) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:44.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:44 smithi016 bash[33851]: cluster 2023-10-09T08:41:43.107494+0000 mgr.y (mgr.25421) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:46.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:41:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:41:45] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-10-09T08:41:46.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:46 smithi094 bash[27803]: cluster 2023-10-09T08:41:45.108495+0000 mgr.y (mgr.25421) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:46.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:46 smithi016 bash[33851]: cluster 2023-10-09T08:41:45.108495+0000 mgr.y (mgr.25421) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:48.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:48 smithi094 bash[27803]: cluster 2023-10-09T08:41:47.109150+0000 mgr.y (mgr.25421) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:48.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:48 smithi016 bash[33851]: cluster 2023-10-09T08:41:47.109150+0000 mgr.y (mgr.25421) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:48.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:41:48 smithi094 bash[55195]: ts=2023-10-09T08:41:48.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:41:49.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:49 smithi094 bash[27803]: cluster 2023-10-09T08:41:49.109804+0000 mgr.y (mgr.25421) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:49.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:49 smithi094 bash[27803]: audit 2023-10-09T08:41:49.193253+0000 mon.c (mon.2) 261 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:49 smithi016 bash[33851]: cluster 2023-10-09T08:41:49.109804+0000 mgr.y (mgr.25421) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:50.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:49 smithi016 bash[33851]: audit 2023-10-09T08:41:49.193253+0000 mon.c (mon.2) 261 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:41:52.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:52 smithi094 bash[27803]: cluster 2023-10-09T08:41:51.111030+0000 mgr.y (mgr.25421) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:52.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:52 smithi016 bash[33851]: cluster 2023-10-09T08:41:51.111030+0000 mgr.y (mgr.25421) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:54.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:54 smithi094 bash[27803]: cluster 2023-10-09T08:41:53.111643+0000 mgr.y (mgr.25421) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:54.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:54 smithi016 bash[33851]: cluster 2023-10-09T08:41:53.111643+0000 mgr.y (mgr.25421) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:56.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:41:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:41:55] "GET /metrics HTTP/1.1" 200 33703 "" "Prometheus/2.33.4" 2023-10-09T08:41:56.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:56 smithi094 bash[27803]: cluster 2023-10-09T08:41:55.112699+0000 mgr.y (mgr.25421) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:56.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:56 smithi016 bash[33851]: cluster 2023-10-09T08:41:55.112699+0000 mgr.y (mgr.25421) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:58.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:58 smithi094 bash[27803]: cluster 2023-10-09T08:41:57.113247+0000 mgr.y (mgr.25421) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:58.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:58 smithi016 bash[33851]: cluster 2023-10-09T08:41:57.113247+0000 mgr.y (mgr.25421) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:41:58.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:41:58 smithi094 bash[55195]: ts=2023-10-09T08:41:58.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:41:59.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T08:41:59.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:41:59 smithi094 bash[27803]: cluster 2023-10-09T08:41:59.113809+0000 mgr.y (mgr.25421) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:00.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:41:59 smithi016 bash[33851]: cluster 2023-10-09T08:41:59.113809+0000 mgr.y (mgr.25421) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:02.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:02 smithi016 bash[33851]: cluster 2023-10-09T08:42:01.114918+0000 mgr.y (mgr.25421) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:02.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:02 smithi094 bash[27803]: cluster 2023-10-09T08:42:01.114918+0000 mgr.y (mgr.25421) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:03.243 INFO:teuthology.orchestra.run.smithi016.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:42:04.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T08:42:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:04 smithi094 bash[27803]: cluster 2023-10-09T08:42:03.115619+0000 mgr.y (mgr.25421) 53 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:04 smithi094 bash[27803]: audit 2023-10-09T08:42:03.229325+0000 mgr.y (mgr.25421) 54 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:42:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:04 smithi094 bash[27803]: cephadm 2023-10-09T08:42:03.230024+0000 mgr.y (mgr.25421) 55 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:42:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:04 smithi094 bash[27803]: audit 2023-10-09T08:42:03.237814+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:04 smithi094 bash[27803]: audit 2023-10-09T08:42:03.278975+0000 mon.c (mon.2) 262 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:42:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:04 smithi094 bash[27803]: audit 2023-10-09T08:42:04.193601+0000 mon.c (mon.2) 263 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:42:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:04 smithi016 bash[33851]: cluster 2023-10-09T08:42:03.115619+0000 mgr.y (mgr.25421) 53 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:04 smithi016 bash[33851]: audit 2023-10-09T08:42:03.229325+0000 mgr.y (mgr.25421) 54 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:42:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:04 smithi016 bash[33851]: cephadm 2023-10-09T08:42:03.230024+0000 mgr.y (mgr.25421) 55 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:42:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:04 smithi016 bash[33851]: audit 2023-10-09T08:42:03.237814+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:04 smithi016 bash[33851]: audit 2023-10-09T08:42:03.278975+0000 mon.c (mon.2) 262 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:42:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:04 smithi016 bash[33851]: audit 2023-10-09T08:42:04.193601+0000 mon.c (mon.2) 263 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:42:06.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:42:05] "GET /metrics HTTP/1.1" 200 33698 "" "Prometheus/2.33.4" 2023-10-09T08:42:06.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:06 smithi016 bash[33851]: cluster 2023-10-09T08:42:05.116542+0000 mgr.y (mgr.25421) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:06.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:06 smithi094 bash[27803]: cluster 2023-10-09T08:42:05.116542+0000 mgr.y (mgr.25421) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:07.453 INFO:teuthology.orchestra.run.smithi016.stdout:wait for majority of mons to upgrade 2023-10-09T08:42:08.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T08:42:08.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:08 smithi016 bash[33851]: cluster 2023-10-09T08:42:07.117278+0000 mgr.y (mgr.25421) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:08.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:08 smithi094 bash[27803]: cluster 2023-10-09T08:42:07.117278+0000 mgr.y (mgr.25421) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:08.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:42:08 smithi094 bash[55195]: ts=2023-10-09T08:42:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:42:09.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:09 smithi094 bash[27803]: cluster 2023-10-09T08:42:09.117792+0000 mgr.y (mgr.25421) 58 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:09.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:09 smithi094 bash[27803]: audit 2023-10-09T08:42:09.626709+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:09.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:09 smithi094 bash[27803]: audit 2023-10-09T08:42:09.635660+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:10.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:09 smithi016 bash[33851]: cluster 2023-10-09T08:42:09.117792+0000 mgr.y (mgr.25421) 58 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:10.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:09 smithi016 bash[33851]: audit 2023-10-09T08:42:09.626709+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:10.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:09 smithi016 bash[33851]: audit 2023-10-09T08:42:09.635660+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:10.443445+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:10.455408+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:10.928483+0000 mon.c (mon.2) 264 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:10.930299+0000 mon.c (mon.2) 265 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:10.942736+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:11.025673+0000 mon.c (mon.2) 266 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:11.032493+0000 mon.c (mon.2) 267 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:42:11.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:11.034749+0000 mon.c (mon.2) 268 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:42:11.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: audit 2023-10-09T08:42:11.035917+0000 mon.c (mon.2) 269 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T08:42:11.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: cephadm 2023-10-09T08:42:11.036775+0000 mgr.y (mgr.25421) 59 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T08:42:11.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:11 smithi094 bash[27803]: cluster 2023-10-09T08:42:11.118807+0000 mgr.y (mgr.25421) 60 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:10.443445+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:10.455408+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:10.928483+0000 mon.c (mon.2) 264 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:10.930299+0000 mon.c (mon.2) 265 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:10.942736+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:11.025673+0000 mon.c (mon.2) 266 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:11.032493+0000 mon.c (mon.2) 267 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:42:11.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:11.034749+0000 mon.c (mon.2) 268 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:42:11.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: audit 2023-10-09T08:42:11.035917+0000 mon.c (mon.2) 269 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T08:42:11.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: cephadm 2023-10-09T08:42:11.036775+0000 mgr.y (mgr.25421) 59 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T08:42:11.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:11 smithi016 bash[33851]: cluster 2023-10-09T08:42:11.118807+0000 mgr.y (mgr.25421) 60 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:12.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:12 smithi094 bash[27803]: audit 2023-10-09T08:42:11.775263+0000 mon.a (mon.0) 45 : audit [DBG] from='client.? 172.21.15.16:0/2542512327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:42:12.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:12 smithi016 bash[33851]: audit 2023-10-09T08:42:11.775263+0000 mon.a (mon.0) 45 : audit [DBG] from='client.? 172.21.15.16:0/2542512327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:42:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:13 smithi094 bash[27803]: cluster 2023-10-09T08:42:13.119496+0000 mgr.y (mgr.25421) 61 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:13.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:13 smithi016 bash[33851]: cluster 2023-10-09T08:42:13.119496+0000 mgr.y (mgr.25421) 61 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:14.899 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:14 smithi016 bash[33851]: cephadm 2023-10-09T08:42:13.606688+0000 mgr.y (mgr.25421) 62 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T08:42:14.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:14 smithi016 bash[33851]: audit 2023-10-09T08:42:13.616508+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:14.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:14 smithi016 bash[33851]: audit 2023-10-09T08:42:13.618614+0000 mon.c (mon.2) 270 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:42:14.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:14 smithi016 bash[33851]: audit 2023-10-09T08:42:13.620171+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:42:14.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:14 smithi016 bash[33851]: audit 2023-10-09T08:42:13.621477+0000 mon.c (mon.2) 271 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:42:14.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:14 smithi016 bash[33851]: cephadm 2023-10-09T08:42:13.622849+0000 mgr.y (mgr.25421) 63 : cephadm [INF] Deploying daemon mon.c on smithi016 2023-10-09T08:42:14.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:14 smithi016 bash[33851]: audit 2023-10-09T08:42:14.223255+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.16:0/1457947433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:42:14.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:14 smithi094 bash[27803]: cephadm 2023-10-09T08:42:13.606688+0000 mgr.y (mgr.25421) 62 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T08:42:14.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:14 smithi094 bash[27803]: audit 2023-10-09T08:42:13.616508+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25421 ' entity='mgr.y' 2023-10-09T08:42:14.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:14 smithi094 bash[27803]: audit 2023-10-09T08:42:13.618614+0000 mon.c (mon.2) 270 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:42:14.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:14 smithi094 bash[27803]: audit 2023-10-09T08:42:13.620171+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:42:14.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:14 smithi094 bash[27803]: audit 2023-10-09T08:42:13.621477+0000 mon.c (mon.2) 271 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:42:14.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:14 smithi094 bash[27803]: cephadm 2023-10-09T08:42:13.622849+0000 mgr.y (mgr.25421) 63 : cephadm [INF] Deploying daemon mon.c on smithi016 2023-10-09T08:42:14.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:14 smithi094 bash[27803]: audit 2023-10-09T08:42:14.223255+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.16:0/1457947433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:42:15.938 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:42:15] "GET /metrics HTTP/1.1" 200 33698 "" "Prometheus/2.33.4" 2023-10-09T08:42:15.939 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:15 smithi016 bash[33851]: cluster 2023-10-09T08:42:15.120379+0000 mgr.y (mgr.25421) 64 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:15.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:15 smithi094 bash[27803]: cluster 2023-10-09T08:42:15.120379+0000 mgr.y (mgr.25421) 64 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:16.473 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[71225]: [09/Oct/2023:08:42:16] ENGINE Bus STOPPING 2023-10-09T08:42:16.474 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[71225]: [09/Oct/2023:08:42:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T08:42:16.474 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[71225]: [09/Oct/2023:08:42:16] ENGINE Bus STOPPED 2023-10-09T08:42:16.474 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[71225]: [09/Oct/2023:08:42:16] ENGINE Bus STARTING 2023-10-09T08:42:16.474 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[71225]: [09/Oct/2023:08:42:16] ENGINE Serving on http://:::9283 2023-10-09T08:42:16.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:16 smithi016 systemd[1]: Stopping Ceph mon.c for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:42:16.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[33851]: debug 2023-10-09T08:42:16.341+0000 7fa9b7847700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T08:42:16.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[33851]: debug 2023-10-09T08:42:16.341+0000 7fa9b7847700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T08:42:16.753 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[71225]: [09/Oct/2023:08:42:16] ENGINE Bus STARTED 2023-10-09T08:42:17.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:16 smithi016 bash[77737]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T08:42:17.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:17 smithi016 bash[77836]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T08:42:17.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:17 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T08:42:17.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:17 smithi016 systemd[1]: Stopped Ceph mon.c for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:42:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:17 smithi016 systemd[1]: Started Ceph mon.c for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:42:18.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.485+0000 7f248eb94c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T08:42:18.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.485+0000 7f248eb94c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T08:42:18.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.485+0000 7f248eb94c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T08:42:18.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 0 load: jerasure load: lrc 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Git sha 0 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: DB SUMMARY 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: DB Session ID: EU9K8NEZ16XBNJR9CYIA 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1276 Bytes 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000026.sst 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000024.log size: 1819394 ; 2023-10-09T08:42:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.env: 0x55f06fd20720 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.info_log: 0x55f0725c1400 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T08:42:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T08:42:18.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T08:42:18.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T08:42:18.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T08:42:18.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.db_log_dir: 2023-10-09T08:42:18.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.wal_dir: 2023-10-09T08:42:18.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T08:42:18.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.write_buffer_manager: 0x55f07184e5a0 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T08:42:18.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.row_cache: None 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.wal_filter: None 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T08:42:18.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T08:42:18.761 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T08:42:18.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T08:42:18.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Compression algorithms supported: 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kZSTD supported: 0 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-09T08:42:18.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.merge_operator: 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f0725c1520) 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: cache_index_and_filter_blocks: 1 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T08:42:18.765 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: pin_top_level_index_and_filter: 1 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: index_type: 0 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: data_block_index_type: 0 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: index_shortening: 1 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: checksum: 4 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: no_block_cache: 0 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_cache: 0x55f0718d11f0 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_cache_name: BinnedLRUCache 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_cache_options: 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: capacity : 536870912 2023-10-09T08:42:18.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: num_shard_bits : 4 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: strict_capacity_limit : 0 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: high_pri_pool_ratio: 0.000 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_cache_compressed: (nil) 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: persistent_cache: (nil) 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_size: 4096 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_size_deviation: 10 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_restart_interval: 16 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: index_block_restart_interval: 1 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: metadata_block_size: 4096 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: partition_filters: 0 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: use_delta_encoding: 1 2023-10-09T08:42:18.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: filter_policy: bloomfilter 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: whole_key_filtering: 1 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: verify_compression: 0 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: read_amp_bytes_per_bit: 0 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: format_version: 5 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: enable_index_compression: 1 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: block_align: 0 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: max_auto_readahead_size: 262144 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: prepopulate_block_cache: 0 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: initial_auto_readahead_size: 8192 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: num_file_reads_for_auto_readahead: 2 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T08:42:18.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.num_levels: 7 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T08:42:18.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:42:18.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T08:42:18.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T08:42:18.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T08:42:18.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T08:42:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T08:42:18.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f24833b0700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 28, last_sequence is 17677, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dbc011e2-b7af-493c-9b3b-8181971c3d87 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696840938499295, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.493+0000 7f248eb94c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f248eb94c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696840938505259, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 1209403, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17672, "largest_seqno": 18045, "table_properties": {"data_size": 1206440, "index_size": 1631, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2579, "raw_average_key_size": 21, "raw_value_size": 1202869, "raw_average_value_size": 10023, "num_data_blocks": 71, "num_entries": 120, "num_filter_entries": 120, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696840938, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dbc011e2-b7af-493c-9b3b-8181971c3d87", "db_session_id": "EU9K8NEZ16XBNJR9CYIA", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f248eb94c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696840938505391, "job": 1, "event": "recovery_finished"} 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f248eb94c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-10-09T08:42:18.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f248eb94c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f248eb94c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f071972000 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f248eb94c80 4 rocksdb: DB pointer 0x55f071960000 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f247d3a4700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.501+0000 7f247d3a4700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: ** DB Stats ** 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:42:18.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: ** Compaction Stats [default] ** 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: L0 1/0 1.15 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 376.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: L6 1/0 32.23 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Sum 2/0 33.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 376.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 376.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: ** Compaction Stats [default] ** 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:42:18.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 376.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T08:42:18.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T08:42:18.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:42:18.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T08:42:18.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:42:18.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Cumulative compaction: 0.00 GB write, 119.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Interval compaction: 0.00 GB write, 119.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Block cache BinnedLRUCache@0x55f0718d11f0#7 capacity: 512.00 MB usage: 2.20 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] at bind addrs [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:42:18.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 1 mon.c@-1(???) e3 preinit fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 0 mon.c@-1(???).mds e7 new map 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 0 mon.c@-1(???).mds e7 print_map 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: e7 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: legacy client fscid: 1 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Filesystem 'foo' (1) 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: fs_name foo 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: epoch 5 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T08:42:18.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: created 2023-10-09T08:34:27.739170+0000 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: modified 2023-10-09T08:34:35.063882+0000 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: tableserver 0 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: root 0 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: session_timeout 60 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: session_autoclose 300 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: max_file_size 1099511627776 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: required_client_features {} 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: last_failure 0 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: last_failure_osd_epoch 0 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:42:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: max_mds 1 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: in 0 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: up {0=14634} 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: failed 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: damaged 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: stopped 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: data_pools [4] 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: metadata_pool 3 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: inline_data disabled 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: balancer 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: bal_rank_mask -1 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: standby_count_wanted 1 2023-10-09T08:42:18.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: [mds.foo.smithi016.ccdktz{0:14634} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: Standby daemons: 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: [mds.foo.smithi094.zfssci{-1:24523} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.94:6832/1000218818,v1:172.21.15.94:6833/1000218818] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 0 mon.c@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 0 mon.c@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 0 mon.c@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 0 mon.c@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.505+0000 7f248eb94c80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T08:42:18.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T08:42:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:42:18 smithi016 bash[77892]: debug 2023-10-09T08:42:18.509+0000 7f248eb94c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T08:42:18.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:42:18 smithi094 bash[55195]: ts=2023-10-09T08:42:18.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:42:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:23 smithi094 bash[27803]: audit 2023-10-09T08:42:16.351539+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:42:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:23 smithi094 bash[27803]: audit 2023-10-09T08:42:16.353776+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:42:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:23 smithi094 bash[27803]: audit 2023-10-09T08:42:16.354084+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:42:25.156 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T08:42:25.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.744328+0000 mon.a (mon.0) 66 : cluster [INF] mon.a calling monitor election 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.745989+0000 mon.b (mon.1) 51 : cluster [INF] mon.b calling monitor election 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.782924+0000 mon.a (mon.0) 67 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.789695+0000 mon.a (mon.0) 68 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.790718+0000 mon.a (mon.0) 69 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.790781+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.791347+0000 mon.a (mon.0) 71 : cluster [DBG] mgrmap e36: y(active, since 95s), standbys: x 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.791780+0000 mon.a (mon.0) 72 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:42:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:24 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.791828+0000 mon.a (mon.0) 73 : cluster [INF] Cluster is now healthy 2023-10-09T08:42:26.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:42:25] "GET /metrics HTTP/1.1" 200 33697 "" "Prometheus/2.33.4" 2023-10-09T08:42:26.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T08:42:26.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:25 smithi094 bash[27803]: cluster 2023-10-09T08:42:18.721886+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T08:42:26.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:25 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.813995+0000 mon.a (mon.0) 74 : cluster [INF] overall HEALTH_OK 2023-10-09T08:42:26.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:25 smithi094 bash[27803]: cluster 2023-10-09T08:42:24.826688+0000 mon.a (mon.0) 75 : cluster [DBG] mgrmap e37: y(active, since 95s), standbys: x 2023-10-09T08:42:26.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:25 smithi094 bash[27803]: cluster 2023-10-09T08:42:25.123940+0000 mgr.y (mgr.25421) 69 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:42:26.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:25 smithi094 bash[27803]: audit 2023-10-09T08:42:25.136890+0000 mon.a (mon.0) 76 : audit [DBG] from='client.? 172.21.15.16:0/3422363337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:42:28.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:28 smithi094 bash[27803]: cluster 2023-10-09T08:42:27.124569+0000 mgr.y (mgr.25421) 70 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:42:28.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:42:28 smithi094 bash[55195]: ts=2023-10-09T08:42:28.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:42:29.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:28 smithi016 bash[71225]: debug 2023-10-09T08:42:28.513+0000 7f5681439700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T08:42:29.520 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T08:42:29.520 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (3m) - 9m 13.4M - 0.23.0 ba2b418f427c b9989d9bd43e 2023-10-09T08:42:29.520 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi094 *:3000 running (3m) - 9m 40.4M - 8.3.5 dad864ee21e9 f1a12efb3a56 2023-10-09T08:42:29.520 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.ccdktz smithi016 running (7m) - 7m 15.8M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 17a3e2c7a98b 2023-10-09T08:42:29.520 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi094.zfssci smithi094 running (7m) - 7m 13.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a917ff5f77b 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi094 *:8443,9283,8765 running (2m) - 17m 426M - 18.2.0-598-gb4fa663f 3060897fdf97 f0fcc951b93f 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (3m) - 19m 472M - 18.2.0-598-gb4fa663f 3060897fdf97 cdf79c37f30d 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (111s) - 19m 47.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 a8c0ba734014 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi094 running (17m) - 17m 54.7M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5e9c5bdb4c92 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 starting - - - 2048M 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (3m) - 10m 9195k - 1.3.1 1dbe0e931976 6c4a45c1d04a 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi094 *:9100 running (3m) - 10m 8523k - 1.3.1 1dbe0e931976 59bcf67d7eb5 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (16m) - 16m 46.1M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c2ec86d01448 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (15m) - 15m 43.6M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 302ee697c26f 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (14m) - 15m 51.9M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c358a7abf935 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (14m) - 14m 42.9M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0f6ebb2bd802 2023-10-09T08:42:29.521 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi094 running (13m) - 13m 45.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1a20cacc6df3 2023-10-09T08:42:29.522 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi094 running (12m) - 12m 49.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 381d98a4283c 2023-10-09T08:42:29.522 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi094 running (11m) - 11m 42.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d8307333e14e 2023-10-09T08:42:29.522 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi094 running (11m) - 11m 52.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 04152ef43de4 2023-10-09T08:42:29.522 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi094 *:9095 running (2m) - 10m 53.3M - 2.33.4 514e6a882f6e 4aee8e4f263b 2023-10-09T08:42:29.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:29 smithi094 bash[27803]: cluster 2023-10-09T08:42:29.125146+0000 mgr.y (mgr.25421) 71 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:42:30.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T08:42:30.492 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:42:30 smithi094 bash[55195]: ts=2023-10-09T08:42:30.089Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", public_addr=\"172.21.15.16\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.16\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-10-09T08:42:31.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:30 smithi094 bash[27803]: audit 2023-10-09T08:42:29.498558+0000 mgr.y (mgr.25421) 72 : audit [DBG] from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:42:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:30 smithi094 bash[27803]: audit 2023-10-09T08:42:29.751636+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:30 smithi094 bash[27803]: audit 2023-10-09T08:42:29.759311+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:30 smithi094 bash[27803]: audit 2023-10-09T08:42:30.602913+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:31.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:30 smithi094 bash[27803]: audit 2023-10-09T08:42:30.608769+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:31 smithi094 bash[27803]: cluster 2023-10-09T08:42:31.126073+0000 mgr.y (mgr.25421) 73 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:42:33.818 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:42:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:34 smithi094 bash[27803]: cluster 2023-10-09T08:42:33.126795+0000 mgr.y (mgr.25421) 74 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:34 smithi094 bash[27803]: audit 2023-10-09T08:42:33.813436+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:34.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T08:42:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:35 smithi094 bash[27803]: audit 2023-10-09T08:42:33.808234+0000 mgr.y (mgr.25421) 75 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:42:35.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:35 smithi094 bash[27803]: cephadm 2023-10-09T08:42:33.808918+0000 mgr.y (mgr.25421) 76 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:42:35.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:35 smithi094 bash[27803]: audit 2023-10-09T08:42:34.194441+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:42:36.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:42:35] "GET /metrics HTTP/1.1" 200 33690 "" "Prometheus/2.33.4" 2023-10-09T08:42:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:36 smithi094 bash[27803]: cluster 2023-10-09T08:42:35.128006+0000 mgr.y (mgr.25421) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:38.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:38 smithi094 bash[27803]: cluster 2023-10-09T08:42:37.128610+0000 mgr.y (mgr.25421) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:38.993 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:42:38 smithi094 bash[55195]: ts=2023-10-09T08:42:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:42:39.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:39 smithi094 bash[27803]: cluster 2023-10-09T08:42:39.129207+0000 mgr.y (mgr.25421) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: cluster 2023-10-09T08:42:41.130339+0000 mgr.y (mgr.25421) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.559421+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.566812+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.568680+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.570174+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.577608+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.658295+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.661125+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.662461+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:42:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:42 smithi094 bash[27803]: audit 2023-10-09T08:42:41.670736+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:42:44.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:44 smithi094 bash[27803]: cluster 2023-10-09T08:42:43.131018+0000 mgr.y (mgr.25421) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:46.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:42:45] "GET /metrics HTTP/1.1" 200 33690 "" "Prometheus/2.33.4" 2023-10-09T08:42:46.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:46 smithi094 bash[27803]: cluster 2023-10-09T08:42:45.132081+0000 mgr.y (mgr.25421) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:48.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:48 smithi094 bash[27803]: cluster 2023-10-09T08:42:47.132662+0000 mgr.y (mgr.25421) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:48.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:42:48 smithi094 bash[55195]: ts=2023-10-09T08:42:48.665Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:42:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:49 smithi094 bash[27803]: cluster 2023-10-09T08:42:49.133322+0000 mgr.y (mgr.25421) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:49 smithi094 bash[27803]: audit 2023-10-09T08:42:49.194682+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:42:52.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:52 smithi094 bash[27803]: cluster 2023-10-09T08:42:51.134431+0000 mgr.y (mgr.25421) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:54.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:54 smithi094 bash[27803]: cluster 2023-10-09T08:42:53.135086+0000 mgr.y (mgr.25421) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:56.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:42:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:42:55] "GET /metrics HTTP/1.1" 200 33691 "" "Prometheus/2.33.4" 2023-10-09T08:42:56.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:56 smithi094 bash[27803]: cluster 2023-10-09T08:42:55.136125+0000 mgr.y (mgr.25421) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:58.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:58 smithi094 bash[27803]: cluster 2023-10-09T08:42:57.136769+0000 mgr.y (mgr.25421) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:42:58.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:42:58 smithi094 bash[55195]: ts=2023-10-09T08:42:58.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:43:00.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:42:59 smithi094 bash[27803]: cluster 2023-10-09T08:42:59.137426+0000 mgr.y (mgr.25421) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:02.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:02 smithi094 bash[27803]: cluster 2023-10-09T08:43:01.138544+0000 mgr.y (mgr.25421) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:04 smithi094 bash[27803]: cluster 2023-10-09T08:43:03.139210+0000 mgr.y (mgr.25421) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:04.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:04 smithi094 bash[27803]: audit 2023-10-09T08:43:04.195024+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:43:06.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:43:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:43:05] "GET /metrics HTTP/1.1" 200 33704 "" "Prometheus/2.33.4" 2023-10-09T08:43:06.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:06 smithi094 bash[27803]: cluster 2023-10-09T08:43:05.140333+0000 mgr.y (mgr.25421) 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:08.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:08 smithi094 bash[27803]: cluster 2023-10-09T08:43:07.141080+0000 mgr.y (mgr.25421) 93 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:08.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:43:08 smithi094 bash[55195]: ts=2023-10-09T08:43:08.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:43:10.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:09 smithi094 bash[27803]: cluster 2023-10-09T08:43:09.141772+0000 mgr.y (mgr.25421) 94 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:12.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:12 smithi094 bash[27803]: cluster 2023-10-09T08:43:11.142791+0000 mgr.y (mgr.25421) 95 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:14 smithi094 bash[27803]: cluster 2023-10-09T08:43:13.143368+0000 mgr.y (mgr.25421) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:16.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:43:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:43:15] "GET /metrics HTTP/1.1" 200 33704 "" "Prometheus/2.33.4" 2023-10-09T08:43:16.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:16 smithi094 bash[27803]: cluster 2023-10-09T08:43:15.144420+0000 mgr.y (mgr.25421) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:18.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:18 smithi094 bash[27803]: cluster 2023-10-09T08:43:17.145062+0000 mgr.y (mgr.25421) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:18.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:43:18 smithi094 bash[55195]: ts=2023-10-09T08:43:18.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:43:20.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:19 smithi094 bash[27803]: cluster 2023-10-09T08:43:19.145815+0000 mgr.y (mgr.25421) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:19 smithi094 bash[27803]: audit 2023-10-09T08:43:19.195516+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:43:22.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:22 smithi094 bash[27803]: cluster 2023-10-09T08:43:21.146943+0000 mgr.y (mgr.25421) 100 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:24.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:24 smithi094 bash[27803]: cluster 2023-10-09T08:43:23.147568+0000 mgr.y (mgr.25421) 101 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:26.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:43:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:43:25] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-10-09T08:43:26.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:26 smithi094 bash[27803]: cluster 2023-10-09T08:43:25.148576+0000 mgr.y (mgr.25421) 102 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:28.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:28 smithi094 bash[27803]: cluster 2023-10-09T08:43:27.149230+0000 mgr.y (mgr.25421) 103 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:29.001 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:43:28 smithi094 bash[55195]: ts=2023-10-09T08:43:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:43:30.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:29 smithi094 bash[27803]: cluster 2023-10-09T08:43:29.149865+0000 mgr.y (mgr.25421) 104 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:32.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:32 smithi094 bash[27803]: cluster 2023-10-09T08:43:31.150901+0000 mgr.y (mgr.25421) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:34 smithi094 bash[27803]: cluster 2023-10-09T08:43:33.151622+0000 mgr.y (mgr.25421) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:34 smithi094 bash[27803]: audit 2023-10-09T08:43:34.195807+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:43:36.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:43:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:43:35] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-10-09T08:43:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:36 smithi094 bash[27803]: cluster 2023-10-09T08:43:35.152719+0000 mgr.y (mgr.25421) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:38 smithi094 bash[27803]: cluster 2023-10-09T08:43:37.153459+0000 mgr.y (mgr.25421) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:38.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:43:38 smithi094 bash[55195]: ts=2023-10-09T08:43:38.670Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:43:39.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T08:43:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:39 smithi094 bash[27803]: cluster 2023-10-09T08:43:39.154156+0000 mgr.y (mgr.25421) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:42.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:42 smithi094 bash[27803]: cluster 2023-10-09T08:43:41.155311+0000 mgr.y (mgr.25421) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:42 smithi094 bash[27803]: audit 2023-10-09T08:43:41.753992+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:43:42.758 INFO:teuthology.orchestra.run.smithi016.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:43:43.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T08:43:44.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:43 smithi094 bash[27803]: audit 2023-10-09T08:43:42.746990+0000 mgr.y (mgr.25421) 111 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:43:44.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:43 smithi094 bash[27803]: cephadm 2023-10-09T08:43:42.747590+0000 mgr.y (mgr.25421) 112 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:43:44.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:43 smithi094 bash[27803]: audit 2023-10-09T08:43:42.753123+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:43:44.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:43 smithi094 bash[27803]: cluster 2023-10-09T08:43:43.156153+0000 mgr.y (mgr.25421) 113 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:46.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:43:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:43:45] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-10-09T08:43:46.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:46 smithi094 bash[27803]: cluster 2023-10-09T08:43:45.157209+0000 mgr.y (mgr.25421) 114 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:46.605 INFO:teuthology.orchestra.run.smithi016.stdout:wait for all mons to upgrade 2023-10-09T08:43:47.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T08:43:48.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:48 smithi094 bash[27803]: cluster 2023-10-09T08:43:47.157643+0000 mgr.y (mgr.25421) 115 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:48 smithi094 bash[27803]: audit 2023-10-09T08:43:47.996292+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:43:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:48 smithi094 bash[27803]: audit 2023-10-09T08:43:48.003729+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:43:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:48 smithi094 bash[27803]: audit 2023-10-09T08:43:48.143384+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:43:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:48 smithi094 bash[27803]: audit 2023-10-09T08:43:48.150872+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:43:48.992 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 08:43:48 smithi094 bash[55195]: ts=2023-10-09T08:43:48.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi094.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T08:43:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:48.627726+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:43:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:48.629094+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:43:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:48.638713+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:43:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:48.719812+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:43:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:48.724153+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:43:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:48.726424+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:43:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:48.727624+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T08:43:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:49 smithi094 bash[27803]: audit 2023-10-09T08:43:49.196187+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:43:50.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:50 smithi094 bash[27803]: cephadm 2023-10-09T08:43:48.728545+0000 mgr.y (mgr.25421) 116 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-09T08:43:50.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:50 smithi094 bash[27803]: cluster 2023-10-09T08:43:49.158022+0000 mgr.y (mgr.25421) 117 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:51 smithi094 bash[27803]: audit 2023-10-09T08:43:51.019640+0000 mon.a (mon.0) 110 : audit [DBG] from='client.? 172.21.15.16:0/2234495327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:43:52.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:52 smithi094 bash[27803]: cluster 2023-10-09T08:43:51.159175+0000 mgr.y (mgr.25421) 118 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:43:52.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:52 smithi094 bash[27803]: cephadm 2023-10-09T08:43:51.249454+0000 mgr.y (mgr.25421) 119 : cephadm [INF] Upgrade: Updating mon.b 2023-10-09T08:43:52.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:52 smithi094 bash[27803]: audit 2023-10-09T08:43:51.256418+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:43:52.525 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:52 smithi094 bash[27803]: audit 2023-10-09T08:43:51.258303+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:43:52.525 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:52 smithi094 bash[27803]: audit 2023-10-09T08:43:51.259618+0000 mon.a (mon.0) 113 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:43:52.525 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:52 smithi094 bash[27803]: audit 2023-10-09T08:43:51.260740+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:43:52.525 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:52 smithi094 bash[27803]: cephadm 2023-10-09T08:43:51.262069+0000 mgr.y (mgr.25421) 120 : cephadm [INF] Deploying daemon mon.b on smithi094 2023-10-09T08:43:53.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:53 smithi094 systemd[1]: Stopping Ceph mon.b for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:43:54.309 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:54 smithi094 bash[27803]: debug 2023-10-09T08:43:54.012+0000 7fb4f4920700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T08:43:54.309 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:54 smithi094 bash[27803]: debug 2023-10-09T08:43:54.012+0000 7fb4f4920700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T08:43:55.190 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:54 smithi094 bash[59556]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T08:43:55.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:55 smithi094 bash[59622]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T08:43:55.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:55 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T08:43:55.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:55 smithi094 systemd[1]: Stopped Ceph mon.b for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:43:55.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:55 smithi094 systemd[1]: Started Ceph mon.b for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:43:56.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:43:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:43:55] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-10-09T08:43:56.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.708+0000 7f9d8b9a7c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T08:43:56.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.708+0000 7f9d8b9a7c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T08:43:56.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.708+0000 7f9d8b9a7c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T08:43:56.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 0 load: jerasure load: lrc 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Git sha 0 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: DB SUMMARY 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: DB Session ID: DG95IICUDE4A5GEFR5GH 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1663 Bytes 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000032.sst 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000030.log size: 2191832 ; 2023-10-09T08:43:56.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.env: 0x55d123804720 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.info_log: 0x55d125507400 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T08:43:56.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T08:43:56.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T08:43:56.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T08:43:56.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T08:43:56.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T08:43:56.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T08:43:56.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T08:43:56.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T08:43:56.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.db_log_dir: 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.wal_dir: 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T08:43:56.998 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.write_buffer_manager: 0x55d1247945a0 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T08:43:56.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.row_cache: None 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.wal_filter: None 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T08:43:57.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T08:43:57.001 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T08:43:57.002 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T08:43:57.003 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Compression algorithms supported: 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kZSTD supported: 0 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T08:43:57.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.merge_operator: 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T08:43:57.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d125507520) 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: cache_index_and_filter_blocks: 1 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: pin_top_level_index_and_filter: 1 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: index_type: 0 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: data_block_index_type: 0 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: index_shortening: 1 2023-10-09T08:43:57.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: checksum: 4 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: no_block_cache: 0 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_cache: 0x55d1248171f0 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_cache_name: BinnedLRUCache 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_cache_options: 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: capacity : 536870912 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: num_shard_bits : 4 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: strict_capacity_limit : 0 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: high_pri_pool_ratio: 0.000 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_cache_compressed: (nil) 2023-10-09T08:43:57.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: persistent_cache: (nil) 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_size: 4096 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_size_deviation: 10 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_restart_interval: 16 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: index_block_restart_interval: 1 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: metadata_block_size: 4096 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: partition_filters: 0 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: use_delta_encoding: 1 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: filter_policy: bloomfilter 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: whole_key_filtering: 1 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: verify_compression: 0 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: read_amp_bytes_per_bit: 0 2023-10-09T08:43:57.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: format_version: 5 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: enable_index_compression: 1 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: block_align: 0 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: max_auto_readahead_size: 262144 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: prepopulate_block_cache: 0 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: initial_auto_readahead_size: 8192 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: num_file_reads_for_auto_readahead: 2 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T08:43:57.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.num_levels: 7 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T08:43:57.010 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T08:43:57.011 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T08:43:57.012 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T08:43:57.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T08:43:57.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T08:43:57.015 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T08:43:57.015 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T08:43:57.016 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T08:43:57.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d8b9a7c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.712+0000 7f9d811c5700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 32.sst 2023-10-09T08:43:57.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.716+0000 7f9d8b9a7c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 34, last_sequence is 19653, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.716+0000 7f9d8b9a7c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.716+0000 7f9d8b9a7c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: cf18d9eb-8f3f-4be2-adc0-2dac5fe32293 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.716+0000 7f9d8b9a7c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696841036721827, "job": 1, "event": "recovery_started", "wal_files": [30]} 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.716+0000 7f9d8b9a7c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.724+0000 7f9d8b9a7c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696841036731003, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 1455743, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19648, "largest_seqno": 20113, "table_properties": {"data_size": 1452384, "index_size": 1961, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3034, "raw_average_key_size": 21, "raw_value_size": 1448122, "raw_average_value_size": 10056, "num_data_blocks": 86, "num_entries": 144, "num_filter_entries": 144, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696841036, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "cf18d9eb-8f3f-4be2-adc0-2dac5fe32293", "db_session_id": "DG95IICUDE4A5GEFR5GH", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.724+0000 7f9d8b9a7c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696841036731139, "job": 1, "event": "recovery_finished"} 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.724+0000 7f9d8b9a7c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 37 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d1248b8000 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 4 rocksdb: DB pointer 0x55d1248a6000 2023-10-09T08:43:57.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d7a1b7700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d7a1b7700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: ** DB Stats ** 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: ** Compaction Stats [default] ** 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T08:43:57.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: L0 1/0 1.39 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 339.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: L6 1/0 29.06 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Sum 2/0 30.44 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 339.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 339.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: ** Compaction Stats [default] ** 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 339.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T08:43:57.021 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Cumulative compaction: 0.00 GB write, 103.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Interval compaction: 0.00 GB write, 103.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Block cache BinnedLRUCache@0x55d1248171f0#7 capacity: 512.00 MB usage: 2.52 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.4e-05 secs_since: 0 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,2.08 KB,0.000396371%) Misc(1,0.00 KB,0%) 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] at bind addrs [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:43:57.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 1 mon.b@-1(???) e3 preinit fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 0 mon.b@-1(???).mds e7 new map 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 0 mon.b@-1(???).mds e7 print_map 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: e7 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: legacy client fscid: 1 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Filesystem 'foo' (1) 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: fs_name foo 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: epoch 5 2023-10-09T08:43:57.023 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: created 2023-10-09T08:34:27.739170+0000 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: modified 2023-10-09T08:34:35.063882+0000 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: tableserver 0 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: root 0 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: session_timeout 60 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: session_autoclose 300 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: max_file_size 1099511627776 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: required_client_features {} 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: last_failure 0 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: last_failure_osd_epoch 0 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:43:57.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: max_mds 1 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: in 0 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: up {0=14634} 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: failed 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: damaged 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: stopped 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: data_pools [4] 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: metadata_pool 3 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: inline_data disabled 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: balancer 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: bal_rank_mask -1 2023-10-09T08:43:57.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: standby_count_wanted 1 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: [mds.foo.smithi016.ccdktz{0:14634} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.16:6834/186141155,v1:172.21.15.16:6835/186141155] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: Standby daemons: 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: [mds.foo.smithi094.zfssci{-1:24523} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.94:6832/1000218818,v1:172.21.15.94:6833/1000218818] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 0 mon.b@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 0 mon.b@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 0 mon.b@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.728+0000 7f9d8b9a7c80 0 mon.b@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T08:43:57.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:43:56 smithi094 bash[59677]: debug 2023-10-09T08:43:56.732+0000 7f9d8b9a7c80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2023-10-09T08:44:02.006 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:44:01 smithi016 bash[71225]: debug 2023-10-09T08:44:01.739+0000 7f5681439700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T08:44:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:23:18.622286+0000 mon.a (mon.0) 0 : 1 mkfs f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:44:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:38:54.223270+0000 mgr.x (mgr.24790) 46 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T08:44:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:38:54.224111+0000 mgr.x (mgr.24790) 47 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi094 2023-10-09T08:44:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:38:56.192222+0000 mgr.x (mgr.24790) 49 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T08:44:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:38:56.215527+0000 mgr.x (mgr.24790) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi094 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:21.027458+0000 mgr.x (mgr.24790) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:21.028145+0000 mgr.x (mgr.24790) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:23.247533+0000 mgr.x (mgr.24790) 70 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:23.266588+0000 mgr.x (mgr.24790) 71 : cephadm 1 Deploying daemon mgr.y on smithi016 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:43.996320+0000 mgr.y (mgr.15261) 1 : cephadm 1 [09/Oct/2023:08:39:43] ENGINE Bus STARTING 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:44.098807+0000 mgr.y (mgr.15261) 2 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:44.279552+0000 mgr.y (mgr.15261) 3 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:39:44.290367+0000 mgr.y (mgr.15261) 4 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Bus STARTED 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.023103+0000 mgr.y (mgr.15261) 14 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.023394+0000 mgr.y (mgr.15261) 15 : cephadm 1 Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:44:02.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.187916+0000 mgr.y (mgr.15261) 16 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.239122+0000 mgr.y (mgr.15261) 17 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.352424+0000 mgr.y (mgr.15261) 18 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.404481+0000 mgr.y (mgr.15261) 20 : cephadm 1 Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.517057+0000 mgr.y (mgr.15261) 21 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.568903+0000 mgr.y (mgr.15261) 22 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:01.831073+0000 mgr.y (mgr.15261) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:02.099144+0000 mgr.y (mgr.15261) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:44:02.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:10.527339+0000 mgr.y (mgr.15261) 30 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T08:44:02.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:10.533809+0000 mgr.y (mgr.15261) 31 : cephadm 1 Deploying daemon mgr.x on smithi094 2023-10-09T08:44:02.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:31.042350+0000 mgr.y (mgr.15261) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:31.068083+0000 mgr.y (mgr.15261) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:33.589082+0000 mgr.y (mgr.15261) 46 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:33.604198+0000 mgr.y (mgr.15261) 47 : cephadm 1 Deploying daemon mon.a on smithi016 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:49.942190+0000 mgr.y (mgr.25421) 1 : cephadm 1 [09/Oct/2023:08:40:49] ENGINE Bus STARTING 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:50.063452+0000 mgr.y (mgr.25421) 2 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:50.165832+0000 mgr.y (mgr.25421) 5 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:50.166180+0000 mgr.y (mgr.25421) 6 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Bus STARTED 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:40:54.760144+0000 mgr.y (mgr.25421) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:05.988785+0000 mgr.y (mgr.25421) 17 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:05.989266+0000 mgr.y (mgr.25421) 18 : cephadm 1 Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:44:02.747 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:06.154257+0000 mgr.y (mgr.25421) 19 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:06.204964+0000 mgr.y (mgr.25421) 20 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:06.319115+0000 mgr.y (mgr.25421) 21 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:06.369781+0000 mgr.y (mgr.25421) 22 : cephadm 1 Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:06.534574+0000 mgr.y (mgr.25421) 23 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:41:06.584631+0000 mgr.y (mgr.25421) 24 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:42:03.230024+0000 mgr.y (mgr.25421) 55 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:42:11.036775+0000 mgr.y (mgr.25421) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:42:13.606688+0000 mgr.y (mgr.25421) 62 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:13.618614+0000 mon.c (mon.2) 270 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:13.620171+0000 mon.a (mon.0) 47 : audit 0 from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:02.748 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:13.621477+0000 mon.c (mon.2) 271 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:42:13.622849+0000 mgr.y (mgr.25421) 63 : cephadm 1 Deploying daemon mon.c on smithi016 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:14.223255+0000 mon.c (mon.2) 272 : audit 0 from='client.? 172.21.15.16:0/1457947433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:16.351539+0000 mon.a (mon.0) 48 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:16.353776+0000 mon.a (mon.0) 49 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:16.354084+0000 mon.a (mon.0) 50 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:18.721886+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:24.791780+0000 mon.a (mon.0) 72 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:24.791828+0000 mon.a (mon.0) 73 : cluster 1 Cluster is now healthy 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:24.813995+0000 mon.a (mon.0) 74 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:24.826688+0000 mon.a (mon.0) 75 : cluster 0 mgrmap e37: y(active, since 95s), standbys: x 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:25.123940+0000 mgr.y (mgr.25421) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:25.136890+0000 mon.a (mon.0) 76 : audit 0 from='client.? 172.21.15.16:0/3422363337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:27.124569+0000 mgr.y (mgr.25421) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:29.125146+0000 mgr.y (mgr.25421) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:29.498558+0000 mgr.y (mgr.25421) 72 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:29.751636+0000 mon.a (mon.0) 77 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:29.759311+0000 mon.a (mon.0) 78 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:30.602913+0000 mon.a (mon.0) 79 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:30.608769+0000 mon.a (mon.0) 80 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:31.126073+0000 mgr.y (mgr.25421) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:33.126795+0000 mgr.y (mgr.25421) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:33.808234+0000 mgr.y (mgr.25421) 75 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:02.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:42:33.808918+0000 mgr.y (mgr.25421) 76 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:33.813436+0000 mon.a (mon.0) 81 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:34.194441+0000 mon.a (mon.0) 82 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:35.128006+0000 mgr.y (mgr.25421) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:37.128610+0000 mgr.y (mgr.25421) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:39.129207+0000 mgr.y (mgr.25421) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:41.130339+0000 mgr.y (mgr.25421) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.559421+0000 mon.a (mon.0) 83 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.566812+0000 mon.a (mon.0) 84 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.568680+0000 mon.a (mon.0) 85 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.570174+0000 mon.a (mon.0) 86 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:02.751 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.577608+0000 mon.a (mon.0) 87 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.658295+0000 mon.a (mon.0) 88 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.661125+0000 mon.a (mon.0) 89 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.662461+0000 mon.a (mon.0) 90 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:41.670736+0000 mon.a (mon.0) 91 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:43.131018+0000 mgr.y (mgr.25421) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:45.132081+0000 mgr.y (mgr.25421) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:47.132662+0000 mgr.y (mgr.25421) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:49.133322+0000 mgr.y (mgr.25421) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:42:49.194682+0000 mon.a (mon.0) 92 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:51.134431+0000 mgr.y (mgr.25421) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:53.135086+0000 mgr.y (mgr.25421) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:55.136125+0000 mgr.y (mgr.25421) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:57.136769+0000 mgr.y (mgr.25421) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:42:59.137426+0000 mgr.y (mgr.25421) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:01.138544+0000 mgr.y (mgr.25421) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:03.139210+0000 mgr.y (mgr.25421) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:04.195024+0000 mon.a (mon.0) 93 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:05.140333+0000 mgr.y (mgr.25421) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:07.141080+0000 mgr.y (mgr.25421) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:09.141772+0000 mgr.y (mgr.25421) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:11.142791+0000 mgr.y (mgr.25421) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:13.143368+0000 mgr.y (mgr.25421) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:15.144420+0000 mgr.y (mgr.25421) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:17.145062+0000 mgr.y (mgr.25421) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:19.145815+0000 mgr.y (mgr.25421) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:19.195516+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:21.146943+0000 mgr.y (mgr.25421) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:23.147568+0000 mgr.y (mgr.25421) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:25.148576+0000 mgr.y (mgr.25421) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:27.149230+0000 mgr.y (mgr.25421) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:29.149865+0000 mgr.y (mgr.25421) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:31.150901+0000 mgr.y (mgr.25421) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:33.151622+0000 mgr.y (mgr.25421) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.754 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:34.195807+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:35.152719+0000 mgr.y (mgr.25421) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:37.153459+0000 mgr.y (mgr.25421) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:39.154156+0000 mgr.y (mgr.25421) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:41.155311+0000 mgr.y (mgr.25421) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:41.753992+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:42.746990+0000 mgr.y (mgr.25421) 111 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:43:42.747590+0000 mgr.y (mgr.25421) 112 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:42.753123+0000 mon.a (mon.0) 97 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:43.156153+0000 mgr.y (mgr.25421) 113 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:45.157209+0000 mgr.y (mgr.25421) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:47.157643+0000 mgr.y (mgr.25421) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:47.996292+0000 mon.a (mon.0) 98 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.003729+0000 mon.a (mon.0) 99 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.143384+0000 mon.a (mon.0) 100 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.150872+0000 mon.a (mon.0) 101 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.627726+0000 mon.a (mon.0) 102 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.629094+0000 mon.a (mon.0) 103 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.638713+0000 mon.a (mon.0) 104 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.719812+0000 mon.a (mon.0) 105 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.724153+0000 mon.a (mon.0) 106 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.726424+0000 mon.a (mon.0) 107 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:48.727624+0000 mon.a (mon.0) 108 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T08:44:02.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:43:48.728545+0000 mgr.y (mgr.25421) 116 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:49.158022+0000 mgr.y (mgr.25421) 117 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:49.196187+0000 mon.a (mon.0) 109 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:51.019640+0000 mon.a (mon.0) 110 : audit 0 from='client.? 172.21.15.16:0/2234495327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:51.159175+0000 mgr.y (mgr.25421) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:43:51.249454+0000 mgr.y (mgr.25421) 119 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:51.256418+0000 mon.a (mon.0) 111 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:51.258303+0000 mon.a (mon.0) 112 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:51.259618+0000 mon.a (mon.0) 113 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:51.260740+0000 mon.a (mon.0) 114 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cephadm 2023-10-09T08:43:51.262069+0000 mgr.y (mgr.25421) 120 : cephadm 1 Deploying daemon mon.b on smithi094 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:53.159759+0000 mgr.y (mgr.25421) 121 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.757 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:53.467310+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.16:0/1552263051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:57.161504+0000 mgr.y (mgr.25421) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:57.217721+0000 mon.a (mon.0) 121 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:57.218167+0000 mon.a (mon.0) 122 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:57.218752+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:57.219101+0000 mon.a (mon.0) 124 : cluster 1 mon.a calling monitor election 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:43:57.220862+0000 mon.a (mon.0) 125 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:57.221080+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:43:59.162174+0000 mgr.y (mgr.25421) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:01.163203+0000 mgr.y (mgr.25421) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.220877+0000 mon.a (mon.0) 126 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:44:02.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.228048+0000 mon.a (mon.0) 127 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.248955+0000 mon.a (mon.0) 128 : cluster 0 fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.249015+0000 mon.a (mon.0) 129 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.249567+0000 mon.a (mon.0) 130 : cluster 0 mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.249891+0000 mon.a (mon.0) 131 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.258366+0000 mon.a (mon.0) 132 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.258406+0000 mon.a (mon.0) 133 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: cluster 2023-10-09T08:44:02.258424+0000 mon.a (mon.0) 134 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] is down (out of quorum) 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:44:02.268269+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:02 smithi094 bash[59677]: audit 2023-10-09T08:44:02.276065+0000 mon.a (mon.0) 136 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:23:18.622286+0000 mon.a (mon.0) 0 : 1 mkfs f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:38:54.223270+0000 mgr.x (mgr.24790) 46 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:38:54.224111+0000 mgr.x (mgr.24790) 47 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi094 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:38:56.192222+0000 mgr.x (mgr.24790) 49 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:38:56.215527+0000 mgr.x (mgr.24790) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi094 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:21.027458+0000 mgr.x (mgr.24790) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:21.028145+0000 mgr.x (mgr.24790) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:23.247533+0000 mgr.x (mgr.24790) 70 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:23.266588+0000 mgr.x (mgr.24790) 71 : cephadm 1 Deploying daemon mgr.y on smithi016 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:43.996320+0000 mgr.y (mgr.15261) 1 : cephadm 1 [09/Oct/2023:08:39:43] ENGINE Bus STARTING 2023-10-09T08:44:02.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:44.098807+0000 mgr.y (mgr.15261) 2 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:44.279552+0000 mgr.y (mgr.15261) 3 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:39:44.290367+0000 mgr.y (mgr.15261) 4 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Bus STARTED 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.023103+0000 mgr.y (mgr.15261) 14 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.023394+0000 mgr.y (mgr.15261) 15 : cephadm 1 Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.187916+0000 mgr.y (mgr.15261) 16 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.239122+0000 mgr.y (mgr.15261) 17 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.352424+0000 mgr.y (mgr.15261) 18 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.404481+0000 mgr.y (mgr.15261) 20 : cephadm 1 Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.517057+0000 mgr.y (mgr.15261) 21 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.568903+0000 mgr.y (mgr.15261) 22 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:01.831073+0000 mgr.y (mgr.15261) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:02.099144+0000 mgr.y (mgr.15261) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:10.527339+0000 mgr.y (mgr.15261) 30 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:10.533809+0000 mgr.y (mgr.15261) 31 : cephadm 1 Deploying daemon mgr.x on smithi094 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:31.042350+0000 mgr.y (mgr.15261) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:31.068083+0000 mgr.y (mgr.15261) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:33.589082+0000 mgr.y (mgr.15261) 46 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:33.604198+0000 mgr.y (mgr.15261) 47 : cephadm 1 Deploying daemon mon.a on smithi016 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:49.942190+0000 mgr.y (mgr.25421) 1 : cephadm 1 [09/Oct/2023:08:40:49] ENGINE Bus STARTING 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:50.063452+0000 mgr.y (mgr.25421) 2 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:50.165832+0000 mgr.y (mgr.25421) 5 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:50.166180+0000 mgr.y (mgr.25421) 6 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Bus STARTED 2023-10-09T08:44:02.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:40:54.760144+0000 mgr.y (mgr.25421) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:05.988785+0000 mgr.y (mgr.25421) 17 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:05.989266+0000 mgr.y (mgr.25421) 18 : cephadm 1 Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:06.154257+0000 mgr.y (mgr.25421) 19 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:06.204964+0000 mgr.y (mgr.25421) 20 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:06.319115+0000 mgr.y (mgr.25421) 21 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:06.369781+0000 mgr.y (mgr.25421) 22 : cephadm 1 Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:06.534574+0000 mgr.y (mgr.25421) 23 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:41:06.584631+0000 mgr.y (mgr.25421) 24 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:42:03.230024+0000 mgr.y (mgr.25421) 55 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:42:11.036775+0000 mgr.y (mgr.25421) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:42:13.606688+0000 mgr.y (mgr.25421) 62 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T08:44:02.764 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:13.618614+0000 mon.c (mon.2) 270 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:13.620171+0000 mon.a (mon.0) 47 : audit 0 from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:13.621477+0000 mon.c (mon.2) 271 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:42:13.622849+0000 mgr.y (mgr.25421) 63 : cephadm 1 Deploying daemon mon.c on smithi016 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:14.223255+0000 mon.c (mon.2) 272 : audit 0 from='client.? 172.21.15.16:0/1457947433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:16.351539+0000 mon.a (mon.0) 48 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:16.353776+0000 mon.a (mon.0) 49 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:16.354084+0000 mon.a (mon.0) 50 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:18.721886+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:24.791780+0000 mon.a (mon.0) 72 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:24.791828+0000 mon.a (mon.0) 73 : cluster 1 Cluster is now healthy 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:24.813995+0000 mon.a (mon.0) 74 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:02.765 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:24.826688+0000 mon.a (mon.0) 75 : cluster 0 mgrmap e37: y(active, since 95s), standbys: x 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:25.123940+0000 mgr.y (mgr.25421) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:25.136890+0000 mon.a (mon.0) 76 : audit 0 from='client.? 172.21.15.16:0/3422363337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:27.124569+0000 mgr.y (mgr.25421) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:29.125146+0000 mgr.y (mgr.25421) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:29.498558+0000 mgr.y (mgr.25421) 72 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:29.751636+0000 mon.a (mon.0) 77 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:29.759311+0000 mon.a (mon.0) 78 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:30.602913+0000 mon.a (mon.0) 79 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:30.608769+0000 mon.a (mon.0) 80 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:31.126073+0000 mgr.y (mgr.25421) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:02.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:33.126795+0000 mgr.y (mgr.25421) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:33.808234+0000 mgr.y (mgr.25421) 75 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:42:33.808918+0000 mgr.y (mgr.25421) 76 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:33.813436+0000 mon.a (mon.0) 81 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:34.194441+0000 mon.a (mon.0) 82 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:35.128006+0000 mgr.y (mgr.25421) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:37.128610+0000 mgr.y (mgr.25421) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:39.129207+0000 mgr.y (mgr.25421) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:41.130339+0000 mgr.y (mgr.25421) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.559421+0000 mon.a (mon.0) 83 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.566812+0000 mon.a (mon.0) 84 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.568680+0000 mon.a (mon.0) 85 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.570174+0000 mon.a (mon.0) 86 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.577608+0000 mon.a (mon.0) 87 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.658295+0000 mon.a (mon.0) 88 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.661125+0000 mon.a (mon.0) 89 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.662461+0000 mon.a (mon.0) 90 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:41.670736+0000 mon.a (mon.0) 91 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:43.131018+0000 mgr.y (mgr.25421) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:45.132081+0000 mgr.y (mgr.25421) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:47.132662+0000 mgr.y (mgr.25421) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:49.133322+0000 mgr.y (mgr.25421) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:42:49.194682+0000 mon.a (mon.0) 92 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:51.134431+0000 mgr.y (mgr.25421) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:53.135086+0000 mgr.y (mgr.25421) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:55.136125+0000 mgr.y (mgr.25421) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:57.136769+0000 mgr.y (mgr.25421) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:42:59.137426+0000 mgr.y (mgr.25421) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:01.138544+0000 mgr.y (mgr.25421) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:03.139210+0000 mgr.y (mgr.25421) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:04.195024+0000 mon.a (mon.0) 93 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:05.140333+0000 mgr.y (mgr.25421) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:07.141080+0000 mgr.y (mgr.25421) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:09.141772+0000 mgr.y (mgr.25421) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:11.142791+0000 mgr.y (mgr.25421) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:13.143368+0000 mgr.y (mgr.25421) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:15.144420+0000 mgr.y (mgr.25421) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:17.145062+0000 mgr.y (mgr.25421) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:19.145815+0000 mgr.y (mgr.25421) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:19.195516+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:21.146943+0000 mgr.y (mgr.25421) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:23.147568+0000 mgr.y (mgr.25421) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:25.148576+0000 mgr.y (mgr.25421) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:27.149230+0000 mgr.y (mgr.25421) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:29.149865+0000 mgr.y (mgr.25421) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:31.150901+0000 mgr.y (mgr.25421) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:33.151622+0000 mgr.y (mgr.25421) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:34.195807+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:35.152719+0000 mgr.y (mgr.25421) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:37.153459+0000 mgr.y (mgr.25421) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:39.154156+0000 mgr.y (mgr.25421) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:41.155311+0000 mgr.y (mgr.25421) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:41.753992+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:42.746990+0000 mgr.y (mgr.25421) 111 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:43:42.747590+0000 mgr.y (mgr.25421) 112 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:42.753123+0000 mon.a (mon.0) 97 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:43.156153+0000 mgr.y (mgr.25421) 113 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:45.157209+0000 mgr.y (mgr.25421) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:47.157643+0000 mgr.y (mgr.25421) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:47.996292+0000 mon.a (mon.0) 98 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.003729+0000 mon.a (mon.0) 99 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.143384+0000 mon.a (mon.0) 100 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.150872+0000 mon.a (mon.0) 101 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.627726+0000 mon.a (mon.0) 102 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.629094+0000 mon.a (mon.0) 103 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.638713+0000 mon.a (mon.0) 104 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.719812+0000 mon.a (mon.0) 105 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.724153+0000 mon.a (mon.0) 106 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:44:02.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.726424+0000 mon.a (mon.0) 107 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:48.727624+0000 mon.a (mon.0) 108 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:43:48.728545+0000 mgr.y (mgr.25421) 116 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:49.158022+0000 mgr.y (mgr.25421) 117 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:49.196187+0000 mon.a (mon.0) 109 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:51.019640+0000 mon.a (mon.0) 110 : audit 0 from='client.? 172.21.15.16:0/2234495327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:51.159175+0000 mgr.y (mgr.25421) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:43:51.249454+0000 mgr.y (mgr.25421) 119 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:51.256418+0000 mon.a (mon.0) 111 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:51.258303+0000 mon.a (mon.0) 112 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:51.259618+0000 mon.a (mon.0) 113 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:51.260740+0000 mon.a (mon.0) 114 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:02.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cephadm 2023-10-09T08:43:51.262069+0000 mgr.y (mgr.25421) 120 : cephadm 1 Deploying daemon mon.b on smithi094 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:53.159759+0000 mgr.y (mgr.25421) 121 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:53.467310+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.16:0/1552263051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:57.161504+0000 mgr.y (mgr.25421) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:57.217721+0000 mon.a (mon.0) 121 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:57.218167+0000 mon.a (mon.0) 122 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:57.218752+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:57.219101+0000 mon.a (mon.0) 124 : cluster 1 mon.a calling monitor election 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:43:57.220862+0000 mon.a (mon.0) 125 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:57.221080+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:43:59.162174+0000 mgr.y (mgr.25421) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:01.163203+0000 mgr.y (mgr.25421) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.220877+0000 mon.a (mon.0) 126 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.228048+0000 mon.a (mon.0) 127 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.248955+0000 mon.a (mon.0) 128 : cluster 0 fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.249015+0000 mon.a (mon.0) 129 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.249567+0000 mon.a (mon.0) 130 : cluster 0 mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.249891+0000 mon.a (mon.0) 131 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.258366+0000 mon.a (mon.0) 132 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.258406+0000 mon.a (mon.0) 133 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: cluster 2023-10-09T08:44:02.258424+0000 mon.a (mon.0) 134 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] is down (out of quorum) 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:44:02.268269+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:02 smithi016 bash[75179]: audit 2023-10-09T08:44:02.276065+0000 mon.a (mon.0) 136 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:02.841 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T08:44:03.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:03 smithi016 bash[75179]: audit 2023-10-09T08:44:02.307209+0000 mon.a (mon.0) 137 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:03.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:03 smithi016 bash[75179]: audit 2023-10-09T08:44:02.820281+0000 mon.a (mon.0) 138 : audit 0 from='client.? 172.21.15.16:0/3160936529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:03.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:03 smithi016 bash[75179]: cluster 2023-10-09T08:44:03.163856+0000 mgr.y (mgr.25421) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:03.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:03 smithi094 bash[59677]: audit 2023-10-09T08:44:02.307209+0000 mon.a (mon.0) 137 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:03.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:03 smithi094 bash[59677]: audit 2023-10-09T08:44:02.820281+0000 mon.a (mon.0) 138 : audit 0 from='client.? 172.21.15.16:0/3160936529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:03.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:03 smithi094 bash[59677]: cluster 2023-10-09T08:44:03.163856+0000 mgr.y (mgr.25421) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:03.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T08:44:04.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:23:18.622286+0000 mon.a (mon.0) 0 : 1 mkfs f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:44:04.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:38:54.223270+0000 mgr.x (mgr.24790) 46 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:38:54.224111+0000 mgr.x (mgr.24790) 47 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi094 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:38:56.192222+0000 mgr.x (mgr.24790) 49 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:38:56.215527+0000 mgr.x (mgr.24790) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi094 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:21.027458+0000 mgr.x (mgr.24790) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:21.028145+0000 mgr.x (mgr.24790) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:23.247533+0000 mgr.x (mgr.24790) 70 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:23.266588+0000 mgr.x (mgr.24790) 71 : cephadm 1 Deploying daemon mgr.y on smithi016 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:43.996320+0000 mgr.y (mgr.15261) 1 : cephadm 1 [09/Oct/2023:08:39:43] ENGINE Bus STARTING 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:44.098807+0000 mgr.y (mgr.15261) 2 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:44.279552+0000 mgr.y (mgr.15261) 3 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:44:04.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:39:44.290367+0000 mgr.y (mgr.15261) 4 : cephadm 1 [09/Oct/2023:08:39:44] ENGINE Bus STARTED 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.023103+0000 mgr.y (mgr.15261) 14 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.023394+0000 mgr.y (mgr.15261) 15 : cephadm 1 Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.187916+0000 mgr.y (mgr.15261) 16 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.239122+0000 mgr.y (mgr.15261) 17 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.352424+0000 mgr.y (mgr.15261) 18 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.404481+0000 mgr.y (mgr.15261) 20 : cephadm 1 Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.517057+0000 mgr.y (mgr.15261) 21 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.568903+0000 mgr.y (mgr.15261) 22 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:01.831073+0000 mgr.y (mgr.15261) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:02.099144+0000 mgr.y (mgr.15261) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi094 2023-10-09T08:44:04.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:10.527339+0000 mgr.y (mgr.15261) 30 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:10.533809+0000 mgr.y (mgr.15261) 31 : cephadm 1 Deploying daemon mgr.x on smithi094 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:31.042350+0000 mgr.y (mgr.15261) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:31.068083+0000 mgr.y (mgr.15261) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:33.589082+0000 mgr.y (mgr.15261) 46 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:33.604198+0000 mgr.y (mgr.15261) 47 : cephadm 1 Deploying daemon mon.a on smithi016 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:49.942190+0000 mgr.y (mgr.25421) 1 : cephadm 1 [09/Oct/2023:08:40:49] ENGINE Bus STARTING 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:50.063452+0000 mgr.y (mgr.25421) 2 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Serving on https://172.21.15.16:7150 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:50.165832+0000 mgr.y (mgr.25421) 5 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Serving on http://172.21.15.16:8765 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:50.166180+0000 mgr.y (mgr.25421) 6 : cephadm 1 [09/Oct/2023:08:40:50] ENGINE Bus STARTED 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:40:54.760144+0000 mgr.y (mgr.25421) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:05.988785+0000 mgr.y (mgr.25421) 17 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-10-09T08:44:04.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:05.989266+0000 mgr.y (mgr.25421) 18 : cephadm 1 Updating smithi094:/etc/ceph/ceph.conf 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:06.154257+0000 mgr.y (mgr.25421) 19 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:06.204964+0000 mgr.y (mgr.25421) 20 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:06.319115+0000 mgr.y (mgr.25421) 21 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:06.369781+0000 mgr.y (mgr.25421) 22 : cephadm 1 Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:06.534574+0000 mgr.y (mgr.25421) 23 : cephadm 1 Updating smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:41:06.584631+0000 mgr.y (mgr.25421) 24 : cephadm 1 Updating smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:42:03.230024+0000 mgr.y (mgr.25421) 55 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:42:11.036775+0000 mgr.y (mgr.25421) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:42:13.606688+0000 mgr.y (mgr.25421) 62 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T08:44:04.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:13.618614+0000 mon.c (mon.2) 270 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:04.259 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:13.620171+0000 mon.a (mon.0) 47 : audit 0 from='mgr.25421 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:04.259 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:13.621477+0000 mon.c (mon.2) 271 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:04.259 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:42:13.622849+0000 mgr.y (mgr.25421) 63 : cephadm 1 Deploying daemon mon.c on smithi016 2023-10-09T08:44:04.259 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:14.223255+0000 mon.c (mon.2) 272 : audit 0 from='client.? 172.21.15.16:0/1457947433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:04.259 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:16.351539+0000 mon.a (mon.0) 48 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:44:04.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:16.353776+0000 mon.a (mon.0) 49 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:44:04.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:16.354084+0000 mon.a (mon.0) 50 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:44:04.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:18.721886+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T08:44:04.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:24.791780+0000 mon.a (mon.0) 72 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:44:04.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:24.791828+0000 mon.a (mon.0) 73 : cluster 1 Cluster is now healthy 2023-10-09T08:44:04.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:24.813995+0000 mon.a (mon.0) 74 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:04.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:24.826688+0000 mon.a (mon.0) 75 : cluster 0 mgrmap e37: y(active, since 95s), standbys: x 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:25.123940+0000 mgr.y (mgr.25421) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:25.136890+0000 mon.a (mon.0) 76 : audit 0 from='client.? 172.21.15.16:0/3422363337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:27.124569+0000 mgr.y (mgr.25421) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:29.125146+0000 mgr.y (mgr.25421) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:29.498558+0000 mgr.y (mgr.25421) 72 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:29.751636+0000 mon.a (mon.0) 77 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:29.759311+0000 mon.a (mon.0) 78 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:30.602913+0000 mon.a (mon.0) 79 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:30.608769+0000 mon.a (mon.0) 80 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:31.126073+0000 mgr.y (mgr.25421) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T08:44:04.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:33.126795+0000 mgr.y (mgr.25421) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:33.808234+0000 mgr.y (mgr.25421) 75 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:42:33.808918+0000 mgr.y (mgr.25421) 76 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:33.813436+0000 mon.a (mon.0) 81 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:34.194441+0000 mon.a (mon.0) 82 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:35.128006+0000 mgr.y (mgr.25421) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:37.128610+0000 mgr.y (mgr.25421) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:39.129207+0000 mgr.y (mgr.25421) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:41.130339+0000 mgr.y (mgr.25421) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.559421+0000 mon.a (mon.0) 83 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.566812+0000 mon.a (mon.0) 84 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.568680+0000 mon.a (mon.0) 85 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.570174+0000 mon.a (mon.0) 86 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.577608+0000 mon.a (mon.0) 87 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.658295+0000 mon.a (mon.0) 88 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.661125+0000 mon.a (mon.0) 89 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.662461+0000 mon.a (mon.0) 90 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:41.670736+0000 mon.a (mon.0) 91 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:43.131018+0000 mgr.y (mgr.25421) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:45.132081+0000 mgr.y (mgr.25421) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:47.132662+0000 mgr.y (mgr.25421) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:49.133322+0000 mgr.y (mgr.25421) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:42:49.194682+0000 mon.a (mon.0) 92 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:04.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:51.134431+0000 mgr.y (mgr.25421) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:53.135086+0000 mgr.y (mgr.25421) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:55.136125+0000 mgr.y (mgr.25421) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:57.136769+0000 mgr.y (mgr.25421) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:42:59.137426+0000 mgr.y (mgr.25421) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:01.138544+0000 mgr.y (mgr.25421) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:03.139210+0000 mgr.y (mgr.25421) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:04.195024+0000 mon.a (mon.0) 93 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:05.140333+0000 mgr.y (mgr.25421) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:07.141080+0000 mgr.y (mgr.25421) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:09.141772+0000 mgr.y (mgr.25421) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:11.142791+0000 mgr.y (mgr.25421) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:13.143368+0000 mgr.y (mgr.25421) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:15.144420+0000 mgr.y (mgr.25421) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:17.145062+0000 mgr.y (mgr.25421) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:19.145815+0000 mgr.y (mgr.25421) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:19.195516+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:21.146943+0000 mgr.y (mgr.25421) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:23.147568+0000 mgr.y (mgr.25421) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:25.148576+0000 mgr.y (mgr.25421) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:27.149230+0000 mgr.y (mgr.25421) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:29.149865+0000 mgr.y (mgr.25421) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:31.150901+0000 mgr.y (mgr.25421) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:33.151622+0000 mgr.y (mgr.25421) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:34.195807+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:35.152719+0000 mgr.y (mgr.25421) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:37.153459+0000 mgr.y (mgr.25421) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:39.154156+0000 mgr.y (mgr.25421) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:41.155311+0000 mgr.y (mgr.25421) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:41.753992+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:42.746990+0000 mgr.y (mgr.25421) 111 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:43:42.747590+0000 mgr.y (mgr.25421) 112 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:42.753123+0000 mon.a (mon.0) 97 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:43.156153+0000 mgr.y (mgr.25421) 113 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:45.157209+0000 mgr.y (mgr.25421) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:47.157643+0000 mgr.y (mgr.25421) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:47.996292+0000 mon.a (mon.0) 98 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.003729+0000 mon.a (mon.0) 99 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.143384+0000 mon.a (mon.0) 100 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.150872+0000 mon.a (mon.0) 101 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.627726+0000 mon.a (mon.0) 102 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.629094+0000 mon.a (mon.0) 103 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.638713+0000 mon.a (mon.0) 104 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.719812+0000 mon.a (mon.0) 105 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.724153+0000 mon.a (mon.0) 106 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.726424+0000 mon.a (mon.0) 107 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T08:44:04.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:48.727624+0000 mon.a (mon.0) 108 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:43:48.728545+0000 mgr.y (mgr.25421) 116 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:49.158022+0000 mgr.y (mgr.25421) 117 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:49.196187+0000 mon.a (mon.0) 109 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:51.019640+0000 mon.a (mon.0) 110 : audit 0 from='client.? 172.21.15.16:0/2234495327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:51.159175+0000 mgr.y (mgr.25421) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:43:51.249454+0000 mgr.y (mgr.25421) 119 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:51.256418+0000 mon.a (mon.0) 111 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:51.258303+0000 mon.a (mon.0) 112 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:51.259618+0000 mon.a (mon.0) 113 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:51.260740+0000 mon.a (mon.0) 114 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:04.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cephadm 2023-10-09T08:43:51.262069+0000 mgr.y (mgr.25421) 120 : cephadm 1 Deploying daemon mon.b on smithi094 2023-10-09T08:44:04.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:53.159759+0000 mgr.y (mgr.25421) 121 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:53.467310+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.16:0/1552263051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:04.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:57.161504+0000 mgr.y (mgr.25421) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:57.217721+0000 mon.a (mon.0) 121 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:04.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:57.218167+0000 mon.a (mon.0) 122 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:44:04.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:57.218752+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:44:04.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:57.219101+0000 mon.a (mon.0) 124 : cluster 1 mon.a calling monitor election 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:43:57.220862+0000 mon.a (mon.0) 125 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:57.221080+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:59.162174+0000 mgr.y (mgr.25421) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:01.163203+0000 mgr.y (mgr.25421) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.220877+0000 mon.a (mon.0) 126 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.228048+0000 mon.a (mon.0) 127 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.248955+0000 mon.a (mon.0) 128 : cluster 0 fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.249015+0000 mon.a (mon.0) 129 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.249567+0000 mon.a (mon.0) 130 : cluster 0 mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.249891+0000 mon.a (mon.0) 131 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T08:44:04.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.258366+0000 mon.a (mon.0) 132 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T08:44:04.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.258406+0000 mon.a (mon.0) 133 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T08:44:04.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:02.258424+0000 mon.a (mon.0) 134 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] is down (out of quorum) 2023-10-09T08:44:04.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:44:02.268269+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:44:02.276065+0000 mon.a (mon.0) 136 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:04.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:44:02.307209+0000 mon.a (mon.0) 137 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:04.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:44:02.820281+0000 mon.a (mon.0) 138 : audit 0 from='client.? 172.21.15.16:0/3160936529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T08:44:04.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:03.163856+0000 mgr.y (mgr.25421) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:05.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:43:57.220589+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T08:44:05.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.158601+0000 mon.a (mon.0) 139 : cluster 1 mon.a calling monitor election 2023-10-09T08:44:05.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.161096+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T08:44:05.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.180680+0000 mon.a (mon.0) 140 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:44:05.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.187552+0000 mon.a (mon.0) 141 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:44:05.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.204590+0000 mon.a (mon.0) 142 : cluster 0 fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:44:05.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.204653+0000 mon.a (mon.0) 143 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.205352+0000 mon.a (mon.0) 144 : cluster 0 mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.205653+0000 mon.a (mon.0) 145 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.205685+0000 mon.a (mon.0) 146 : cluster 1 Cluster is now healthy 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: cluster 2023-10-09T08:44:04.228658+0000 mon.a (mon.0) 147 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: audit 2023-10-09T08:44:04.239638+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[75179]: audit 2023-10-09T08:44:04.240603+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:43:57.220589+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.158601+0000 mon.a (mon.0) 139 : cluster 1 mon.a calling monitor election 2023-10-09T08:44:05.128 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.161096+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.180680+0000 mon.a (mon.0) 140 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.187552+0000 mon.a (mon.0) 141 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.204590+0000 mon.a (mon.0) 142 : cluster 0 fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.204653+0000 mon.a (mon.0) 143 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.205352+0000 mon.a (mon.0) 144 : cluster 0 mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.205653+0000 mon.a (mon.0) 145 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.205685+0000 mon.a (mon.0) 146 : cluster 1 Cluster is now healthy 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: cluster 2023-10-09T08:44:04.228658+0000 mon.a (mon.0) 147 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:44:04.239638+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:05.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:04 smithi016 bash[77892]: audit 2023-10-09T08:44:04.240603+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:43:57.220589+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T08:44:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.158601+0000 mon.a (mon.0) 139 : cluster 1 mon.a calling monitor election 2023-10-09T08:44:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.161096+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T08:44:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.180680+0000 mon.a (mon.0) 140 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:44:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.187552+0000 mon.a (mon.0) 141 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-10-09T08:44:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.204590+0000 mon.a (mon.0) 142 : cluster 0 fsmap foo:1 {0=foo.smithi016.ccdktz=up:active} 1 up:standby 2023-10-09T08:44:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.204653+0000 mon.a (mon.0) 143 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2023-10-09T08:44:05.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.205352+0000 mon.a (mon.0) 144 : cluster 0 mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T08:44:05.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.205653+0000 mon.a (mon.0) 145 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:44:05.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.205685+0000 mon.a (mon.0) 146 : cluster 1 Cluster is now healthy 2023-10-09T08:44:05.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: cluster 2023-10-09T08:44:04.228658+0000 mon.a (mon.0) 147 : cluster 1 overall HEALTH_OK 2023-10-09T08:44:05.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: audit 2023-10-09T08:44:04.239638+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:05.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:04 smithi094 bash[59677]: audit 2023-10-09T08:44:04.240603+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:06.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:05 smithi016 bash[75179]: cluster 2023-10-09T08:44:05.164914+0000 mgr.y (mgr.25421) 127 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:06.004 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:44:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:44:05] "GET /metrics HTTP/1.1" 200 33762 "" "Prometheus/2.33.4" 2023-10-09T08:44:06.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:05 smithi016 bash[77892]: cluster 2023-10-09T08:44:05.164914+0000 mgr.y (mgr.25421) 127 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:05 smithi094 bash[59677]: cluster 2023-10-09T08:44:05.164914+0000 mgr.y (mgr.25421) 127 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:07.675 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T08:44:07.675 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (5m) - 11m 13.2M - 0.23.0 ba2b418f427c b9989d9bd43e 2023-10-09T08:44:07.675 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi094 *:3000 running (4m) - 10m 41.6M - 8.3.5 dad864ee21e9 f1a12efb3a56 2023-10-09T08:44:07.675 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.ccdktz smithi016 running (9m) - 9m 16.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 17a3e2c7a98b 2023-10-09T08:44:07.675 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi094.zfssci smithi094 running (9m) - 9m 13.8M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a917ff5f77b 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi094 *:8443,9283,8765 running (3m) - 18m 427M - 18.2.0-598-gb4fa663f 3060897fdf97 f0fcc951b93f 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (4m) - 20m 475M - 18.2.0-598-gb4fa663f 3060897fdf97 cdf79c37f30d 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (3m) - 20m 57.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 a8c0ba734014 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi094 starting - - - 2048M 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (110s) - 19m 35.4M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 320971806e89 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (5m) - 11m 8964k - 1.3.1 1dbe0e931976 6c4a45c1d04a 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi094 *:9100 running (5m) - 11m 9279k - 1.3.1 1dbe0e931976 59bcf67d7eb5 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (18m) - 18m 46.6M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c2ec86d01448 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (17m) - 17m 43.9M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 302ee697c26f 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (16m) - 16m 52.1M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c358a7abf935 2023-10-09T08:44:07.676 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (15m) - 15m 43.1M 2525M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0f6ebb2bd802 2023-10-09T08:44:07.677 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi094 running (15m) - 15m 45.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1a20cacc6df3 2023-10-09T08:44:07.677 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi094 running (14m) - 14m 49.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 381d98a4283c 2023-10-09T08:44:07.677 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi094 running (13m) - 13m 42.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d8307333e14e 2023-10-09T08:44:07.677 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi094 running (12m) - 12m 53.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 04152ef43de4 2023-10-09T08:44:07.677 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi094 *:9095 running (3m) - 11m 55.8M - 2.33.4 514e6a882f6e 4aee8e4f263b 2023-10-09T08:44:07.981 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:07 smithi016 bash[75179]: cluster 2023-10-09T08:44:07.165508+0000 mgr.y (mgr.25421) 128 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:07.982 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:07 smithi016 bash[77892]: cluster 2023-10-09T08:44:07.165508+0000 mgr.y (mgr.25421) 128 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:07 smithi094 bash[59677]: cluster 2023-10-09T08:44:07.165508+0000 mgr.y (mgr.25421) 128 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:08.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T08:44:09.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:08 smithi016 bash[75179]: audit 2023-10-09T08:44:07.652244+0000 mgr.y (mgr.25421) 129 : audit 0 from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:09.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:08 smithi016 bash[75179]: audit 2023-10-09T08:44:08.555051+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:09.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:08 smithi016 bash[75179]: audit 2023-10-09T08:44:08.564478+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:09.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:08 smithi016 bash[77892]: audit 2023-10-09T08:44:07.652244+0000 mgr.y (mgr.25421) 129 : audit 0 from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:09.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:08 smithi016 bash[77892]: audit 2023-10-09T08:44:08.555051+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:09.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:08 smithi016 bash[77892]: audit 2023-10-09T08:44:08.564478+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:09.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:08 smithi094 bash[59677]: audit 2023-10-09T08:44:07.652244+0000 mgr.y (mgr.25421) 129 : audit 0 from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:08 smithi094 bash[59677]: audit 2023-10-09T08:44:08.555051+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:08 smithi094 bash[59677]: audit 2023-10-09T08:44:08.564478+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:09.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:09 smithi016 bash[75179]: cluster 2023-10-09T08:44:09.165923+0000 mgr.y (mgr.25421) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:10.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:09 smithi016 bash[75179]: audit 2023-10-09T08:44:09.357128+0000 mon.a (mon.0) 152 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:10.007 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:09 smithi016 bash[75179]: audit 2023-10-09T08:44:09.363734+0000 mon.a (mon.0) 153 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:10.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:09 smithi016 bash[77892]: cluster 2023-10-09T08:44:09.165923+0000 mgr.y (mgr.25421) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:10.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:09 smithi016 bash[77892]: audit 2023-10-09T08:44:09.357128+0000 mon.a (mon.0) 152 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:10.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:09 smithi016 bash[77892]: audit 2023-10-09T08:44:09.363734+0000 mon.a (mon.0) 153 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:10.160 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:09 smithi094 bash[59677]: cluster 2023-10-09T08:44:09.165923+0000 mgr.y (mgr.25421) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:10.160 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:09 smithi094 bash[59677]: audit 2023-10-09T08:44:09.357128+0000 mon.a (mon.0) 152 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:10.161 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:09 smithi094 bash[59677]: audit 2023-10-09T08:44:09.363734+0000 mon.a (mon.0) 153 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:12.011 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:12.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:12 smithi094 bash[59677]: cluster 2023-10-09T08:44:11.166957+0000 mgr.y (mgr.25421) 131 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:12.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:12 smithi094 bash[59677]: audit 2023-10-09T08:44:12.006290+0000 mon.a (mon.0) 154 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:12.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:12 smithi016 bash[75179]: cluster 2023-10-09T08:44:11.166957+0000 mgr.y (mgr.25421) 131 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:12.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:12 smithi016 bash[75179]: audit 2023-10-09T08:44:12.006290+0000 mon.a (mon.0) 154 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:12.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:12 smithi016 bash[77892]: cluster 2023-10-09T08:44:11.166957+0000 mgr.y (mgr.25421) 131 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:12.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:12 smithi016 bash[77892]: audit 2023-10-09T08:44:12.006290+0000 mon.a (mon.0) 154 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:12.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T08:44:13.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:13 smithi016 bash[77892]: audit 2023-10-09T08:44:12.001294+0000 mgr.y (mgr.25421) 132 : audit 0 from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:13.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:13 smithi016 bash[77892]: cephadm 2023-10-09T08:44:12.001696+0000 mgr.y (mgr.25421) 133 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:13.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:13 smithi094 bash[59677]: audit 2023-10-09T08:44:12.001294+0000 mgr.y (mgr.25421) 132 : audit 0 from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:13.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:13 smithi094 bash[59677]: cephadm 2023-10-09T08:44:12.001696+0000 mgr.y (mgr.25421) 133 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:13.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:13 smithi016 bash[75179]: audit 2023-10-09T08:44:12.001294+0000 mgr.y (mgr.25421) 132 : audit 0 from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:44:13.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:13 smithi016 bash[75179]: cephadm 2023-10-09T08:44:12.001696+0000 mgr.y (mgr.25421) 133 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:44:14.291 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:14 smithi016 bash[75179]: cluster 2023-10-09T08:44:13.167624+0000 mgr.y (mgr.25421) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:14.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:14 smithi016 bash[77892]: cluster 2023-10-09T08:44:13.167624+0000 mgr.y (mgr.25421) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:14 smithi094 bash[59677]: cluster 2023-10-09T08:44:13.167624+0000 mgr.y (mgr.25421) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:15.994 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:44:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:44:15] "GET /metrics HTTP/1.1" 200 33762 "" "Prometheus/2.33.4" 2023-10-09T08:44:16.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:16 smithi016 bash[75179]: cluster 2023-10-09T08:44:15.168735+0000 mgr.y (mgr.25421) 135 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:16.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:16 smithi016 bash[77892]: cluster 2023-10-09T08:44:15.168735+0000 mgr.y (mgr.25421) 135 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:16.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:16 smithi094 bash[59677]: cluster 2023-10-09T08:44:15.168735+0000 mgr.y (mgr.25421) 135 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:18.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:18 smithi016 bash[75179]: cluster 2023-10-09T08:44:17.169362+0000 mgr.y (mgr.25421) 136 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:18.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:18 smithi016 bash[77892]: cluster 2023-10-09T08:44:17.169362+0000 mgr.y (mgr.25421) 136 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:18 smithi094 bash[59677]: cluster 2023-10-09T08:44:17.169362+0000 mgr.y (mgr.25421) 136 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.064831+0000 mon.a (mon.0) 155 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.073186+0000 mon.a (mon.0) 156 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.075015+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.076622+0000 mon.a (mon.0) 158 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.084032+0000 mon.a (mon.0) 159 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: cephadm 2023-10-09T08:44:19.154867+0000 mgr.y (mgr.25421) 137 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T08:44:20.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.155524+0000 mon.a (mon.0) 160 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:20.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.157380+0000 mon.a (mon.0) 161 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:20.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.159067+0000 mon.a (mon.0) 162 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:20.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: cephadm 2023-10-09T08:44:19.160880+0000 mgr.y (mgr.25421) 138 : cephadm 1 Reconfiguring daemon mon.a on smithi016 2023-10-09T08:44:20.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: cluster 2023-10-09T08:44:19.169880+0000 mgr.y (mgr.25421) 139 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:20.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.202266+0000 mon.a (mon.0) 163 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:20 smithi094 bash[59677]: audit 2023-10-09T08:44:19.204368+0000 mon.a (mon.0) 164 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:20.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.064831+0000 mon.a (mon.0) 155 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.073186+0000 mon.a (mon.0) 156 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.075015+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:20.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.076622+0000 mon.a (mon.0) 158 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:20.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.084032+0000 mon.a (mon.0) 159 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: cephadm 2023-10-09T08:44:19.154867+0000 mgr.y (mgr.25421) 137 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.155524+0000 mon.a (mon.0) 160 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.157380+0000 mon.a (mon.0) 161 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.159067+0000 mon.a (mon.0) 162 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: cephadm 2023-10-09T08:44:19.160880+0000 mgr.y (mgr.25421) 138 : cephadm 1 Reconfiguring daemon mon.a on smithi016 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: cluster 2023-10-09T08:44:19.169880+0000 mgr.y (mgr.25421) 139 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.202266+0000 mon.a (mon.0) 163 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[75179]: audit 2023-10-09T08:44:19.204368+0000 mon.a (mon.0) 164 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:20.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.064831+0000 mon.a (mon.0) 155 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.073186+0000 mon.a (mon.0) 156 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.075015+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.076622+0000 mon.a (mon.0) 158 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.084032+0000 mon.a (mon.0) 159 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: cephadm 2023-10-09T08:44:19.154867+0000 mgr.y (mgr.25421) 137 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.155524+0000 mon.a (mon.0) 160 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.157380+0000 mon.a (mon.0) 161 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.159067+0000 mon.a (mon.0) 162 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: cephadm 2023-10-09T08:44:19.160880+0000 mgr.y (mgr.25421) 138 : cephadm 1 Reconfiguring daemon mon.a on smithi016 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: cluster 2023-10-09T08:44:19.169880+0000 mgr.y (mgr.25421) 139 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:20.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.202266+0000 mon.a (mon.0) 163 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:20.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:20 smithi016 bash[77892]: audit 2023-10-09T08:44:19.204368+0000 mon.a (mon.0) 164 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:22.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:22 smithi094 bash[59677]: cluster 2023-10-09T08:44:21.170904+0000 mgr.y (mgr.25421) 140 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:22.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:22 smithi094 bash[59677]: audit 2023-10-09T08:44:21.625289+0000 mon.a (mon.0) 165 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:22.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:22 smithi094 bash[59677]: audit 2023-10-09T08:44:21.633227+0000 mon.a (mon.0) 166 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:22.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:22 smithi094 bash[59677]: audit 2023-10-09T08:44:21.639275+0000 mon.a (mon.0) 167 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:44:22.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:22 smithi094 bash[59677]: audit 2023-10-09T08:44:21.640667+0000 mon.a (mon.0) 168 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:44:22.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:22 smithi094 bash[59677]: audit 2023-10-09T08:44:21.642018+0000 mon.a (mon.0) 169 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:22.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[77892]: cluster 2023-10-09T08:44:21.170904+0000 mgr.y (mgr.25421) 140 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[77892]: audit 2023-10-09T08:44:21.625289+0000 mon.a (mon.0) 165 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[77892]: audit 2023-10-09T08:44:21.633227+0000 mon.a (mon.0) 166 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[77892]: audit 2023-10-09T08:44:21.639275+0000 mon.a (mon.0) 167 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[77892]: audit 2023-10-09T08:44:21.640667+0000 mon.a (mon.0) 168 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[77892]: audit 2023-10-09T08:44:21.642018+0000 mon.a (mon.0) 169 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[75179]: cluster 2023-10-09T08:44:21.170904+0000 mgr.y (mgr.25421) 140 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[75179]: audit 2023-10-09T08:44:21.625289+0000 mon.a (mon.0) 165 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[75179]: audit 2023-10-09T08:44:21.633227+0000 mon.a (mon.0) 166 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:22.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[75179]: audit 2023-10-09T08:44:21.639275+0000 mon.a (mon.0) 167 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:44:22.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[75179]: audit 2023-10-09T08:44:21.640667+0000 mon.a (mon.0) 168 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:44:22.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:22 smithi016 bash[75179]: audit 2023-10-09T08:44:21.642018+0000 mon.a (mon.0) 169 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:23 smithi094 bash[59677]: cephadm 2023-10-09T08:44:21.638691+0000 mgr.y (mgr.25421) 141 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T08:44:23.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:23 smithi094 bash[59677]: cephadm 2023-10-09T08:44:21.643449+0000 mgr.y (mgr.25421) 142 : cephadm 1 Reconfiguring daemon mgr.y on smithi016 2023-10-09T08:44:23.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:23 smithi016 bash[75179]: cephadm 2023-10-09T08:44:21.638691+0000 mgr.y (mgr.25421) 141 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T08:44:23.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:23 smithi016 bash[75179]: cephadm 2023-10-09T08:44:21.643449+0000 mgr.y (mgr.25421) 142 : cephadm 1 Reconfiguring daemon mgr.y on smithi016 2023-10-09T08:44:23.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:23 smithi016 bash[77892]: cephadm 2023-10-09T08:44:21.638691+0000 mgr.y (mgr.25421) 141 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T08:44:23.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:23 smithi016 bash[77892]: cephadm 2023-10-09T08:44:21.643449+0000 mgr.y (mgr.25421) 142 : cephadm 1 Reconfiguring daemon mgr.y on smithi016 2023-10-09T08:44:24.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:24 smithi094 bash[59677]: cluster 2023-10-09T08:44:23.171514+0000 mgr.y (mgr.25421) 143 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:24.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:24 smithi094 bash[59677]: audit 2023-10-09T08:44:23.926429+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:24.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:24 smithi094 bash[59677]: audit 2023-10-09T08:44:23.935217+0000 mon.a (mon.0) 171 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:24.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:24 smithi094 bash[59677]: audit 2023-10-09T08:44:23.964191+0000 mon.a (mon.0) 172 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:24.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:24 smithi094 bash[59677]: audit 2023-10-09T08:44:23.966012+0000 mon.a (mon.0) 173 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:24.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:24 smithi094 bash[59677]: audit 2023-10-09T08:44:23.967668+0000 mon.a (mon.0) 174 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:24.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[77892]: cluster 2023-10-09T08:44:23.171514+0000 mgr.y (mgr.25421) 143 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:24.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[77892]: audit 2023-10-09T08:44:23.926429+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[77892]: audit 2023-10-09T08:44:23.935217+0000 mon.a (mon.0) 171 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[77892]: audit 2023-10-09T08:44:23.964191+0000 mon.a (mon.0) 172 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[77892]: audit 2023-10-09T08:44:23.966012+0000 mon.a (mon.0) 173 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[77892]: audit 2023-10-09T08:44:23.967668+0000 mon.a (mon.0) 174 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[75179]: cluster 2023-10-09T08:44:23.171514+0000 mgr.y (mgr.25421) 143 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[75179]: audit 2023-10-09T08:44:23.926429+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[75179]: audit 2023-10-09T08:44:23.935217+0000 mon.a (mon.0) 171 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[75179]: audit 2023-10-09T08:44:23.964191+0000 mon.a (mon.0) 172 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:24.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[75179]: audit 2023-10-09T08:44:23.966012+0000 mon.a (mon.0) 173 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:24.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:24 smithi016 bash[75179]: audit 2023-10-09T08:44:23.967668+0000 mon.a (mon.0) 174 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:25.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:25 smithi016 bash[75179]: cephadm 2023-10-09T08:44:23.963517+0000 mgr.y (mgr.25421) 144 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T08:44:25.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:25 smithi016 bash[75179]: cephadm 2023-10-09T08:44:23.969562+0000 mgr.y (mgr.25421) 145 : cephadm 1 Reconfiguring daemon mon.c on smithi016 2023-10-09T08:44:25.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:25 smithi016 bash[77892]: cephadm 2023-10-09T08:44:23.963517+0000 mgr.y (mgr.25421) 144 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T08:44:25.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:25 smithi016 bash[77892]: cephadm 2023-10-09T08:44:23.969562+0000 mgr.y (mgr.25421) 145 : cephadm 1 Reconfiguring daemon mon.c on smithi016 2023-10-09T08:44:25.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:25 smithi094 bash[59677]: cephadm 2023-10-09T08:44:23.963517+0000 mgr.y (mgr.25421) 144 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T08:44:25.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:25 smithi094 bash[59677]: cephadm 2023-10-09T08:44:23.969562+0000 mgr.y (mgr.25421) 145 : cephadm 1 Reconfiguring daemon mon.c on smithi016 2023-10-09T08:44:26.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:44:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:44:25] "GET /metrics HTTP/1.1" 200 33763 "" "Prometheus/2.33.4" 2023-10-09T08:44:26.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:26 smithi016 bash[75179]: cluster 2023-10-09T08:44:25.172185+0000 mgr.y (mgr.25421) 146 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:26.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:26 smithi016 bash[75179]: audit 2023-10-09T08:44:26.235858+0000 mon.a (mon.0) 175 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:26.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:26 smithi016 bash[77892]: cluster 2023-10-09T08:44:25.172185+0000 mgr.y (mgr.25421) 146 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:26.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:26 smithi016 bash[77892]: audit 2023-10-09T08:44:26.235858+0000 mon.a (mon.0) 175 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:26.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:26 smithi094 bash[59677]: cluster 2023-10-09T08:44:25.172185+0000 mgr.y (mgr.25421) 146 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:26.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:26 smithi094 bash[59677]: audit 2023-10-09T08:44:26.235858+0000 mon.a (mon.0) 175 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:27.550 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:27 smithi016 bash[75179]: audit 2023-10-09T08:44:26.244640+0000 mon.a (mon.0) 176 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:27.550 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:27 smithi016 bash[75179]: audit 2023-10-09T08:44:26.290830+0000 mon.a (mon.0) 177 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:44:27.550 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:27 smithi016 bash[75179]: audit 2023-10-09T08:44:26.292631+0000 mon.a (mon.0) 178 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:27.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:27 smithi016 bash[77892]: audit 2023-10-09T08:44:26.244640+0000 mon.a (mon.0) 176 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:27.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:27 smithi016 bash[77892]: audit 2023-10-09T08:44:26.290830+0000 mon.a (mon.0) 177 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:44:27.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:27 smithi016 bash[77892]: audit 2023-10-09T08:44:26.292631+0000 mon.a (mon.0) 178 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:27 smithi094 bash[59677]: audit 2023-10-09T08:44:26.244640+0000 mon.a (mon.0) 176 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:27 smithi094 bash[59677]: audit 2023-10-09T08:44:26.290830+0000 mon.a (mon.0) 177 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:44:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:27 smithi094 bash[59677]: audit 2023-10-09T08:44:26.292631+0000 mon.a (mon.0) 178 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:28.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:28 smithi094 bash[59677]: cephadm 2023-10-09T08:44:26.290177+0000 mgr.y (mgr.25421) 147 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T08:44:28.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:28 smithi094 bash[59677]: cephadm 2023-10-09T08:44:26.297538+0000 mgr.y (mgr.25421) 148 : cephadm 1 Reconfiguring daemon osd.0 on smithi016 2023-10-09T08:44:28.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:28 smithi094 bash[59677]: cluster 2023-10-09T08:44:27.172740+0000 mgr.y (mgr.25421) 149 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:28.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:28 smithi016 bash[75179]: cephadm 2023-10-09T08:44:26.290177+0000 mgr.y (mgr.25421) 147 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T08:44:28.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:28 smithi016 bash[75179]: cephadm 2023-10-09T08:44:26.297538+0000 mgr.y (mgr.25421) 148 : cephadm 1 Reconfiguring daemon osd.0 on smithi016 2023-10-09T08:44:28.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:28 smithi016 bash[75179]: cluster 2023-10-09T08:44:27.172740+0000 mgr.y (mgr.25421) 149 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:28.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:28 smithi016 bash[77892]: cephadm 2023-10-09T08:44:26.290177+0000 mgr.y (mgr.25421) 147 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T08:44:28.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:28 smithi016 bash[77892]: cephadm 2023-10-09T08:44:26.297538+0000 mgr.y (mgr.25421) 148 : cephadm 1 Reconfiguring daemon osd.0 on smithi016 2023-10-09T08:44:28.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:28 smithi016 bash[77892]: cluster 2023-10-09T08:44:27.172740+0000 mgr.y (mgr.25421) 149 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[77892]: audit 2023-10-09T08:44:28.552776+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[77892]: audit 2023-10-09T08:44:28.561152+0000 mon.a (mon.0) 180 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[77892]: cephadm 2023-10-09T08:44:28.566765+0000 mgr.y (mgr.25421) 150 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[77892]: audit 2023-10-09T08:44:28.567197+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[77892]: audit 2023-10-09T08:44:28.568490+0000 mon.a (mon.0) 182 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[77892]: cephadm 2023-10-09T08:44:28.571240+0000 mgr.y (mgr.25421) 151 : cephadm 1 Reconfiguring daemon osd.1 on smithi016 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[77892]: cluster 2023-10-09T08:44:29.173388+0000 mgr.y (mgr.25421) 152 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:29.830 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[75179]: audit 2023-10-09T08:44:28.552776+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:29.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[75179]: audit 2023-10-09T08:44:28.561152+0000 mon.a (mon.0) 180 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:29.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[75179]: cephadm 2023-10-09T08:44:28.566765+0000 mgr.y (mgr.25421) 150 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T08:44:29.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[75179]: audit 2023-10-09T08:44:28.567197+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:44:29.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[75179]: audit 2023-10-09T08:44:28.568490+0000 mon.a (mon.0) 182 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:29.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[75179]: cephadm 2023-10-09T08:44:28.571240+0000 mgr.y (mgr.25421) 151 : cephadm 1 Reconfiguring daemon osd.1 on smithi016 2023-10-09T08:44:29.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:29 smithi016 bash[75179]: cluster 2023-10-09T08:44:29.173388+0000 mgr.y (mgr.25421) 152 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:29 smithi094 bash[59677]: audit 2023-10-09T08:44:28.552776+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:29 smithi094 bash[59677]: audit 2023-10-09T08:44:28.561152+0000 mon.a (mon.0) 180 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:29 smithi094 bash[59677]: cephadm 2023-10-09T08:44:28.566765+0000 mgr.y (mgr.25421) 150 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T08:44:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:29 smithi094 bash[59677]: audit 2023-10-09T08:44:28.567197+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:44:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:29 smithi094 bash[59677]: audit 2023-10-09T08:44:28.568490+0000 mon.a (mon.0) 182 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:29 smithi094 bash[59677]: cephadm 2023-10-09T08:44:28.571240+0000 mgr.y (mgr.25421) 151 : cephadm 1 Reconfiguring daemon osd.1 on smithi016 2023-10-09T08:44:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:29 smithi094 bash[59677]: cluster 2023-10-09T08:44:29.173388+0000 mgr.y (mgr.25421) 152 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:32.090 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[75179]: audit 2023-10-09T08:44:30.790846+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:32.090 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[75179]: audit 2023-10-09T08:44:30.798720+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[75179]: cephadm 2023-10-09T08:44:30.840613+0000 mgr.y (mgr.25421) 153 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[75179]: audit 2023-10-09T08:44:30.841247+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[75179]: audit 2023-10-09T08:44:30.843091+0000 mon.a (mon.0) 186 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[75179]: cephadm 2023-10-09T08:44:30.847445+0000 mgr.y (mgr.25421) 154 : cephadm 1 Reconfiguring daemon osd.2 on smithi016 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[75179]: cluster 2023-10-09T08:44:31.174312+0000 mgr.y (mgr.25421) 155 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[77892]: audit 2023-10-09T08:44:30.790846+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[77892]: audit 2023-10-09T08:44:30.798720+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[77892]: cephadm 2023-10-09T08:44:30.840613+0000 mgr.y (mgr.25421) 153 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[77892]: audit 2023-10-09T08:44:30.841247+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:44:32.091 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[77892]: audit 2023-10-09T08:44:30.843091+0000 mon.a (mon.0) 186 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:32.092 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[77892]: cephadm 2023-10-09T08:44:30.847445+0000 mgr.y (mgr.25421) 154 : cephadm 1 Reconfiguring daemon osd.2 on smithi016 2023-10-09T08:44:32.092 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:31 smithi016 bash[77892]: cluster 2023-10-09T08:44:31.174312+0000 mgr.y (mgr.25421) 155 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:31 smithi094 bash[59677]: audit 2023-10-09T08:44:30.790846+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:31 smithi094 bash[59677]: audit 2023-10-09T08:44:30.798720+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:31 smithi094 bash[59677]: cephadm 2023-10-09T08:44:30.840613+0000 mgr.y (mgr.25421) 153 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T08:44:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:31 smithi094 bash[59677]: audit 2023-10-09T08:44:30.841247+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:44:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:31 smithi094 bash[59677]: audit 2023-10-09T08:44:30.843091+0000 mon.a (mon.0) 186 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:31 smithi094 bash[59677]: cephadm 2023-10-09T08:44:30.847445+0000 mgr.y (mgr.25421) 154 : cephadm 1 Reconfiguring daemon osd.2 on smithi016 2023-10-09T08:44:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:31 smithi094 bash[59677]: cluster 2023-10-09T08:44:31.174312+0000 mgr.y (mgr.25421) 155 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[75179]: audit 2023-10-09T08:44:33.130837+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[75179]: audit 2023-10-09T08:44:33.137445+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[75179]: cephadm 2023-10-09T08:44:33.165953+0000 mgr.y (mgr.25421) 156 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T08:44:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[75179]: audit 2023-10-09T08:44:33.166547+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:44:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[75179]: audit 2023-10-09T08:44:33.168165+0000 mon.a (mon.0) 190 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:34.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[75179]: cephadm 2023-10-09T08:44:33.172345+0000 mgr.y (mgr.25421) 157 : cephadm 1 Reconfiguring daemon osd.3 on smithi016 2023-10-09T08:44:34.449 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[75179]: cluster 2023-10-09T08:44:33.174817+0000 mgr.y (mgr.25421) 158 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:34.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[77892]: audit 2023-10-09T08:44:33.130837+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:34.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[77892]: audit 2023-10-09T08:44:33.137445+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:34.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[77892]: cephadm 2023-10-09T08:44:33.165953+0000 mgr.y (mgr.25421) 156 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T08:44:34.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[77892]: audit 2023-10-09T08:44:33.166547+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:44:34.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[77892]: audit 2023-10-09T08:44:33.168165+0000 mon.a (mon.0) 190 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:34.450 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[77892]: cephadm 2023-10-09T08:44:33.172345+0000 mgr.y (mgr.25421) 157 : cephadm 1 Reconfiguring daemon osd.3 on smithi016 2023-10-09T08:44:34.450 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:34 smithi016 bash[77892]: cluster 2023-10-09T08:44:33.174817+0000 mgr.y (mgr.25421) 158 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:34 smithi094 bash[59677]: audit 2023-10-09T08:44:33.130837+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:34 smithi094 bash[59677]: audit 2023-10-09T08:44:33.137445+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:34 smithi094 bash[59677]: cephadm 2023-10-09T08:44:33.165953+0000 mgr.y (mgr.25421) 156 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T08:44:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:34 smithi094 bash[59677]: audit 2023-10-09T08:44:33.166547+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:44:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:34 smithi094 bash[59677]: audit 2023-10-09T08:44:33.168165+0000 mon.a (mon.0) 190 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:34 smithi094 bash[59677]: cephadm 2023-10-09T08:44:33.172345+0000 mgr.y (mgr.25421) 157 : cephadm 1 Reconfiguring daemon osd.3 on smithi016 2023-10-09T08:44:34.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:34 smithi094 bash[59677]: cluster 2023-10-09T08:44:33.174817+0000 mgr.y (mgr.25421) 158 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:35 smithi094 bash[59677]: audit 2023-10-09T08:44:34.197338+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:35.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:35 smithi016 bash[75179]: audit 2023-10-09T08:44:34.197338+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:35.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:35 smithi016 bash[77892]: audit 2023-10-09T08:44:34.197338+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:36.004 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:44:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:44:35] "GET /metrics HTTP/1.1" 200 33761 "" "Prometheus/2.33.4" 2023-10-09T08:44:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:36 smithi094 bash[59677]: cluster 2023-10-09T08:44:35.175867+0000 mgr.y (mgr.25421) 159 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:36 smithi094 bash[59677]: audit 2023-10-09T08:44:35.436233+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:36 smithi094 bash[59677]: audit 2023-10-09T08:44:35.445350+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:36.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:36 smithi094 bash[59677]: audit 2023-10-09T08:44:35.492985+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:44:36.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:36 smithi094 bash[59677]: audit 2023-10-09T08:44:35.494851+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:36.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[75179]: cluster 2023-10-09T08:44:35.175867+0000 mgr.y (mgr.25421) 159 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:36.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[75179]: audit 2023-10-09T08:44:35.436233+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:36.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[75179]: audit 2023-10-09T08:44:35.445350+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:36.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[75179]: audit 2023-10-09T08:44:35.492985+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:44:36.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[75179]: audit 2023-10-09T08:44:35.494851+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:36.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[77892]: cluster 2023-10-09T08:44:35.175867+0000 mgr.y (mgr.25421) 159 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:36.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[77892]: audit 2023-10-09T08:44:35.436233+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:36.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[77892]: audit 2023-10-09T08:44:35.445350+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:36.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[77892]: audit 2023-10-09T08:44:35.492985+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.ccdktz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:44:36.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:36 smithi016 bash[77892]: audit 2023-10-09T08:44:35.494851+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:37.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:37 smithi016 bash[75179]: cephadm 2023-10-09T08:44:35.492375+0000 mgr.y (mgr.25421) 160 : cephadm 1 Reconfiguring mds.foo.smithi016.ccdktz (monmap changed)... 2023-10-09T08:44:37.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:37 smithi016 bash[75179]: cephadm 2023-10-09T08:44:35.496987+0000 mgr.y (mgr.25421) 161 : cephadm 1 Reconfiguring daemon mds.foo.smithi016.ccdktz on smithi016 2023-10-09T08:44:37.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:37 smithi016 bash[77892]: cephadm 2023-10-09T08:44:35.492375+0000 mgr.y (mgr.25421) 160 : cephadm 1 Reconfiguring mds.foo.smithi016.ccdktz (monmap changed)... 2023-10-09T08:44:37.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:37 smithi016 bash[77892]: cephadm 2023-10-09T08:44:35.496987+0000 mgr.y (mgr.25421) 161 : cephadm 1 Reconfiguring daemon mds.foo.smithi016.ccdktz on smithi016 2023-10-09T08:44:37.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:37 smithi094 bash[59677]: cephadm 2023-10-09T08:44:35.492375+0000 mgr.y (mgr.25421) 160 : cephadm 1 Reconfiguring mds.foo.smithi016.ccdktz (monmap changed)... 2023-10-09T08:44:37.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:37 smithi094 bash[59677]: cephadm 2023-10-09T08:44:35.496987+0000 mgr.y (mgr.25421) 161 : cephadm 1 Reconfiguring daemon mds.foo.smithi016.ccdktz on smithi016 2023-10-09T08:44:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:38 smithi094 bash[59677]: cluster 2023-10-09T08:44:37.176436+0000 mgr.y (mgr.25421) 162 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:38 smithi094 bash[59677]: audit 2023-10-09T08:44:37.960428+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:38 smithi094 bash[59677]: audit 2023-10-09T08:44:37.969534+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:38 smithi094 bash[59677]: audit 2023-10-09T08:44:38.017967+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:38 smithi094 bash[59677]: audit 2023-10-09T08:44:38.019802+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:38 smithi094 bash[59677]: audit 2023-10-09T08:44:38.021508+0000 mon.a (mon.0) 200 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[75179]: cluster 2023-10-09T08:44:37.176436+0000 mgr.y (mgr.25421) 162 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[75179]: audit 2023-10-09T08:44:37.960428+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[75179]: audit 2023-10-09T08:44:37.969534+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[75179]: audit 2023-10-09T08:44:38.017967+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[75179]: audit 2023-10-09T08:44:38.019802+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[75179]: audit 2023-10-09T08:44:38.021508+0000 mon.a (mon.0) 200 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[77892]: cluster 2023-10-09T08:44:37.176436+0000 mgr.y (mgr.25421) 162 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:38.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[77892]: audit 2023-10-09T08:44:37.960428+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:38.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[77892]: audit 2023-10-09T08:44:37.969534+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:38.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[77892]: audit 2023-10-09T08:44:38.017967+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:44:38.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[77892]: audit 2023-10-09T08:44:38.019802+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:44:38.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:38 smithi016 bash[77892]: audit 2023-10-09T08:44:38.021508+0000 mon.a (mon.0) 200 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:39.438 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:39 smithi094 bash[59677]: cephadm 2023-10-09T08:44:38.017251+0000 mgr.y (mgr.25421) 163 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T08:44:39.438 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:39 smithi094 bash[59677]: cephadm 2023-10-09T08:44:38.023438+0000 mgr.y (mgr.25421) 164 : cephadm 1 Reconfiguring daemon mon.b on smithi094 2023-10-09T08:44:39.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:39 smithi016 bash[75179]: cephadm 2023-10-09T08:44:38.017251+0000 mgr.y (mgr.25421) 163 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T08:44:39.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:39 smithi016 bash[75179]: cephadm 2023-10-09T08:44:38.023438+0000 mgr.y (mgr.25421) 164 : cephadm 1 Reconfiguring daemon mon.b on smithi094 2023-10-09T08:44:39.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:39 smithi016 bash[77892]: cephadm 2023-10-09T08:44:38.017251+0000 mgr.y (mgr.25421) 163 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T08:44:39.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:39 smithi016 bash[77892]: cephadm 2023-10-09T08:44:38.023438+0000 mgr.y (mgr.25421) 164 : cephadm 1 Reconfiguring daemon mon.b on smithi094 2023-10-09T08:44:40.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:40 smithi094 bash[59677]: cluster 2023-10-09T08:44:39.177021+0000 mgr.y (mgr.25421) 165 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:40.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:40 smithi016 bash[75179]: cluster 2023-10-09T08:44:39.177021+0000 mgr.y (mgr.25421) 165 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:40.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:40 smithi016 bash[77892]: cluster 2023-10-09T08:44:39.177021+0000 mgr.y (mgr.25421) 165 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:41.688 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: audit 2023-10-09T08:44:40.365964+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:41.688 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: audit 2023-10-09T08:44:40.373941+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:41.688 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: cephadm 2023-10-09T08:44:40.394204+0000 mgr.y (mgr.25421) 166 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T08:44:41.688 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: audit 2023-10-09T08:44:40.394862+0000 mon.a (mon.0) 203 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:44:41.689 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: audit 2023-10-09T08:44:40.396838+0000 mon.a (mon.0) 204 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:44:41.689 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: audit 2023-10-09T08:44:40.398419+0000 mon.a (mon.0) 205 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:41.689 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: cephadm 2023-10-09T08:44:40.400299+0000 mgr.y (mgr.25421) 167 : cephadm 1 Reconfiguring daemon mgr.x on smithi094 2023-10-09T08:44:41.689 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:41 smithi094 bash[59677]: cluster 2023-10-09T08:44:41.178074+0000 mgr.y (mgr.25421) 168 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:41.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: audit 2023-10-09T08:44:40.365964+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:41.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: audit 2023-10-09T08:44:40.373941+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:41.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: cephadm 2023-10-09T08:44:40.394204+0000 mgr.y (mgr.25421) 166 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T08:44:41.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: audit 2023-10-09T08:44:40.394862+0000 mon.a (mon.0) 203 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:44:41.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: audit 2023-10-09T08:44:40.396838+0000 mon.a (mon.0) 204 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:44:41.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: audit 2023-10-09T08:44:40.398419+0000 mon.a (mon.0) 205 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:41.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: cephadm 2023-10-09T08:44:40.400299+0000 mgr.y (mgr.25421) 167 : cephadm 1 Reconfiguring daemon mgr.x on smithi094 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[75179]: cluster 2023-10-09T08:44:41.178074+0000 mgr.y (mgr.25421) 168 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: audit 2023-10-09T08:44:40.365964+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: audit 2023-10-09T08:44:40.373941+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: cephadm 2023-10-09T08:44:40.394204+0000 mgr.y (mgr.25421) 166 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: audit 2023-10-09T08:44:40.394862+0000 mon.a (mon.0) 203 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: audit 2023-10-09T08:44:40.396838+0000 mon.a (mon.0) 204 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: audit 2023-10-09T08:44:40.398419+0000 mon.a (mon.0) 205 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:41.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: cephadm 2023-10-09T08:44:40.400299+0000 mgr.y (mgr.25421) 167 : cephadm 1 Reconfiguring daemon mgr.x on smithi094 2023-10-09T08:44:41.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:41 smithi016 bash[77892]: cluster 2023-10-09T08:44:41.178074+0000 mgr.y (mgr.25421) 168 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:43.894 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:43 smithi094 bash[59677]: audit 2023-10-09T08:44:42.630423+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:43.894 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:43 smithi094 bash[59677]: audit 2023-10-09T08:44:42.638429+0000 mon.a (mon.0) 207 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:43.894 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:43 smithi094 bash[59677]: cephadm 2023-10-09T08:44:42.670274+0000 mgr.y (mgr.25421) 169 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T08:44:43.894 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:43 smithi094 bash[59677]: audit 2023-10-09T08:44:42.671000+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:44:43.895 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:43 smithi094 bash[59677]: audit 2023-10-09T08:44:42.672755+0000 mon.a (mon.0) 209 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:43.895 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:43 smithi094 bash[59677]: cephadm 2023-10-09T08:44:42.677271+0000 mgr.y (mgr.25421) 170 : cephadm 1 Reconfiguring daemon osd.4 on smithi094 2023-10-09T08:44:43.895 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:43 smithi094 bash[59677]: cluster 2023-10-09T08:44:43.178772+0000 mgr.y (mgr.25421) 171 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:44.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[75179]: audit 2023-10-09T08:44:42.630423+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[75179]: audit 2023-10-09T08:44:42.638429+0000 mon.a (mon.0) 207 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[75179]: cephadm 2023-10-09T08:44:42.670274+0000 mgr.y (mgr.25421) 169 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[75179]: audit 2023-10-09T08:44:42.671000+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[75179]: audit 2023-10-09T08:44:42.672755+0000 mon.a (mon.0) 209 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[75179]: cephadm 2023-10-09T08:44:42.677271+0000 mgr.y (mgr.25421) 170 : cephadm 1 Reconfiguring daemon osd.4 on smithi094 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[75179]: cluster 2023-10-09T08:44:43.178772+0000 mgr.y (mgr.25421) 171 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[77892]: audit 2023-10-09T08:44:42.630423+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:44.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[77892]: audit 2023-10-09T08:44:42.638429+0000 mon.a (mon.0) 207 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:44.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[77892]: cephadm 2023-10-09T08:44:42.670274+0000 mgr.y (mgr.25421) 169 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T08:44:44.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[77892]: audit 2023-10-09T08:44:42.671000+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:44:44.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[77892]: audit 2023-10-09T08:44:42.672755+0000 mon.a (mon.0) 209 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:44.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[77892]: cephadm 2023-10-09T08:44:42.677271+0000 mgr.y (mgr.25421) 170 : cephadm 1 Reconfiguring daemon osd.4 on smithi094 2023-10-09T08:44:44.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:43 smithi016 bash[77892]: cluster 2023-10-09T08:44:43.178772+0000 mgr.y (mgr.25421) 171 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:46.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:44:45] "GET /metrics HTTP/1.1" 200 33761 "" "Prometheus/2.33.4" 2023-10-09T08:44:46.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[75179]: audit 2023-10-09T08:44:44.981737+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:46.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[75179]: audit 2023-10-09T08:44:44.988660+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:46.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[75179]: cephadm 2023-10-09T08:44:44.996654+0000 mgr.y (mgr.25421) 172 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T08:44:46.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[75179]: audit 2023-10-09T08:44:44.997150+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:44:46.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[75179]: audit 2023-10-09T08:44:44.998499+0000 mon.a (mon.0) 213 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:46.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[75179]: cephadm 2023-10-09T08:44:45.001774+0000 mgr.y (mgr.25421) 173 : cephadm 1 Reconfiguring daemon osd.5 on smithi094 2023-10-09T08:44:46.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[75179]: cluster 2023-10-09T08:44:45.179761+0000 mgr.y (mgr.25421) 174 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:46.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[77892]: audit 2023-10-09T08:44:44.981737+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:46.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[77892]: audit 2023-10-09T08:44:44.988660+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:46.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[77892]: cephadm 2023-10-09T08:44:44.996654+0000 mgr.y (mgr.25421) 172 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T08:44:46.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[77892]: audit 2023-10-09T08:44:44.997150+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:44:46.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[77892]: audit 2023-10-09T08:44:44.998499+0000 mon.a (mon.0) 213 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:46.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[77892]: cephadm 2023-10-09T08:44:45.001774+0000 mgr.y (mgr.25421) 173 : cephadm 1 Reconfiguring daemon osd.5 on smithi094 2023-10-09T08:44:46.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:45 smithi016 bash[77892]: cluster 2023-10-09T08:44:45.179761+0000 mgr.y (mgr.25421) 174 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:45 smithi094 bash[59677]: audit 2023-10-09T08:44:44.981737+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:45 smithi094 bash[59677]: audit 2023-10-09T08:44:44.988660+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:45 smithi094 bash[59677]: cephadm 2023-10-09T08:44:44.996654+0000 mgr.y (mgr.25421) 172 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T08:44:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:45 smithi094 bash[59677]: audit 2023-10-09T08:44:44.997150+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:44:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:45 smithi094 bash[59677]: audit 2023-10-09T08:44:44.998499+0000 mon.a (mon.0) 213 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:45 smithi094 bash[59677]: cephadm 2023-10-09T08:44:45.001774+0000 mgr.y (mgr.25421) 173 : cephadm 1 Reconfiguring daemon osd.5 on smithi094 2023-10-09T08:44:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:45 smithi094 bash[59677]: cluster 2023-10-09T08:44:45.179761+0000 mgr.y (mgr.25421) 174 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:48 smithi094 bash[59677]: audit 2023-10-09T08:44:47.148232+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:48 smithi094 bash[59677]: audit 2023-10-09T08:44:47.156647+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:48 smithi094 bash[59677]: cephadm 2023-10-09T08:44:47.172004+0000 mgr.y (mgr.25421) 175 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T08:44:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:48 smithi094 bash[59677]: audit 2023-10-09T08:44:47.172740+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:44:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:48 smithi094 bash[59677]: audit 2023-10-09T08:44:47.174697+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:48 smithi094 bash[59677]: cephadm 2023-10-09T08:44:47.178520+0000 mgr.y (mgr.25421) 176 : cephadm 1 Reconfiguring daemon osd.6 on smithi094 2023-10-09T08:44:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:48 smithi094 bash[59677]: cluster 2023-10-09T08:44:47.180278+0000 mgr.y (mgr.25421) 177 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[75179]: audit 2023-10-09T08:44:47.148232+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[75179]: audit 2023-10-09T08:44:47.156647+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[75179]: cephadm 2023-10-09T08:44:47.172004+0000 mgr.y (mgr.25421) 175 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T08:44:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[75179]: audit 2023-10-09T08:44:47.172740+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:44:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[75179]: audit 2023-10-09T08:44:47.174697+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[75179]: cephadm 2023-10-09T08:44:47.178520+0000 mgr.y (mgr.25421) 176 : cephadm 1 Reconfiguring daemon osd.6 on smithi094 2023-10-09T08:44:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[75179]: cluster 2023-10-09T08:44:47.180278+0000 mgr.y (mgr.25421) 177 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:48.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[77892]: audit 2023-10-09T08:44:47.148232+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:48.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[77892]: audit 2023-10-09T08:44:47.156647+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:48.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[77892]: cephadm 2023-10-09T08:44:47.172004+0000 mgr.y (mgr.25421) 175 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T08:44:48.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[77892]: audit 2023-10-09T08:44:47.172740+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:44:48.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[77892]: audit 2023-10-09T08:44:47.174697+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:48.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[77892]: cephadm 2023-10-09T08:44:47.178520+0000 mgr.y (mgr.25421) 176 : cephadm 1 Reconfiguring daemon osd.6 on smithi094 2023-10-09T08:44:48.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:48 smithi016 bash[77892]: cluster 2023-10-09T08:44:47.180278+0000 mgr.y (mgr.25421) 177 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:50.449 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: audit 2023-10-09T08:44:49.161161+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:50.449 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: audit 2023-10-09T08:44:49.169154+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:50.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: cluster 2023-10-09T08:44:49.180867+0000 mgr.y (mgr.25421) 178 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:50.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: audit 2023-10-09T08:44:49.197873+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:50.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: cephadm 2023-10-09T08:44:49.198212+0000 mgr.y (mgr.25421) 179 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T08:44:50.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: audit 2023-10-09T08:44:49.198788+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T08:44:50.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: audit 2023-10-09T08:44:49.201143+0000 mon.a (mon.0) 222 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:50.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:50 smithi094 bash[59677]: cephadm 2023-10-09T08:44:49.212522+0000 mgr.y (mgr.25421) 180 : cephadm 1 Reconfiguring daemon osd.7 on smithi094 2023-10-09T08:44:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: audit 2023-10-09T08:44:49.161161+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: audit 2023-10-09T08:44:49.169154+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: cluster 2023-10-09T08:44:49.180867+0000 mgr.y (mgr.25421) 178 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: audit 2023-10-09T08:44:49.197873+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: cephadm 2023-10-09T08:44:49.198212+0000 mgr.y (mgr.25421) 179 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T08:44:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: audit 2023-10-09T08:44:49.198788+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T08:44:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: audit 2023-10-09T08:44:49.201143+0000 mon.a (mon.0) 222 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[75179]: cephadm 2023-10-09T08:44:49.212522+0000 mgr.y (mgr.25421) 180 : cephadm 1 Reconfiguring daemon osd.7 on smithi094 2023-10-09T08:44:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: audit 2023-10-09T08:44:49.161161+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: audit 2023-10-09T08:44:49.169154+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: cluster 2023-10-09T08:44:49.180867+0000 mgr.y (mgr.25421) 178 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: audit 2023-10-09T08:44:49.197873+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:44:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: cephadm 2023-10-09T08:44:49.198212+0000 mgr.y (mgr.25421) 179 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T08:44:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: audit 2023-10-09T08:44:49.198788+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T08:44:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: audit 2023-10-09T08:44:49.201143+0000 mon.a (mon.0) 222 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:50.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:50 smithi016 bash[77892]: cephadm 2023-10-09T08:44:49.212522+0000 mgr.y (mgr.25421) 180 : cephadm 1 Reconfiguring daemon osd.7 on smithi094 2023-10-09T08:44:52.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[77892]: cluster 2023-10-09T08:44:51.181897+0000 mgr.y (mgr.25421) 181 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[77892]: audit 2023-10-09T08:44:51.458600+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[77892]: audit 2023-10-09T08:44:51.465664+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[77892]: audit 2023-10-09T08:44:51.504602+0000 mon.a (mon.0) 225 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[77892]: audit 2023-10-09T08:44:51.506783+0000 mon.a (mon.0) 226 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[75179]: cluster 2023-10-09T08:44:51.181897+0000 mgr.y (mgr.25421) 181 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[75179]: audit 2023-10-09T08:44:51.458600+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[75179]: audit 2023-10-09T08:44:51.465664+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[75179]: audit 2023-10-09T08:44:51.504602+0000 mon.a (mon.0) 225 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:44:52.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:52 smithi016 bash[75179]: audit 2023-10-09T08:44:51.506783+0000 mon.a (mon.0) 226 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:52.708 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:52 smithi094 bash[59677]: cluster 2023-10-09T08:44:51.181897+0000 mgr.y (mgr.25421) 181 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:52.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:52 smithi094 bash[59677]: audit 2023-10-09T08:44:51.458600+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:52.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:52 smithi094 bash[59677]: audit 2023-10-09T08:44:51.465664+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:52.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:52 smithi094 bash[59677]: audit 2023-10-09T08:44:51.504602+0000 mon.a (mon.0) 225 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi094.zfssci", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T08:44:52.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:52 smithi094 bash[59677]: audit 2023-10-09T08:44:51.506783+0000 mon.a (mon.0) 226 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:44:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:53 smithi094 bash[59677]: cephadm 2023-10-09T08:44:51.503873+0000 mgr.y (mgr.25421) 182 : cephadm 1 Reconfiguring mds.foo.smithi094.zfssci (monmap changed)... 2023-10-09T08:44:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:53 smithi094 bash[59677]: cephadm 2023-10-09T08:44:51.508687+0000 mgr.y (mgr.25421) 183 : cephadm 1 Reconfiguring daemon mds.foo.smithi094.zfssci on smithi094 2023-10-09T08:44:53.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:53 smithi016 bash[75179]: cephadm 2023-10-09T08:44:51.503873+0000 mgr.y (mgr.25421) 182 : cephadm 1 Reconfiguring mds.foo.smithi094.zfssci (monmap changed)... 2023-10-09T08:44:53.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:53 smithi016 bash[75179]: cephadm 2023-10-09T08:44:51.508687+0000 mgr.y (mgr.25421) 183 : cephadm 1 Reconfiguring daemon mds.foo.smithi094.zfssci on smithi094 2023-10-09T08:44:53.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:53 smithi016 bash[77892]: cephadm 2023-10-09T08:44:51.503873+0000 mgr.y (mgr.25421) 182 : cephadm 1 Reconfiguring mds.foo.smithi094.zfssci (monmap changed)... 2023-10-09T08:44:53.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:53 smithi016 bash[77892]: cephadm 2023-10-09T08:44:51.508687+0000 mgr.y (mgr.25421) 183 : cephadm 1 Reconfiguring daemon mds.foo.smithi094.zfssci on smithi094 2023-10-09T08:44:54.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[75179]: cluster 2023-10-09T08:44:53.182486+0000 mgr.y (mgr.25421) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:54.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[75179]: audit 2023-10-09T08:44:53.729101+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[75179]: audit 2023-10-09T08:44:53.737438+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[75179]: audit 2023-10-09T08:44:53.789669+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:54.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[75179]: audit 2023-10-09T08:44:54.165795+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[75179]: audit 2023-10-09T08:44:54.173749+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[77892]: cluster 2023-10-09T08:44:53.182486+0000 mgr.y (mgr.25421) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:54.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[77892]: audit 2023-10-09T08:44:53.729101+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[77892]: audit 2023-10-09T08:44:53.737438+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[77892]: audit 2023-10-09T08:44:53.789669+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:54.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[77892]: audit 2023-10-09T08:44:54.165795+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:54 smithi016 bash[77892]: audit 2023-10-09T08:44:54.173749+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:54 smithi094 bash[59677]: cluster 2023-10-09T08:44:53.182486+0000 mgr.y (mgr.25421) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:54 smithi094 bash[59677]: audit 2023-10-09T08:44:53.729101+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:54 smithi094 bash[59677]: audit 2023-10-09T08:44:53.737438+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:54 smithi094 bash[59677]: audit 2023-10-09T08:44:53.789669+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:44:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:54 smithi094 bash[59677]: audit 2023-10-09T08:44:54.165795+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:54 smithi094 bash[59677]: audit 2023-10-09T08:44:54.173749+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:44:56.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:44:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:44:55] "GET /metrics HTTP/1.1" 200 33763 "" "Prometheus/2.33.4" 2023-10-09T08:44:56.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:56 smithi094 bash[59677]: cluster 2023-10-09T08:44:55.183487+0000 mgr.y (mgr.25421) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:56.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:56 smithi016 bash[75179]: cluster 2023-10-09T08:44:55.183487+0000 mgr.y (mgr.25421) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:56.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:56 smithi016 bash[77892]: cluster 2023-10-09T08:44:55.183487+0000 mgr.y (mgr.25421) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:58.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:58 smithi094 bash[59677]: cluster 2023-10-09T08:44:57.184204+0000 mgr.y (mgr.25421) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:58.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:58 smithi016 bash[75179]: cluster 2023-10-09T08:44:57.184204+0000 mgr.y (mgr.25421) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:44:58.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:58 smithi016 bash[77892]: cluster 2023-10-09T08:44:57.184204+0000 mgr.y (mgr.25421) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:00.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:44:59 smithi094 bash[59677]: cluster 2023-10-09T08:44:59.184759+0000 mgr.y (mgr.25421) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:00.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:44:59 smithi016 bash[75179]: cluster 2023-10-09T08:44:59.184759+0000 mgr.y (mgr.25421) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:00.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:44:59 smithi016 bash[77892]: cluster 2023-10-09T08:44:59.184759+0000 mgr.y (mgr.25421) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:01.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:01 smithi094 bash[59677]: audit 2023-10-09T08:45:00.267604+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:01 smithi094 bash[59677]: audit 2023-10-09T08:45:00.274740+0000 mon.a (mon.0) 233 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:01 smithi094 bash[59677]: audit 2023-10-09T08:45:00.747021+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:01.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:01 smithi094 bash[59677]: audit 2023-10-09T08:45:00.749019+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:01.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:01 smithi094 bash[59677]: audit 2023-10-09T08:45:00.759038+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[75179]: audit 2023-10-09T08:45:00.267604+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[75179]: audit 2023-10-09T08:45:00.274740+0000 mon.a (mon.0) 233 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[75179]: audit 2023-10-09T08:45:00.747021+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:01.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[75179]: audit 2023-10-09T08:45:00.749019+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:01.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[75179]: audit 2023-10-09T08:45:00.759038+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[77892]: audit 2023-10-09T08:45:00.267604+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[77892]: audit 2023-10-09T08:45:00.274740+0000 mon.a (mon.0) 233 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:01.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[77892]: audit 2023-10-09T08:45:00.747021+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:01.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[77892]: audit 2023-10-09T08:45:00.749019+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:01.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:01 smithi016 bash[77892]: audit 2023-10-09T08:45:00.759038+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:02 smithi094 bash[59677]: cluster 2023-10-09T08:45:01.185802+0000 mgr.y (mgr.25421) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:02.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:02 smithi016 bash[75179]: cluster 2023-10-09T08:45:01.185802+0000 mgr.y (mgr.25421) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:02.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:02 smithi016 bash[77892]: cluster 2023-10-09T08:45:01.185802+0000 mgr.y (mgr.25421) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:04 smithi094 bash[59677]: cluster 2023-10-09T08:45:03.186445+0000 mgr.y (mgr.25421) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:04 smithi094 bash[59677]: audit 2023-10-09T08:45:04.198300+0000 mon.a (mon.0) 237 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:04.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:04 smithi016 bash[75179]: cluster 2023-10-09T08:45:03.186445+0000 mgr.y (mgr.25421) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:04.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:04 smithi016 bash[75179]: audit 2023-10-09T08:45:04.198300+0000 mon.a (mon.0) 237 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:04.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:04 smithi016 bash[77892]: cluster 2023-10-09T08:45:03.186445+0000 mgr.y (mgr.25421) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:04.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:04 smithi016 bash[77892]: audit 2023-10-09T08:45:04.198300+0000 mon.a (mon.0) 237 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:06.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:45:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:45:05] "GET /metrics HTTP/1.1" 200 33760 "" "Prometheus/2.33.4" 2023-10-09T08:45:06.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:06 smithi094 bash[59677]: cluster 2023-10-09T08:45:05.187525+0000 mgr.y (mgr.25421) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:06.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:06 smithi016 bash[75179]: cluster 2023-10-09T08:45:05.187525+0000 mgr.y (mgr.25421) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:06.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:06 smithi016 bash[77892]: cluster 2023-10-09T08:45:05.187525+0000 mgr.y (mgr.25421) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:08.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:08 smithi094 bash[59677]: cluster 2023-10-09T08:45:07.188249+0000 mgr.y (mgr.25421) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:08.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:08 smithi016 bash[75179]: cluster 2023-10-09T08:45:07.188249+0000 mgr.y (mgr.25421) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:08.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:08 smithi016 bash[77892]: cluster 2023-10-09T08:45:07.188249+0000 mgr.y (mgr.25421) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:09 smithi094 bash[59677]: cluster 2023-10-09T08:45:09.188913+0000 mgr.y (mgr.25421) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:10.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:09 smithi016 bash[75179]: cluster 2023-10-09T08:45:09.188913+0000 mgr.y (mgr.25421) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:10.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:09 smithi016 bash[77892]: cluster 2023-10-09T08:45:09.188913+0000 mgr.y (mgr.25421) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:12.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:12 smithi016 bash[75179]: cluster 2023-10-09T08:45:11.189926+0000 mgr.y (mgr.25421) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:12.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:12 smithi016 bash[77892]: cluster 2023-10-09T08:45:11.189926+0000 mgr.y (mgr.25421) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:12.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:12 smithi094 bash[59677]: cluster 2023-10-09T08:45:11.189926+0000 mgr.y (mgr.25421) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:14 smithi094 bash[59677]: cluster 2023-10-09T08:45:13.190557+0000 mgr.y (mgr.25421) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:14.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:14 smithi016 bash[75179]: cluster 2023-10-09T08:45:13.190557+0000 mgr.y (mgr.25421) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:14 smithi016 bash[77892]: cluster 2023-10-09T08:45:13.190557+0000 mgr.y (mgr.25421) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:16.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:45:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:45:15] "GET /metrics HTTP/1.1" 200 33760 "" "Prometheus/2.33.4" 2023-10-09T08:45:16.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:16 smithi016 bash[77892]: cluster 2023-10-09T08:45:15.191570+0000 mgr.y (mgr.25421) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:16.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:16 smithi016 bash[75179]: cluster 2023-10-09T08:45:15.191570+0000 mgr.y (mgr.25421) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:16 smithi094 bash[59677]: cluster 2023-10-09T08:45:15.191570+0000 mgr.y (mgr.25421) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:16.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T08:45:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:18 smithi094 bash[59677]: cluster 2023-10-09T08:45:17.192217+0000 mgr.y (mgr.25421) 196 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:18.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:18 smithi016 bash[75179]: cluster 2023-10-09T08:45:17.192217+0000 mgr.y (mgr.25421) 196 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:18.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:18 smithi016 bash[77892]: cluster 2023-10-09T08:45:17.192217+0000 mgr.y (mgr.25421) 196 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:19 smithi094 bash[59677]: cluster 2023-10-09T08:45:19.192815+0000 mgr.y (mgr.25421) 197 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:19 smithi094 bash[59677]: audit 2023-10-09T08:45:19.198502+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:20.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:19 smithi016 bash[75179]: cluster 2023-10-09T08:45:19.192815+0000 mgr.y (mgr.25421) 197 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:20.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:19 smithi016 bash[75179]: audit 2023-10-09T08:45:19.198502+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:20.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:19 smithi016 bash[77892]: cluster 2023-10-09T08:45:19.192815+0000 mgr.y (mgr.25421) 197 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:20.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:19 smithi016 bash[77892]: audit 2023-10-09T08:45:19.198502+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:20.364 INFO:teuthology.orchestra.run.smithi016.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:45:21.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T08:45:21.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[77892]: audit 2023-10-09T08:45:20.351654+0000 mgr.y (mgr.25421) 198 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:45:21.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[77892]: cephadm 2023-10-09T08:45:20.352340+0000 mgr.y (mgr.25421) 199 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:45:21.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[77892]: audit 2023-10-09T08:45:20.359103+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:21.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[77892]: audit 2023-10-09T08:45:20.361899+0000 mon.a (mon.0) 240 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:21.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[77892]: cluster 2023-10-09T08:45:21.193838+0000 mgr.y (mgr.25421) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:21.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[75179]: audit 2023-10-09T08:45:20.351654+0000 mgr.y (mgr.25421) 198 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:45:21.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[75179]: cephadm 2023-10-09T08:45:20.352340+0000 mgr.y (mgr.25421) 199 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:45:21.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[75179]: audit 2023-10-09T08:45:20.359103+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:21.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[75179]: audit 2023-10-09T08:45:20.361899+0000 mon.a (mon.0) 240 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:21.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:21 smithi016 bash[75179]: cluster 2023-10-09T08:45:21.193838+0000 mgr.y (mgr.25421) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:21 smithi094 bash[59677]: audit 2023-10-09T08:45:20.351654+0000 mgr.y (mgr.25421) 198 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:45:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:21 smithi094 bash[59677]: cephadm 2023-10-09T08:45:20.352340+0000 mgr.y (mgr.25421) 199 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:45:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:21 smithi094 bash[59677]: audit 2023-10-09T08:45:20.359103+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:21.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:21 smithi094 bash[59677]: audit 2023-10-09T08:45:20.361899+0000 mon.a (mon.0) 240 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:21.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:21 smithi094 bash[59677]: cluster 2023-10-09T08:45:21.193838+0000 mgr.y (mgr.25421) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:24.416 INFO:teuthology.orchestra.run.smithi016.stdout:wait for half of osds to upgrade 2023-10-09T08:45:24.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:24 smithi016 bash[75179]: cluster 2023-10-09T08:45:23.194484+0000 mgr.y (mgr.25421) 201 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:24.512 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:24 smithi016 bash[77892]: cluster 2023-10-09T08:45:23.194484+0000 mgr.y (mgr.25421) 201 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:24.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:24 smithi094 bash[59677]: cluster 2023-10-09T08:45:23.194484+0000 mgr.y (mgr.25421) 201 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:25.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T08:45:26.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:45:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:45:25] "GET /metrics HTTP/1.1" 200 33687 "" "Prometheus/2.33.4" 2023-10-09T08:45:26.532 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:26 smithi016 bash[75179]: cluster 2023-10-09T08:45:25.195480+0000 mgr.y (mgr.25421) 202 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:26.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:26 smithi016 bash[77892]: cluster 2023-10-09T08:45:25.195480+0000 mgr.y (mgr.25421) 202 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:26.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:26 smithi094 bash[59677]: cluster 2023-10-09T08:45:25.195480+0000 mgr.y (mgr.25421) 202 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:26.610925+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:26.620002+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:27.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.088096+0000 mon.a (mon.0) 243 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.089706+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.098272+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.175727+0000 mon.a (mon.0) 246 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.182190+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.185133+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: cephadm 2023-10-09T08:45:27.186095+0000 mgr.y (mgr.25421) 203 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.193624+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.194892+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: cluster 2023-10-09T08:45:27.195885+0000 mgr.y (mgr.25421) 204 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.200051+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T08:45:27.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.201228+0000 mon.a (mon.0) 252 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.205962+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.207058+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.212273+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.213808+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: cephadm 2023-10-09T08:45:27.214605+0000 mgr.y (mgr.25421) 205 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.219257+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.221428+0000 mon.a (mon.0) 258 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: audit 2023-10-09T08:45:27.221637+0000 mgr.y (mgr.25421) 206 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T08:45:27.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:27 smithi094 bash[59677]: cephadm 2023-10-09T08:45:27.222970+0000 mgr.y (mgr.25421) 207 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T08:45:28.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:26.610925+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:26.620002+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.088096+0000 mon.a (mon.0) 243 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.089706+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.098272+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.175727+0000 mon.a (mon.0) 246 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.182190+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.185133+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: cephadm 2023-10-09T08:45:27.186095+0000 mgr.y (mgr.25421) 203 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.193624+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.194892+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: cluster 2023-10-09T08:45:27.195885+0000 mgr.y (mgr.25421) 204 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.200051+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T08:45:28.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.201228+0000 mon.a (mon.0) 252 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.205962+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.207058+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.212273+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.213808+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: cephadm 2023-10-09T08:45:27.214605+0000 mgr.y (mgr.25421) 205 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.219257+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.221428+0000 mon.a (mon.0) 258 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: audit 2023-10-09T08:45:27.221637+0000 mgr.y (mgr.25421) 206 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[75179]: cephadm 2023-10-09T08:45:27.222970+0000 mgr.y (mgr.25421) 207 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T08:45:28.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:26.610925+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:26.620002+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.088096+0000 mon.a (mon.0) 243 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.089706+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.098272+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.175727+0000 mon.a (mon.0) 246 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.182190+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.185133+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: cephadm 2023-10-09T08:45:27.186095+0000 mgr.y (mgr.25421) 203 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.193624+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.194892+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: cluster 2023-10-09T08:45:27.195885+0000 mgr.y (mgr.25421) 204 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.200051+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.201228+0000 mon.a (mon.0) 252 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.205962+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.207058+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.212273+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.213808+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: cephadm 2023-10-09T08:45:27.214605+0000 mgr.y (mgr.25421) 205 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.219257+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.221428+0000 mon.a (mon.0) 258 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: audit 2023-10-09T08:45:27.221637+0000 mgr.y (mgr.25421) 206 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T08:45:28.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:27 smithi016 bash[77892]: cephadm 2023-10-09T08:45:27.222970+0000 mgr.y (mgr.25421) 207 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T08:45:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:29 smithi094 bash[59677]: cluster 2023-10-09T08:45:29.196453+0000 mgr.y (mgr.25421) 208 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:29 smithi094 bash[59677]: audit 2023-10-09T08:45:29.526245+0000 mon.a (mon.0) 259 : audit 0 from='client.? 172.21.15.16:0/102406127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:30.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:29 smithi016 bash[75179]: cluster 2023-10-09T08:45:29.196453+0000 mgr.y (mgr.25421) 208 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:30.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:29 smithi016 bash[75179]: audit 2023-10-09T08:45:29.526245+0000 mon.a (mon.0) 259 : audit 0 from='client.? 172.21.15.16:0/102406127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:30.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:29 smithi016 bash[77892]: cluster 2023-10-09T08:45:29.196453+0000 mgr.y (mgr.25421) 208 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:30.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:29 smithi016 bash[77892]: audit 2023-10-09T08:45:29.526245+0000 mon.a (mon.0) 259 : audit 0 from='client.? 172.21.15.16:0/102406127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:31.290 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:30 smithi016 bash[75179]: cephadm 2023-10-09T08:45:29.993586+0000 mgr.y (mgr.25421) 209 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T08:45:31.290 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:30 smithi016 bash[75179]: audit 2023-10-09T08:45:30.000518+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:31.290 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:30 smithi016 bash[75179]: audit 2023-10-09T08:45:30.002388+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:45:31.290 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:31 smithi016 bash[75179]: audit 2023-10-09T08:45:30.003665+0000 mon.a (mon.0) 262 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:31.290 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:31 smithi016 bash[75179]: cephadm 2023-10-09T08:45:30.006586+0000 mgr.y (mgr.25421) 210 : cephadm 1 Deploying daemon osd.0 on smithi016 2023-10-09T08:45:31.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:31 smithi016 bash[77892]: cephadm 2023-10-09T08:45:29.993586+0000 mgr.y (mgr.25421) 209 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T08:45:31.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:31 smithi016 bash[77892]: audit 2023-10-09T08:45:30.000518+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:31.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:31 smithi016 bash[77892]: audit 2023-10-09T08:45:30.002388+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:45:31.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:31 smithi016 bash[77892]: audit 2023-10-09T08:45:30.003665+0000 mon.a (mon.0) 262 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:31.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:31 smithi016 bash[77892]: cephadm 2023-10-09T08:45:30.006586+0000 mgr.y (mgr.25421) 210 : cephadm 1 Deploying daemon osd.0 on smithi016 2023-10-09T08:45:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:31 smithi094 bash[59677]: cephadm 2023-10-09T08:45:29.993586+0000 mgr.y (mgr.25421) 209 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T08:45:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:31 smithi094 bash[59677]: audit 2023-10-09T08:45:30.000518+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:31 smithi094 bash[59677]: audit 2023-10-09T08:45:30.002388+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:45:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:31 smithi094 bash[59677]: audit 2023-10-09T08:45:30.003665+0000 mon.a (mon.0) 262 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:31.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:31 smithi094 bash[59677]: cephadm 2023-10-09T08:45:30.006586+0000 mgr.y (mgr.25421) 210 : cephadm 1 Deploying daemon osd.0 on smithi016 2023-10-09T08:45:32.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:32 smithi094 bash[59677]: cluster 2023-10-09T08:45:31.197482+0000 mgr.y (mgr.25421) 211 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:32.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:32 smithi094 bash[59677]: audit 2023-10-09T08:45:31.897486+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.16:0/1699051892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:32.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:32 smithi016 bash[75179]: cluster 2023-10-09T08:45:31.197482+0000 mgr.y (mgr.25421) 211 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:32.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:32 smithi016 bash[75179]: audit 2023-10-09T08:45:31.897486+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.16:0/1699051892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:32.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:32 smithi016 bash[77892]: cluster 2023-10-09T08:45:31.197482+0000 mgr.y (mgr.25421) 211 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:32.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:32 smithi016 bash[77892]: audit 2023-10-09T08:45:31.897486+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.16:0/1699051892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:34.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:34 smithi016 bash[75179]: cluster 2023-10-09T08:45:33.198075+0000 mgr.y (mgr.25421) 212 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:34.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:34 smithi016 bash[75179]: audit 2023-10-09T08:45:34.198447+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:34.573 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:34 smithi016 bash[77892]: cluster 2023-10-09T08:45:33.198075+0000 mgr.y (mgr.25421) 212 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:34.573 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:34 smithi016 bash[77892]: audit 2023-10-09T08:45:34.198447+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:34 smithi094 bash[59677]: cluster 2023-10-09T08:45:33.198075+0000 mgr.y (mgr.25421) 212 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:34 smithi094 bash[59677]: audit 2023-10-09T08:45:34.198447+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:35.003 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:34 smithi016 systemd[1]: Stopping Ceph osd.0 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:45:35.003 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:34 smithi016 bash[37021]: debug 2023-10-09T08:45:34.757+0000 7efc39b4f700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T08:45:35.004 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:34 smithi016 bash[37021]: debug 2023-10-09T08:45:34.757+0000 7efc39b4f700 -1 osd.0 74 *** Got signal Terminated *** 2023-10-09T08:45:35.004 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:34 smithi016 bash[37021]: debug 2023-10-09T08:45:34.757+0000 7efc39b4f700 -1 osd.0 74 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T08:45:35.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:35 smithi016 bash[75179]: audit 2023-10-09T08:45:34.280193+0000 mon.a (mon.0) 264 : audit 0 from='client.? 172.21.15.16:0/1907444831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:35.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:35 smithi016 bash[75179]: cluster 2023-10-09T08:45:34.761860+0000 mon.a (mon.0) 265 : cluster 1 osd.0 marked itself down and dead 2023-10-09T08:45:35.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:35 smithi016 bash[77892]: audit 2023-10-09T08:45:34.280193+0000 mon.a (mon.0) 264 : audit 0 from='client.? 172.21.15.16:0/1907444831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:35.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:35 smithi016 bash[77892]: cluster 2023-10-09T08:45:34.761860+0000 mon.a (mon.0) 265 : cluster 1 osd.0 marked itself down and dead 2023-10-09T08:45:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:35 smithi094 bash[59677]: audit 2023-10-09T08:45:34.280193+0000 mon.a (mon.0) 264 : audit 0 from='client.? 172.21.15.16:0/1907444831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:35 smithi094 bash[59677]: cluster 2023-10-09T08:45:34.761860+0000 mon.a (mon.0) 265 : cluster 1 osd.0 marked itself down and dead 2023-10-09T08:45:36.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:45:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:45:35] "GET /metrics HTTP/1.1" 200 33759 "" "Prometheus/2.33.4" 2023-10-09T08:45:36.503 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[83168]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T08:45:36.503 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[83233]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T08:45:36.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[75179]: cluster 2023-10-09T08:45:35.199111+0000 mgr.y (mgr.25421) 213 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[75179]: cluster 2023-10-09T08:45:35.254801+0000 mon.a (mon.0) 266 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[75179]: cluster 2023-10-09T08:45:35.265937+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e75 with expected crc 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[75179]: cluster 2023-10-09T08:45:35.267063+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e75 with expected crc 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[75179]: cluster 2023-10-09T08:45:35.267924+0000 mon.a (mon.0) 267 : cluster 0 osdmap e75: 8 total, 7 up, 8 in 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[77892]: cluster 2023-10-09T08:45:35.199111+0000 mgr.y (mgr.25421) 213 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[77892]: cluster 2023-10-09T08:45:35.254801+0000 mon.a (mon.0) 266 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[77892]: cluster 2023-10-09T08:45:35.265937+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e75 with expected crc 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[77892]: cluster 2023-10-09T08:45:35.267063+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e75 with expected crc 2023-10-09T08:45:36.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:36 smithi016 bash[77892]: cluster 2023-10-09T08:45:35.267924+0000 mon.a (mon.0) 267 : cluster 0 osdmap e75: 8 total, 7 up, 8 in 2023-10-09T08:45:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:36 smithi094 bash[59677]: cluster 2023-10-09T08:45:35.199111+0000 mgr.y (mgr.25421) 213 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:36 smithi094 bash[59677]: cluster 2023-10-09T08:45:35.254801+0000 mon.a (mon.0) 266 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:45:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:36 smithi094 bash[59677]: cluster 2023-10-09T08:45:35.265937+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e75 with expected crc 2023-10-09T08:45:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:36 smithi094 bash[59677]: cluster 2023-10-09T08:45:35.267063+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e75 with expected crc 2023-10-09T08:45:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:36 smithi094 bash[59677]: cluster 2023-10-09T08:45:35.267924+0000 mon.a (mon.0) 267 : cluster 0 osdmap e75: 8 total, 7 up, 8 in 2023-10-09T08:45:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.269870+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.270491+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.270832+0000 mon.a (mon.0) 268 : cluster 0 osdmap e76: 8 total, 7 up, 8 in 2023-10-09T08:45:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.271494+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.271495+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.272679+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.273612+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.274064+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.275016+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: cluster 2023-10-09T08:45:36.275109+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:37 smithi094 bash[59677]: audit 2023-10-09T08:45:36.640775+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.16:0/1417942138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:37.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.269870+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.270491+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.270832+0000 mon.a (mon.0) 268 : cluster 0 osdmap e76: 8 total, 7 up, 8 in 2023-10-09T08:45:37.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.271494+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.271495+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.272679+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.273612+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.274064+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.275016+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: cluster 2023-10-09T08:45:36.275109+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[77892]: audit 2023-10-09T08:45:36.640775+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.16:0/1417942138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.269870+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.270491+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.270832+0000 mon.a (mon.0) 268 : cluster 0 osdmap e76: 8 total, 7 up, 8 in 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.271494+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.271495+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.272679+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.273612+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.274064+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.275016+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: cluster 2023-10-09T08:45:36.275109+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e76 with expected crc 2023-10-09T08:45:37.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:37 smithi016 bash[75179]: audit 2023-10-09T08:45:36.640775+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.16:0/1417942138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:38.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:38 smithi016 bash[75179]: cluster 2023-10-09T08:45:37.199806+0000 mgr.y (mgr.25421) 214 : cluster 0 pgmap v149: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:38.698 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:38 smithi016 bash[77892]: cluster 2023-10-09T08:45:37.199806+0000 mgr.y (mgr.25421) 214 : cluster 0 pgmap v149: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:38.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:38 smithi094 bash[59677]: cluster 2023-10-09T08:45:37.199806+0000 mgr.y (mgr.25421) 214 : cluster 0 pgmap v149: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:38.956 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:38 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T08:45:38.957 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:38 smithi016 systemd[1]: Stopped Ceph osd.0 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:45:39.253 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:38 smithi016 systemd[1]: Started Ceph osd.0 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:45:40.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[75179]: audit 2023-10-09T08:45:38.993923+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:40.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[75179]: audit 2023-10-09T08:45:39.002516+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[75179]: audit 2023-10-09T08:45:39.043102+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[75179]: audit 2023-10-09T08:45:39.056217+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.16:0/853384265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[75179]: cluster 2023-10-09T08:45:39.200298+0000 mgr.y (mgr.25421) 215 : cluster 0 pgmap v150: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[77892]: audit 2023-10-09T08:45:38.993923+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[77892]: audit 2023-10-09T08:45:39.002516+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[77892]: audit 2023-10-09T08:45:39.043102+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[77892]: audit 2023-10-09T08:45:39.056217+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.16:0/853384265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:40.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:39 smithi016 bash[77892]: cluster 2023-10-09T08:45:39.200298+0000 mgr.y (mgr.25421) 215 : cluster 0 pgmap v150: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:39 smithi094 bash[59677]: audit 2023-10-09T08:45:38.993923+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:39 smithi094 bash[59677]: audit 2023-10-09T08:45:39.002516+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:39 smithi094 bash[59677]: audit 2023-10-09T08:45:39.043102+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:39 smithi094 bash[59677]: audit 2023-10-09T08:45:39.056217+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.16:0/853384265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:40.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:39 smithi094 bash[59677]: cluster 2023-10-09T08:45:39.200298+0000 mgr.y (mgr.25421) 215 : cluster 0 pgmap v150: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:40.752 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T08:45:40.752 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T08:45:40.752 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T08:45:40.753 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T08:45:40.753 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T08:45:40.753 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T08:45:40.753 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T08:45:40.753 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:40 smithi016 bash[83438]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T08:45:41.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:41 smithi094 bash[59677]: cluster 2023-10-09T08:45:41.201096+0000 mgr.y (mgr.25421) 216 : cluster 0 pgmap v151: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:41.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:41 smithi094 bash[59677]: cluster 2023-10-09T08:45:41.252262+0000 mon.a (mon.0) 272 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T08:45:41.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:41 smithi016 bash[75179]: cluster 2023-10-09T08:45:41.201096+0000 mgr.y (mgr.25421) 216 : cluster 0 pgmap v151: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:41.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:41 smithi016 bash[75179]: cluster 2023-10-09T08:45:41.252262+0000 mon.a (mon.0) 272 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T08:45:41.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:41 smithi016 bash[77892]: cluster 2023-10-09T08:45:41.201096+0000 mgr.y (mgr.25421) 216 : cluster 0 pgmap v151: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:41.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:41 smithi016 bash[77892]: cluster 2023-10-09T08:45:41.252262+0000 mon.a (mon.0) 272 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T08:45:42.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:42 smithi094 bash[59677]: audit 2023-10-09T08:45:41.438305+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.16:0/44578968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:42.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:42 smithi016 bash[75179]: audit 2023-10-09T08:45:41.438305+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.16:0/44578968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:42.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:42 smithi016 bash[77892]: audit 2023-10-09T08:45:41.438305+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.16:0/44578968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:43.321 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:43 smithi016 bash[83762]: debug 2023-10-09T08:45:43.009+0000 7f7ab3a91640 -1 Falling back to public interface 2023-10-09T08:45:43.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:43 smithi094 bash[59677]: cluster 2023-10-09T08:45:43.201864+0000 mgr.y (mgr.25421) 217 : cluster 0 pgmap v152: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:43.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:43 smithi016 bash[75179]: cluster 2023-10-09T08:45:43.201864+0000 mgr.y (mgr.25421) 217 : cluster 0 pgmap v152: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:43.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:43 smithi016 bash[77892]: cluster 2023-10-09T08:45:43.201864+0000 mgr.y (mgr.25421) 217 : cluster 0 pgmap v152: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:45.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:44 smithi094 bash[59677]: audit 2023-10-09T08:45:43.810482+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.16:0/1879059109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:45.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:44 smithi016 bash[75179]: audit 2023-10-09T08:45:43.810482+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.16:0/1879059109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:45.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:44 smithi016 bash[77892]: audit 2023-10-09T08:45:43.810482+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.16:0/1879059109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:46.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:45 smithi016 bash[75179]: cluster 2023-10-09T08:45:45.202760+0000 mgr.y (mgr.25421) 218 : cluster 0 pgmap v153: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:46.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:45:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:45:45] "GET /metrics HTTP/1.1" 200 33759 "" "Prometheus/2.33.4" 2023-10-09T08:45:46.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:45 smithi016 bash[77892]: cluster 2023-10-09T08:45:45.202760+0000 mgr.y (mgr.25421) 218 : cluster 0 pgmap v153: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:46.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:45 smithi094 bash[59677]: cluster 2023-10-09T08:45:45.202760+0000 mgr.y (mgr.25421) 218 : cluster 0 pgmap v153: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:47.003 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:46 smithi016 bash[83762]: debug 2023-10-09T08:45:46.501+0000 7f7ab3a91640 -1 osd.0 74 log_to_monitors true 2023-10-09T08:45:47.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:47 smithi094 bash[59677]: audit 2023-10-09T08:45:46.136636+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:47 smithi094 bash[59677]: audit 2023-10-09T08:45:46.144202+0000 mon.a (mon.0) 274 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:47 smithi094 bash[59677]: audit 2023-10-09T08:45:46.263562+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.16:0/3984738861' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:47.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:47 smithi094 bash[59677]: audit 2023-10-09T08:45:46.509400+0000 mon.a (mon.0) 275 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:45:47.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:47 smithi094 bash[59677]: audit 2023-10-09T08:45:46.995912+0000 mon.a (mon.0) 276 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:47 smithi094 bash[59677]: audit 2023-10-09T08:45:47.003620+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.503 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[83762]: debug 2023-10-09T08:45:47.181+0000 7f7aa26f8700 -1 osd.0 74 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T08:45:47.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[75179]: audit 2023-10-09T08:45:46.136636+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[75179]: audit 2023-10-09T08:45:46.144202+0000 mon.a (mon.0) 274 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[75179]: audit 2023-10-09T08:45:46.263562+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.16:0/3984738861' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:47.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[75179]: audit 2023-10-09T08:45:46.509400+0000 mon.a (mon.0) 275 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:45:47.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[75179]: audit 2023-10-09T08:45:46.995912+0000 mon.a (mon.0) 276 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[75179]: audit 2023-10-09T08:45:47.003620+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[77892]: audit 2023-10-09T08:45:46.136636+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[77892]: audit 2023-10-09T08:45:46.144202+0000 mon.a (mon.0) 274 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[77892]: audit 2023-10-09T08:45:46.263562+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.16:0/3984738861' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:47.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[77892]: audit 2023-10-09T08:45:46.509400+0000 mon.a (mon.0) 275 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:45:47.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[77892]: audit 2023-10-09T08:45:46.995912+0000 mon.a (mon.0) 276 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:47.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:47 smithi016 bash[77892]: audit 2023-10-09T08:45:47.003620+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:48 smithi094 bash[59677]: cluster 2023-10-09T08:45:47.144372+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e77 with expected crc 2023-10-09T08:45:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:48 smithi094 bash[59677]: cluster 2023-10-09T08:45:47.144820+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e77 with expected crc 2023-10-09T08:45:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:48 smithi094 bash[59677]: audit 2023-10-09T08:45:47.146931+0000 mon.a (mon.0) 278 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:45:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:48 smithi094 bash[59677]: cluster 2023-10-09T08:45:47.147017+0000 mon.a (mon.0) 279 : cluster 0 osdmap e77: 8 total, 7 up, 8 in 2023-10-09T08:45:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:48 smithi094 bash[59677]: audit 2023-10-09T08:45:47.147867+0000 mon.a (mon.0) 280 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:45:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:48 smithi094 bash[59677]: cluster 2023-10-09T08:45:47.203483+0000 mgr.y (mgr.25421) 219 : cluster 0 pgmap v155: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:48.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[75179]: cluster 2023-10-09T08:45:47.144372+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e77 with expected crc 2023-10-09T08:45:48.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[75179]: cluster 2023-10-09T08:45:47.144820+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e77 with expected crc 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[75179]: audit 2023-10-09T08:45:47.146931+0000 mon.a (mon.0) 278 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[75179]: cluster 2023-10-09T08:45:47.147017+0000 mon.a (mon.0) 279 : cluster 0 osdmap e77: 8 total, 7 up, 8 in 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[75179]: audit 2023-10-09T08:45:47.147867+0000 mon.a (mon.0) 280 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[75179]: cluster 2023-10-09T08:45:47.203483+0000 mgr.y (mgr.25421) 219 : cluster 0 pgmap v155: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[77892]: cluster 2023-10-09T08:45:47.144372+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e77 with expected crc 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[77892]: cluster 2023-10-09T08:45:47.144820+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e77 with expected crc 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[77892]: audit 2023-10-09T08:45:47.146931+0000 mon.a (mon.0) 278 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[77892]: cluster 2023-10-09T08:45:47.147017+0000 mon.a (mon.0) 279 : cluster 0 osdmap e77: 8 total, 7 up, 8 in 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[77892]: audit 2023-10-09T08:45:47.147867+0000 mon.a (mon.0) 280 : audit 1 from='osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:45:48.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:48 smithi016 bash[77892]: cluster 2023-10-09T08:45:47.203483+0000 mgr.y (mgr.25421) 219 : cluster 0 pgmap v155: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:49 smithi094 bash[59677]: cluster 2023-10-09T08:45:48.144424+0000 mon.a (mon.0) 281 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:45:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:49 smithi094 bash[59677]: cluster 2023-10-09T08:45:48.148574+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-09T08:45:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:49 smithi094 bash[59677]: cluster 2023-10-09T08:45:48.151350+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e78 with expected crc 2023-10-09T08:45:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:49 smithi094 bash[59677]: cluster 2023-10-09T08:45:48.153297+0000 mon.a (mon.0) 282 : cluster 1 osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916] boot 2023-10-09T08:45:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:49 smithi094 bash[59677]: cluster 2023-10-09T08:45:48.153380+0000 mon.a (mon.0) 283 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-10-09T08:45:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:49 smithi094 bash[59677]: audit 2023-10-09T08:45:48.154214+0000 mon.a (mon.0) 284 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:45:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:49 smithi094 bash[59677]: audit 2023-10-09T08:45:48.653391+0000 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.16:0/3143490409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:49.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[77892]: cluster 2023-10-09T08:45:48.144424+0000 mon.a (mon.0) 281 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[77892]: cluster 2023-10-09T08:45:48.148574+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[77892]: cluster 2023-10-09T08:45:48.151350+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e78 with expected crc 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[77892]: cluster 2023-10-09T08:45:48.153297+0000 mon.a (mon.0) 282 : cluster 1 osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916] boot 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[77892]: cluster 2023-10-09T08:45:48.153380+0000 mon.a (mon.0) 283 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[77892]: audit 2023-10-09T08:45:48.154214+0000 mon.a (mon.0) 284 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[77892]: audit 2023-10-09T08:45:48.653391+0000 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.16:0/3143490409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[75179]: cluster 2023-10-09T08:45:48.144424+0000 mon.a (mon.0) 281 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:45:49.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[75179]: cluster 2023-10-09T08:45:48.148574+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-09T08:45:49.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[75179]: cluster 2023-10-09T08:45:48.151350+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e78 with expected crc 2023-10-09T08:45:49.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[75179]: cluster 2023-10-09T08:45:48.153297+0000 mon.a (mon.0) 282 : cluster 1 osd.0 [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916] boot 2023-10-09T08:45:49.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[75179]: cluster 2023-10-09T08:45:48.153380+0000 mon.a (mon.0) 283 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-10-09T08:45:49.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[75179]: audit 2023-10-09T08:45:48.154214+0000 mon.a (mon.0) 284 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:45:49.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:49 smithi016 bash[75179]: audit 2023-10-09T08:45:48.653391+0000 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.16:0/3143490409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.159070+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.160503+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.163347+0000 mon.a (mon.0) 286 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-10-09T08:45:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.163599+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.163739+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.164466+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.165971+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.166040+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.166222+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.166836+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.168979+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.169469+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.170400+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.172145+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: cluster 2023-10-09T08:45:49.204336+0000 mgr.y (mgr.25421) 220 : cluster 0 pgmap v158: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:50.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: audit 2023-10-09T08:45:49.204505+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:50.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:50 smithi094 bash[59677]: audit 2023-10-09T08:45:49.205929+0000 mon.a (mon.0) 288 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.159070+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.160503+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.163347+0000 mon.a (mon.0) 286 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.163599+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.163739+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.164466+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.165971+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.166040+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.166222+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.166836+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.168979+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.169469+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.170400+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.172145+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: cluster 2023-10-09T08:45:49.204336+0000 mgr.y (mgr.25421) 220 : cluster 0 pgmap v158: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: audit 2023-10-09T08:45:49.204505+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[75179]: audit 2023-10-09T08:45:49.205929+0000 mon.a (mon.0) 288 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.159070+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.160503+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.163347+0000 mon.a (mon.0) 286 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.163599+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.163739+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.164466+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.165971+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.166040+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.166222+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.166836+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.168979+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.169469+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.170400+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.172145+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T08:45:50.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: cluster 2023-10-09T08:45:49.204336+0000 mgr.y (mgr.25421) 220 : cluster 0 pgmap v158: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T08:45:50.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: audit 2023-10-09T08:45:49.204505+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:50.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:50 smithi016 bash[77892]: audit 2023-10-09T08:45:49.205929+0000 mon.a (mon.0) 288 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:45:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:51 smithi094 bash[59677]: audit 2023-10-09T08:45:51.026395+0000 mon.a (mon.0) 289 : audit 0 from='client.? 172.21.15.16:0/3328200949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:51.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:51 smithi016 bash[75179]: audit 2023-10-09T08:45:51.026395+0000 mon.a (mon.0) 289 : audit 0 from='client.? 172.21.15.16:0/3328200949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:51.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:51 smithi016 bash[77892]: audit 2023-10-09T08:45:51.026395+0000 mon.a (mon.0) 289 : audit 0 from='client.? 172.21.15.16:0/3328200949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:52.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:52 smithi094 bash[59677]: cluster 2023-10-09T08:45:51.205623+0000 mgr.y (mgr.25421) 221 : cluster 0 pgmap v159: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:45:52.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:52 smithi016 bash[75179]: cluster 2023-10-09T08:45:51.205623+0000 mgr.y (mgr.25421) 221 : cluster 0 pgmap v159: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:45:52.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:52 smithi016 bash[77892]: cluster 2023-10-09T08:45:51.205623+0000 mgr.y (mgr.25421) 221 : cluster 0 pgmap v159: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:45:53.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:53 smithi016 bash[75179]: cluster 2023-10-09T08:45:52.204185+0000 mon.a (mon.0) 290 : cluster 3 Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:45:53.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:53 smithi016 bash[77892]: cluster 2023-10-09T08:45:52.204185+0000 mon.a (mon.0) 290 : cluster 3 Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:45:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:53 smithi094 bash[59677]: cluster 2023-10-09T08:45:52.204185+0000 mon.a (mon.0) 290 : cluster 3 Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:45:54.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[75179]: cluster 2023-10-09T08:45:53.206267+0000 mgr.y (mgr.25421) 222 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:54.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[75179]: cluster 2023-10-09T08:45:53.212094+0000 mon.a (mon.0) 291 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T08:45:54.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[75179]: cluster 2023-10-09T08:45:53.212145+0000 mon.a (mon.0) 292 : cluster 1 Cluster is now healthy 2023-10-09T08:45:54.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[75179]: audit 2023-10-09T08:45:53.400571+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.16:0/2562073051' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:54.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[77892]: cluster 2023-10-09T08:45:53.206267+0000 mgr.y (mgr.25421) 222 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:54.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[77892]: cluster 2023-10-09T08:45:53.212094+0000 mon.a (mon.0) 291 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T08:45:54.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[77892]: cluster 2023-10-09T08:45:53.212145+0000 mon.a (mon.0) 292 : cluster 1 Cluster is now healthy 2023-10-09T08:45:54.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:54 smithi016 bash[77892]: audit 2023-10-09T08:45:53.400571+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.16:0/2562073051' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:54 smithi094 bash[59677]: cluster 2023-10-09T08:45:53.206267+0000 mgr.y (mgr.25421) 222 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:54 smithi094 bash[59677]: cluster 2023-10-09T08:45:53.212094+0000 mon.a (mon.0) 291 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T08:45:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:54 smithi094 bash[59677]: cluster 2023-10-09T08:45:53.212145+0000 mon.a (mon.0) 292 : cluster 1 Cluster is now healthy 2023-10-09T08:45:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:54 smithi094 bash[59677]: audit 2023-10-09T08:45:53.400571+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.16:0/2562073051' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:56.004 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:45:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:45:55] "GET /metrics HTTP/1.1" 200 33681 "" "Prometheus/2.33.4" 2023-10-09T08:45:56.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:56 smithi094 bash[59677]: cluster 2023-10-09T08:45:55.207210+0000 mgr.y (mgr.25421) 223 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:56.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:56 smithi094 bash[59677]: audit 2023-10-09T08:45:55.773710+0000 mon.a (mon.0) 293 : audit 0 from='client.? 172.21.15.16:0/3680665455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:56.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:56 smithi016 bash[75179]: cluster 2023-10-09T08:45:55.207210+0000 mgr.y (mgr.25421) 223 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:56.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:56 smithi016 bash[75179]: audit 2023-10-09T08:45:55.773710+0000 mon.a (mon.0) 293 : audit 0 from='client.? 172.21.15.16:0/3680665455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:56.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:56 smithi016 bash[77892]: cluster 2023-10-09T08:45:55.207210+0000 mgr.y (mgr.25421) 223 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:56.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:56 smithi016 bash[77892]: audit 2023-10-09T08:45:55.773710+0000 mon.a (mon.0) 293 : audit 0 from='client.? 172.21.15.16:0/3680665455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:57.862 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.555357+0000 mon.a (mon.0) 294 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.862 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.564936+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.566711+0000 mon.a (mon.0) 296 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.568042+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.577594+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.660779+0000 mon.a (mon.0) 299 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.665136+0000 mon.a (mon.0) 300 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.667496+0000 mon.a (mon.0) 301 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.669386+0000 mon.a (mon.0) 302 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.671907+0000 mon.a (mon.0) 303 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: audit 2023-10-09T08:45:56.672225+0000 mgr.y (mgr.25421) 224 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T08:45:57.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: cephadm 2023-10-09T08:45:56.673771+0000 mgr.y (mgr.25421) 225 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[75179]: cluster 2023-10-09T08:45:57.207892+0000 mgr.y (mgr.25421) 226 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.555357+0000 mon.a (mon.0) 294 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.564936+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.566711+0000 mon.a (mon.0) 296 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.568042+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.577594+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.660779+0000 mon.a (mon.0) 299 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:57.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.665136+0000 mon.a (mon.0) 300 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.667496+0000 mon.a (mon.0) 301 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.669386+0000 mon.a (mon.0) 302 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.671907+0000 mon.a (mon.0) 303 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T08:45:57.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: audit 2023-10-09T08:45:56.672225+0000 mgr.y (mgr.25421) 224 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T08:45:57.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: cephadm 2023-10-09T08:45:56.673771+0000 mgr.y (mgr.25421) 225 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T08:45:57.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:57 smithi016 bash[77892]: cluster 2023-10-09T08:45:57.207892+0000 mgr.y (mgr.25421) 226 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:57.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.555357+0000 mon.a (mon.0) 294 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.564936+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.566711+0000 mon.a (mon.0) 296 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:45:57.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.568042+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:45:57.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.577594+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.660779+0000 mon.a (mon.0) 299 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.665136+0000 mon.a (mon.0) 300 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.667496+0000 mon.a (mon.0) 301 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.669386+0000 mon.a (mon.0) 302 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.671907+0000 mon.a (mon.0) 303 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: audit 2023-10-09T08:45:56.672225+0000 mgr.y (mgr.25421) 224 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: cephadm 2023-10-09T08:45:56.673771+0000 mgr.y (mgr.25421) 225 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T08:45:57.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:57 smithi094 bash[59677]: cluster 2023-10-09T08:45:57.207892+0000 mgr.y (mgr.25421) 226 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:45:58.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:58 smithi094 bash[59677]: audit 2023-10-09T08:45:58.151849+0000 mon.a (mon.0) 304 : audit 0 from='client.? 172.21.15.16:0/2910653220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:59.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:58 smithi016 bash[75179]: audit 2023-10-09T08:45:58.151849+0000 mon.a (mon.0) 304 : audit 0 from='client.? 172.21.15.16:0/2910653220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:45:59.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:58 smithi016 bash[77892]: audit 2023-10-09T08:45:58.151849+0000 mon.a (mon.0) 304 : audit 0 from='client.? 172.21.15.16:0/2910653220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:00.159 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[75179]: cephadm 2023-10-09T08:45:58.893167+0000 mgr.y (mgr.25421) 227 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T08:46:00.159 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[75179]: audit 2023-10-09T08:45:58.901748+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:00.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[75179]: audit 2023-10-09T08:45:58.903336+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:46:00.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[75179]: audit 2023-10-09T08:45:58.904485+0000 mon.a (mon.0) 307 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:00.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[75179]: cephadm 2023-10-09T08:45:58.907030+0000 mgr.y (mgr.25421) 228 : cephadm 1 Deploying daemon osd.1 on smithi016 2023-10-09T08:46:00.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[75179]: cluster 2023-10-09T08:45:59.208463+0000 mgr.y (mgr.25421) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:00.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[75179]: audit 2023-10-09T08:45:59.240044+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:00.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[77892]: cephadm 2023-10-09T08:45:58.893167+0000 mgr.y (mgr.25421) 227 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T08:46:00.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[77892]: audit 2023-10-09T08:45:58.901748+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:00.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[77892]: audit 2023-10-09T08:45:58.903336+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:46:00.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[77892]: audit 2023-10-09T08:45:58.904485+0000 mon.a (mon.0) 307 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:00.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[77892]: cephadm 2023-10-09T08:45:58.907030+0000 mgr.y (mgr.25421) 228 : cephadm 1 Deploying daemon osd.1 on smithi016 2023-10-09T08:46:00.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[77892]: cluster 2023-10-09T08:45:59.208463+0000 mgr.y (mgr.25421) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:00.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:45:59 smithi016 bash[77892]: audit 2023-10-09T08:45:59.240044+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:00.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:59 smithi094 bash[59677]: cephadm 2023-10-09T08:45:58.893167+0000 mgr.y (mgr.25421) 227 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T08:46:00.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:59 smithi094 bash[59677]: audit 2023-10-09T08:45:58.901748+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:00.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:59 smithi094 bash[59677]: audit 2023-10-09T08:45:58.903336+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:46:00.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:59 smithi094 bash[59677]: audit 2023-10-09T08:45:58.904485+0000 mon.a (mon.0) 307 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:00.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:59 smithi094 bash[59677]: cephadm 2023-10-09T08:45:58.907030+0000 mgr.y (mgr.25421) 228 : cephadm 1 Deploying daemon osd.1 on smithi016 2023-10-09T08:46:00.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:59 smithi094 bash[59677]: cluster 2023-10-09T08:45:59.208463+0000 mgr.y (mgr.25421) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:00.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:45:59 smithi094 bash[59677]: audit 2023-10-09T08:45:59.240044+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:01.183 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:00 smithi016 bash[75179]: audit 2023-10-09T08:46:00.516255+0000 mon.a (mon.0) 309 : audit 0 from='client.? 172.21.15.16:0/4058133880' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:01.183 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:00 smithi016 bash[77892]: audit 2023-10-09T08:46:00.516255+0000 mon.a (mon.0) 309 : audit 0 from='client.? 172.21.15.16:0/4058133880' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:01.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:00 smithi094 bash[59677]: audit 2023-10-09T08:46:00.516255+0000 mon.a (mon.0) 309 : audit 0 from='client.? 172.21.15.16:0/4058133880' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:02.183 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:01 smithi016 bash[75179]: cluster 2023-10-09T08:46:01.209469+0000 mgr.y (mgr.25421) 230 : cluster 0 pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:02.183 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:01 smithi016 bash[77892]: cluster 2023-10-09T08:46:01.209469+0000 mgr.y (mgr.25421) 230 : cluster 0 pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:02.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:01 smithi094 bash[59677]: cluster 2023-10-09T08:46:01.209469+0000 mgr.y (mgr.25421) 230 : cluster 0 pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:03.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:02 smithi016 bash[75179]: audit 2023-10-09T08:46:02.881672+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.16:0/3658686600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:03.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:02 smithi094 bash[59677]: audit 2023-10-09T08:46:02.881672+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.16:0/3658686600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:03.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:03 smithi016 bash[77892]: audit 2023-10-09T08:46:02.881672+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.16:0/3658686600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:04.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:03 smithi094 bash[59677]: cluster 2023-10-09T08:46:03.210057+0000 mgr.y (mgr.25421) 231 : cluster 0 pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:04.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:03 smithi016 bash[75179]: cluster 2023-10-09T08:46:03.210057+0000 mgr.y (mgr.25421) 231 : cluster 0 pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:04.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:03 smithi016 bash[77892]: cluster 2023-10-09T08:46:03.210057+0000 mgr.y (mgr.25421) 231 : cluster 0 pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:04.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:03 smithi016 systemd[1]: Stopping Ceph osd.1 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:46:04.254 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:03 smithi016 bash[41192]: debug 2023-10-09T08:46:03.969+0000 7f04a2216700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T08:46:04.254 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:03 smithi016 bash[41192]: debug 2023-10-09T08:46:03.969+0000 7f04a2216700 -1 osd.1 79 *** Got signal Terminated *** 2023-10-09T08:46:04.254 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:03 smithi016 bash[41192]: debug 2023-10-09T08:46:03.969+0000 7f04a2216700 -1 osd.1 79 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T08:46:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:04 smithi094 bash[59677]: cluster 2023-10-09T08:46:03.974085+0000 mon.a (mon.0) 310 : cluster 1 osd.1 marked itself down and dead 2023-10-09T08:46:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:04 smithi094 bash[59677]: audit 2023-10-09T08:46:04.207570+0000 mon.a (mon.0) 311 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:04 smithi094 bash[59677]: audit 2023-10-09T08:46:04.209370+0000 mon.a (mon.0) 312 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:05.248 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:04 smithi016 bash[75179]: cluster 2023-10-09T08:46:03.974085+0000 mon.a (mon.0) 310 : cluster 1 osd.1 marked itself down and dead 2023-10-09T08:46:05.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:04 smithi016 bash[75179]: audit 2023-10-09T08:46:04.207570+0000 mon.a (mon.0) 311 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:05.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:04 smithi016 bash[75179]: audit 2023-10-09T08:46:04.209370+0000 mon.a (mon.0) 312 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:05.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:04 smithi016 bash[77892]: cluster 2023-10-09T08:46:03.974085+0000 mon.a (mon.0) 310 : cluster 1 osd.1 marked itself down and dead 2023-10-09T08:46:05.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:04 smithi016 bash[77892]: audit 2023-10-09T08:46:04.207570+0000 mon.a (mon.0) 311 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:05.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:04 smithi016 bash[77892]: audit 2023-10-09T08:46:04.209370+0000 mon.a (mon.0) 312 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:05.911 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:46:05] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-09T08:46:05.911 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[86045]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T08:46:05.911 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[86135]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T08:46:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:05 smithi094 bash[59677]: cluster 2023-10-09T08:46:04.924296+0000 mon.a (mon.0) 313 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:46:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:05 smithi094 bash[59677]: cluster 2023-10-09T08:46:04.934540+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T08:46:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:05 smithi094 bash[59677]: cluster 2023-10-09T08:46:04.935014+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T08:46:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:05 smithi094 bash[59677]: cluster 2023-10-09T08:46:04.935308+0000 mon.a (mon.0) 314 : cluster 0 osdmap e80: 8 total, 7 up, 8 in 2023-10-09T08:46:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:05 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.211152+0000 mgr.y (mgr.25421) 232 : cluster 0 pgmap v167: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:05 smithi094 bash[59677]: audit 2023-10-09T08:46:05.270206+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.16:0/3095810656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:06.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[75179]: cluster 2023-10-09T08:46:04.924296+0000 mon.a (mon.0) 313 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:46:06.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[75179]: cluster 2023-10-09T08:46:04.934540+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T08:46:06.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[75179]: cluster 2023-10-09T08:46:04.935014+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T08:46:06.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[75179]: cluster 2023-10-09T08:46:04.935308+0000 mon.a (mon.0) 314 : cluster 0 osdmap e80: 8 total, 7 up, 8 in 2023-10-09T08:46:06.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.211152+0000 mgr.y (mgr.25421) 232 : cluster 0 pgmap v167: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:06.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[75179]: audit 2023-10-09T08:46:05.270206+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.16:0/3095810656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:06.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[77892]: cluster 2023-10-09T08:46:04.924296+0000 mon.a (mon.0) 313 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:46:06.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[77892]: cluster 2023-10-09T08:46:04.934540+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T08:46:06.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[77892]: cluster 2023-10-09T08:46:04.935014+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T08:46:06.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[77892]: cluster 2023-10-09T08:46:04.935308+0000 mon.a (mon.0) 314 : cluster 0 osdmap e80: 8 total, 7 up, 8 in 2023-10-09T08:46:06.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.211152+0000 mgr.y (mgr.25421) 232 : cluster 0 pgmap v167: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:06.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:05 smithi016 bash[77892]: audit 2023-10-09T08:46:05.270206+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.16:0/3095810656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:07.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.933358+0000 mon.a (mon.0) 315 : cluster 3 Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T08:46:07.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.947762+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.949587+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.951146+0000 mon.a (mon.0) 316 : cluster 0 osdmap e81: 8 total, 7 up, 8 in 2023-10-09T08:46:07.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.951897+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.951916+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.953667+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.953728+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.953735+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.954331+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:06 smithi094 bash[59677]: cluster 2023-10-09T08:46:05.957616+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.933358+0000 mon.a (mon.0) 315 : cluster 3 Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T08:46:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.947762+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.949587+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.951146+0000 mon.a (mon.0) 316 : cluster 0 osdmap e81: 8 total, 7 up, 8 in 2023-10-09T08:46:07.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.951897+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.951916+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.953667+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.953728+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.953735+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.954331+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[75179]: cluster 2023-10-09T08:46:05.957616+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.933358+0000 mon.a (mon.0) 315 : cluster 3 Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.947762+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.949587+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.951146+0000 mon.a (mon.0) 316 : cluster 0 osdmap e81: 8 total, 7 up, 8 in 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.951897+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.951916+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.953667+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.953728+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.953735+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.954331+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:07.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:06 smithi016 bash[77892]: cluster 2023-10-09T08:46:05.957616+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T08:46:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:07 smithi094 bash[59677]: cluster 2023-10-09T08:46:07.212195+0000 mgr.y (mgr.25421) 233 : cluster 0 pgmap v169: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:07 smithi094 bash[59677]: audit 2023-10-09T08:46:07.630622+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.16:0/2605785566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:08.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:07 smithi016 bash[75179]: cluster 2023-10-09T08:46:07.212195+0000 mgr.y (mgr.25421) 233 : cluster 0 pgmap v169: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:08.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:07 smithi016 bash[75179]: audit 2023-10-09T08:46:07.630622+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.16:0/2605785566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:08.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:07 smithi016 bash[77892]: cluster 2023-10-09T08:46:07.212195+0000 mgr.y (mgr.25421) 233 : cluster 0 pgmap v169: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:08.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:07 smithi016 bash[77892]: audit 2023-10-09T08:46:07.630622+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.16:0/2605785566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:08.548 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:08 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T08:46:08.548 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:08 smithi016 systemd[1]: Stopped Ceph osd.1 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:46:08.943 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:08 smithi016 systemd[1]: Started Ceph osd.1 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:46:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:08 smithi094 bash[59677]: cluster 2023-10-09T08:46:07.928892+0000 mon.a (mon.0) 318 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:08 smithi094 bash[59677]: audit 2023-10-09T08:46:08.582450+0000 mon.a (mon.0) 319 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:08 smithi094 bash[59677]: audit 2023-10-09T08:46:08.588759+0000 mon.a (mon.0) 320 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:08 smithi094 bash[59677]: audit 2023-10-09T08:46:08.594853+0000 mon.a (mon.0) 321 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:09.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[77892]: cluster 2023-10-09T08:46:07.928892+0000 mon.a (mon.0) 318 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:09.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[77892]: audit 2023-10-09T08:46:08.582450+0000 mon.a (mon.0) 319 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[77892]: audit 2023-10-09T08:46:08.588759+0000 mon.a (mon.0) 320 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[77892]: audit 2023-10-09T08:46:08.594853+0000 mon.a (mon.0) 321 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:09.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[75179]: cluster 2023-10-09T08:46:07.928892+0000 mon.a (mon.0) 318 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:09.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[75179]: audit 2023-10-09T08:46:08.582450+0000 mon.a (mon.0) 319 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[75179]: audit 2023-10-09T08:46:08.588759+0000 mon.a (mon.0) 320 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:08 smithi016 bash[75179]: audit 2023-10-09T08:46:08.594853+0000 mon.a (mon.0) 321 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:09.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:09 smithi016 bash[77892]: audit 2023-10-09T08:46:08.970742+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:09 smithi016 bash[77892]: audit 2023-10-09T08:46:08.978099+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:09.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:09 smithi016 bash[77892]: cluster 2023-10-09T08:46:09.213054+0000 mgr.y (mgr.25421) 234 : cluster 0 pgmap v170: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:09 smithi094 bash[59677]: audit 2023-10-09T08:46:08.970742+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:09 smithi094 bash[59677]: audit 2023-10-09T08:46:08.978099+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:09 smithi094 bash[59677]: cluster 2023-10-09T08:46:09.213054+0000 mgr.y (mgr.25421) 234 : cluster 0 pgmap v170: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T08:46:10.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[86319]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T08:46:10.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:09 smithi016 bash[75179]: audit 2023-10-09T08:46:08.970742+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:10.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:09 smithi016 bash[75179]: audit 2023-10-09T08:46:08.978099+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:10.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:09 smithi016 bash[75179]: cluster 2023-10-09T08:46:09.213054+0000 mgr.y (mgr.25421) 234 : cluster 0 pgmap v170: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:11.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:10 smithi094 bash[59677]: audit 2023-10-09T08:46:09.999445+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.16:0/3615637934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:11.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[75179]: audit 2023-10-09T08:46:09.999445+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.16:0/3615637934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:11.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:10 smithi016 bash[77892]: audit 2023-10-09T08:46:09.999445+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.16:0/3615637934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:12.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:11 smithi094 bash[59677]: cluster 2023-10-09T08:46:11.214162+0000 mgr.y (mgr.25421) 235 : cluster 0 pgmap v171: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:12.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:11 smithi016 bash[75179]: cluster 2023-10-09T08:46:11.214162+0000 mgr.y (mgr.25421) 235 : cluster 0 pgmap v171: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:12.253 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:12 smithi016 bash[86637]: debug 2023-10-09T08:46:11.997+0000 7f3e5bace700 1 -- 172.21.15.16:0/3020856599 <== mon.0 v2:172.21.15.16:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557f669c04e0 con 0x557f66a1b000 2023-10-09T08:46:12.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:11 smithi016 bash[77892]: cluster 2023-10-09T08:46:11.214162+0000 mgr.y (mgr.25421) 235 : cluster 0 pgmap v171: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:12.987 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:12 smithi016 bash[86637]: debug 2023-10-09T08:46:12.553+0000 7f3e64c54640 -1 Falling back to public interface 2023-10-09T08:46:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:12 smithi094 bash[59677]: cluster 2023-10-09T08:46:11.979165+0000 mon.a (mon.0) 324 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2023-10-09T08:46:13.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:12 smithi094 bash[59677]: audit 2023-10-09T08:46:12.384507+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.16:0/3912545572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:13.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:12 smithi016 bash[75179]: cluster 2023-10-09T08:46:11.979165+0000 mon.a (mon.0) 324 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2023-10-09T08:46:13.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:12 smithi016 bash[75179]: audit 2023-10-09T08:46:12.384507+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.16:0/3912545572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:13.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:12 smithi016 bash[77892]: cluster 2023-10-09T08:46:11.979165+0000 mon.a (mon.0) 324 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2023-10-09T08:46:13.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:12 smithi016 bash[77892]: audit 2023-10-09T08:46:12.384507+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.16:0/3912545572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:14.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:13 smithi016 bash[75179]: cluster 2023-10-09T08:46:13.215039+0000 mgr.y (mgr.25421) 236 : cluster 0 pgmap v172: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:14.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:13 smithi016 bash[75179]: cluster 2023-10-09T08:46:13.804929+0000 mon.a (mon.0) 325 : cluster 3 Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T08:46:14.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:13 smithi016 bash[77892]: cluster 2023-10-09T08:46:13.215039+0000 mgr.y (mgr.25421) 236 : cluster 0 pgmap v172: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:14.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:13 smithi016 bash[77892]: cluster 2023-10-09T08:46:13.804929+0000 mon.a (mon.0) 325 : cluster 3 Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T08:46:14.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:13 smithi094 bash[59677]: cluster 2023-10-09T08:46:13.215039+0000 mgr.y (mgr.25421) 236 : cluster 0 pgmap v172: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:14.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:13 smithi094 bash[59677]: cluster 2023-10-09T08:46:13.804929+0000 mon.a (mon.0) 325 : cluster 3 Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T08:46:15.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:14 smithi016 bash[75179]: audit 2023-10-09T08:46:14.759681+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.16:0/3290514880' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:15.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:14 smithi016 bash[77892]: audit 2023-10-09T08:46:14.759681+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.16:0/3290514880' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:15.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:14 smithi094 bash[59677]: audit 2023-10-09T08:46:14.759681+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.16:0/3290514880' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:16.003 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:46:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:46:15] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-09T08:46:16.003 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:15 smithi016 bash[86637]: debug 2023-10-09T08:46:15.745+0000 7f3e64c54640 -1 osd.1 79 log_to_monitors true 2023-10-09T08:46:16.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:16 smithi094 bash[59677]: cluster 2023-10-09T08:46:15.216015+0000 mgr.y (mgr.25421) 237 : cluster 0 pgmap v173: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:16.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:16 smithi094 bash[59677]: audit 2023-10-09T08:46:15.639584+0000 mon.a (mon.0) 326 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:16.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:16 smithi094 bash[59677]: audit 2023-10-09T08:46:15.647575+0000 mon.a (mon.0) 327 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:16.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:16 smithi094 bash[59677]: audit 2023-10-09T08:46:15.753978+0000 mon.c (mon.2) 17 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:46:16.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:16 smithi094 bash[59677]: audit 2023-10-09T08:46:15.754433+0000 mon.a (mon.0) 328 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:46:16.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:15 smithi016 bash[75179]: cluster 2023-10-09T08:46:15.216015+0000 mgr.y (mgr.25421) 237 : cluster 0 pgmap v173: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:16.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:15 smithi016 bash[75179]: audit 2023-10-09T08:46:15.639584+0000 mon.a (mon.0) 326 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:16.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[75179]: audit 2023-10-09T08:46:15.647575+0000 mon.a (mon.0) 327 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:16.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[75179]: audit 2023-10-09T08:46:15.753978+0000 mon.c (mon.2) 17 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:46:16.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[75179]: audit 2023-10-09T08:46:15.754433+0000 mon.a (mon.0) 328 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:46:16.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[77892]: cluster 2023-10-09T08:46:15.216015+0000 mgr.y (mgr.25421) 237 : cluster 0 pgmap v173: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:16.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[77892]: audit 2023-10-09T08:46:15.639584+0000 mon.a (mon.0) 326 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:16.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[77892]: audit 2023-10-09T08:46:15.647575+0000 mon.a (mon.0) 327 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:16.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[77892]: audit 2023-10-09T08:46:15.753978+0000 mon.c (mon.2) 17 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:46:16.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[77892]: audit 2023-10-09T08:46:15.754433+0000 mon.a (mon.0) 328 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:46:17.003 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 08:46:16 smithi016 bash[86637]: debug 2023-10-09T08:46:16.693+0000 7f3e558bf700 -1 osd.1 79 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T08:46:17.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: audit 2023-10-09T08:46:16.440053+0000 mon.a (mon.0) 329 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:17.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: audit 2023-10-09T08:46:16.447026+0000 mon.a (mon.0) 330 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:17.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: cluster 2023-10-09T08:46:16.649631+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:17.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: audit 2023-10-09T08:46:16.653400+0000 mon.a (mon.0) 331 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:46:17.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: cluster 2023-10-09T08:46:16.653614+0000 mon.a (mon.0) 332 : cluster 0 osdmap e82: 8 total, 7 up, 8 in 2023-10-09T08:46:17.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: audit 2023-10-09T08:46:16.654512+0000 mon.c (mon.2) 18 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:17.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: audit 2023-10-09T08:46:16.655093+0000 mon.a (mon.0) 333 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:17.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: audit 2023-10-09T08:46:17.127176+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.16:0/2652448740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:17.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:17 smithi094 bash[59677]: cluster 2023-10-09T08:46:17.216775+0000 mgr.y (mgr.25421) 238 : cluster 0 pgmap v175: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: audit 2023-10-09T08:46:16.440053+0000 mon.a (mon.0) 329 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: audit 2023-10-09T08:46:16.447026+0000 mon.a (mon.0) 330 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: cluster 2023-10-09T08:46:16.649631+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: audit 2023-10-09T08:46:16.653400+0000 mon.a (mon.0) 331 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: cluster 2023-10-09T08:46:16.653614+0000 mon.a (mon.0) 332 : cluster 0 osdmap e82: 8 total, 7 up, 8 in 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: audit 2023-10-09T08:46:16.654512+0000 mon.c (mon.2) 18 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: audit 2023-10-09T08:46:16.655093+0000 mon.a (mon.0) 333 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: audit 2023-10-09T08:46:17.127176+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.16:0/2652448740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[75179]: cluster 2023-10-09T08:46:17.216775+0000 mgr.y (mgr.25421) 238 : cluster 0 pgmap v175: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: audit 2023-10-09T08:46:16.440053+0000 mon.a (mon.0) 329 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: audit 2023-10-09T08:46:16.447026+0000 mon.a (mon.0) 330 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: cluster 2023-10-09T08:46:16.649631+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: audit 2023-10-09T08:46:16.653400+0000 mon.a (mon.0) 331 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: cluster 2023-10-09T08:46:16.653614+0000 mon.a (mon.0) 332 : cluster 0 osdmap e82: 8 total, 7 up, 8 in 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: audit 2023-10-09T08:46:16.654512+0000 mon.c (mon.2) 18 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: audit 2023-10-09T08:46:16.655093+0000 mon.a (mon.0) 333 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:17.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: audit 2023-10-09T08:46:17.127176+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.16:0/2652448740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:17.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:17 smithi016 bash[77892]: cluster 2023-10-09T08:46:17.216775+0000 mgr.y (mgr.25421) 238 : cluster 0 pgmap v175: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:18 smithi094 bash[59677]: cluster 2023-10-09T08:46:16.650917+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:18 smithi094 bash[59677]: cluster 2023-10-09T08:46:17.649018+0000 mon.a (mon.0) 334 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:46:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:18 smithi094 bash[59677]: cluster 2023-10-09T08:46:17.652847+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:18 smithi094 bash[59677]: cluster 2023-10-09T08:46:17.657632+0000 mon.a (mon.0) 335 : cluster 1 osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354] boot 2023-10-09T08:46:18.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:18 smithi094 bash[59677]: cluster 2023-10-09T08:46:17.657712+0000 mon.a (mon.0) 336 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T08:46:18.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:18 smithi094 bash[59677]: audit 2023-10-09T08:46:17.661740+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:46:18.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:18 smithi094 bash[59677]: cluster 2023-10-09T08:46:17.669519+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[75179]: cluster 2023-10-09T08:46:16.650917+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[75179]: cluster 2023-10-09T08:46:17.649018+0000 mon.a (mon.0) 334 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:46:18.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[75179]: cluster 2023-10-09T08:46:17.652847+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[75179]: cluster 2023-10-09T08:46:17.657632+0000 mon.a (mon.0) 335 : cluster 1 osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354] boot 2023-10-09T08:46:18.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[75179]: cluster 2023-10-09T08:46:17.657712+0000 mon.a (mon.0) 336 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T08:46:18.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[75179]: audit 2023-10-09T08:46:17.661740+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:46:18.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[75179]: cluster 2023-10-09T08:46:17.669519+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[77892]: cluster 2023-10-09T08:46:16.650917+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[77892]: cluster 2023-10-09T08:46:17.649018+0000 mon.a (mon.0) 334 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:46:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[77892]: cluster 2023-10-09T08:46:17.652847+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[77892]: cluster 2023-10-09T08:46:17.657632+0000 mon.a (mon.0) 335 : cluster 1 osd.1 [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354] boot 2023-10-09T08:46:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[77892]: cluster 2023-10-09T08:46:17.657712+0000 mon.a (mon.0) 336 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T08:46:18.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[77892]: audit 2023-10-09T08:46:17.661740+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:46:18.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:18 smithi016 bash[77892]: cluster 2023-10-09T08:46:17.669519+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T08:46:19.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.656927+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.657705+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.661571+0000 mon.a (mon.0) 338 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.662507+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.662599+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.663414+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.663699+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.664112+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.664125+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:18.666269+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: audit 2023-10-09T08:46:19.205556+0000 mon.a (mon.0) 339 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: audit 2023-10-09T08:46:19.207145+0000 mon.a (mon.0) 340 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: cluster 2023-10-09T08:46:19.217343+0000 mgr.y (mgr.25421) 239 : cluster 0 pgmap v178: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:19.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:19 smithi094 bash[59677]: audit 2023-10-09T08:46:19.499738+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.16:0/3032182848' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:20.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.656927+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.657705+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.661571+0000 mon.a (mon.0) 338 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.662507+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.662599+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.663414+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.663699+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.664112+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.664125+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:18.666269+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: audit 2023-10-09T08:46:19.205556+0000 mon.a (mon.0) 339 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: audit 2023-10-09T08:46:19.207145+0000 mon.a (mon.0) 340 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: cluster 2023-10-09T08:46:19.217343+0000 mgr.y (mgr.25421) 239 : cluster 0 pgmap v178: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:20.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[75179]: audit 2023-10-09T08:46:19.499738+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.16:0/3032182848' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.656927+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.657705+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.661571+0000 mon.a (mon.0) 338 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.662507+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.662599+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.663414+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.663699+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.664112+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.664125+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:18.666269+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T08:46:20.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: audit 2023-10-09T08:46:19.205556+0000 mon.a (mon.0) 339 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:20.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: audit 2023-10-09T08:46:19.207145+0000 mon.a (mon.0) 340 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:20.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: cluster 2023-10-09T08:46:19.217343+0000 mgr.y (mgr.25421) 239 : cluster 0 pgmap v178: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:46:20.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:19 smithi016 bash[77892]: audit 2023-10-09T08:46:19.499738+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.16:0/3032182848' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:21 smithi094 bash[59677]: cluster 2023-10-09T08:46:21.218437+0000 mgr.y (mgr.25421) 240 : cluster 0 pgmap v179: 73 pgs: 1 active+undersized+degraded, 11 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:21 smithi094 bash[59677]: cluster 2023-10-09T08:46:21.270188+0000 mon.a (mon.0) 341 : cluster 3 Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:21.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:21 smithi016 bash[75179]: cluster 2023-10-09T08:46:21.218437+0000 mgr.y (mgr.25421) 240 : cluster 0 pgmap v179: 73 pgs: 1 active+undersized+degraded, 11 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:21.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:21 smithi016 bash[75179]: cluster 2023-10-09T08:46:21.270188+0000 mon.a (mon.0) 341 : cluster 3 Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:21.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:21 smithi016 bash[77892]: cluster 2023-10-09T08:46:21.218437+0000 mgr.y (mgr.25421) 240 : cluster 0 pgmap v179: 73 pgs: 1 active+undersized+degraded, 11 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T08:46:21.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:21 smithi016 bash[77892]: cluster 2023-10-09T08:46:21.270188+0000 mon.a (mon.0) 341 : cluster 3 Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:22.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:22 smithi094 bash[59677]: audit 2023-10-09T08:46:21.855494+0000 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.16:0/2584821003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:22.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:22 smithi016 bash[75179]: audit 2023-10-09T08:46:21.855494+0000 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.16:0/2584821003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:22.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:22 smithi016 bash[77892]: audit 2023-10-09T08:46:21.855494+0000 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.16:0/2584821003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:23.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:23 smithi016 bash[75179]: cluster 2023-10-09T08:46:23.219074+0000 mgr.y (mgr.25421) 241 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:23.636 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:23 smithi016 bash[77892]: cluster 2023-10-09T08:46:23.219074+0000 mgr.y (mgr.25421) 241 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:23.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:23 smithi094 bash[59677]: cluster 2023-10-09T08:46:23.219074+0000 mgr.y (mgr.25421) 241 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:24.610 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:24 smithi016 bash[75179]: cluster 2023-10-09T08:46:23.324998+0000 mon.a (mon.0) 343 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T08:46:24.610 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:24 smithi016 bash[75179]: cluster 2023-10-09T08:46:23.325052+0000 mon.a (mon.0) 344 : cluster 1 Cluster is now healthy 2023-10-09T08:46:24.610 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:24 smithi016 bash[75179]: audit 2023-10-09T08:46:24.220656+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.16:0/2428564458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:24.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:24 smithi016 bash[77892]: cluster 2023-10-09T08:46:23.324998+0000 mon.a (mon.0) 343 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T08:46:24.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:24 smithi016 bash[77892]: cluster 2023-10-09T08:46:23.325052+0000 mon.a (mon.0) 344 : cluster 1 Cluster is now healthy 2023-10-09T08:46:24.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:24 smithi016 bash[77892]: audit 2023-10-09T08:46:24.220656+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.16:0/2428564458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:24 smithi094 bash[59677]: cluster 2023-10-09T08:46:23.324998+0000 mon.a (mon.0) 343 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T08:46:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:24 smithi094 bash[59677]: cluster 2023-10-09T08:46:23.325052+0000 mon.a (mon.0) 344 : cluster 1 Cluster is now healthy 2023-10-09T08:46:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:24 smithi094 bash[59677]: audit 2023-10-09T08:46:24.220656+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.16:0/2428564458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:25.688 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:25 smithi016 bash[75179]: cluster 2023-10-09T08:46:25.219766+0000 mgr.y (mgr.25421) 242 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:25.688 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:25 smithi016 bash[77892]: cluster 2023-10-09T08:46:25.219766+0000 mgr.y (mgr.25421) 242 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:25.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:25 smithi094 bash[59677]: cluster 2023-10-09T08:46:25.219766+0000 mgr.y (mgr.25421) 242 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:26.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:46:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:46:25] "GET /metrics HTTP/1.1" 200 33673 "" "Prometheus/2.33.4" 2023-10-09T08:46:27.593 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.295498+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.593 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.304538+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.593 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.305805+0000 mon.a (mon.0) 347 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:27.593 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.306934+0000 mon.a (mon.0) 348 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.314117+0000 mon.a (mon.0) 349 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.396809+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.401305+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.403331+0000 mon.a (mon.0) 352 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.404803+0000 mon.a (mon.0) 353 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.406673+0000 mon.a (mon.0) 354 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[75179]: audit 2023-10-09T08:46:26.594106+0000 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.16:0/4283922227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.295498+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.304538+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.305805+0000 mon.a (mon.0) 347 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.306934+0000 mon.a (mon.0) 348 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.314117+0000 mon.a (mon.0) 349 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.396809+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.401305+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.403331+0000 mon.a (mon.0) 352 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.404803+0000 mon.a (mon.0) 353 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.406673+0000 mon.a (mon.0) 354 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T08:46:27.595 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:27 smithi016 bash[77892]: audit 2023-10-09T08:46:26.594106+0000 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.16:0/4283922227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.295498+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.304538+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.305805+0000 mon.a (mon.0) 347 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.306934+0000 mon.a (mon.0) 348 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:46:27.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.314117+0000 mon.a (mon.0) 349 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:27.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.396809+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:27.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.401305+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.403331+0000 mon.a (mon.0) 352 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.404803+0000 mon.a (mon.0) 353 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:27.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.406673+0000 mon.a (mon.0) 354 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T08:46:27.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:27 smithi094 bash[59677]: audit 2023-10-09T08:46:26.594106+0000 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.16:0/4283922227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:28.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:28 smithi016 bash[75179]: audit 2023-10-09T08:46:26.406927+0000 mgr.y (mgr.25421) 243 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T08:46:28.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:28 smithi016 bash[75179]: cephadm 2023-10-09T08:46:26.408263+0000 mgr.y (mgr.25421) 244 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T08:46:28.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:28 smithi016 bash[75179]: cluster 2023-10-09T08:46:27.220653+0000 mgr.y (mgr.25421) 245 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:28.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:28 smithi016 bash[77892]: audit 2023-10-09T08:46:26.406927+0000 mgr.y (mgr.25421) 243 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T08:46:28.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:28 smithi016 bash[77892]: cephadm 2023-10-09T08:46:26.408263+0000 mgr.y (mgr.25421) 244 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T08:46:28.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:28 smithi016 bash[77892]: cluster 2023-10-09T08:46:27.220653+0000 mgr.y (mgr.25421) 245 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:28.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:28 smithi094 bash[59677]: audit 2023-10-09T08:46:26.406927+0000 mgr.y (mgr.25421) 243 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T08:46:28.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:28 smithi094 bash[59677]: cephadm 2023-10-09T08:46:26.408263+0000 mgr.y (mgr.25421) 244 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T08:46:28.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:28 smithi094 bash[59677]: cluster 2023-10-09T08:46:27.220653+0000 mgr.y (mgr.25421) 245 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:29.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: cephadm 2023-10-09T08:46:28.679464+0000 mgr.y (mgr.25421) 246 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T08:46:29.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: audit 2023-10-09T08:46:28.688681+0000 mon.a (mon.0) 356 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:29.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: audit 2023-10-09T08:46:28.690410+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: audit 2023-10-09T08:46:28.691250+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: cephadm 2023-10-09T08:46:28.693452+0000 mgr.y (mgr.25421) 247 : cephadm 1 Deploying daemon osd.2 on smithi016 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: audit 2023-10-09T08:46:28.985340+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.16:0/2181558421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: cluster 2023-10-09T08:46:29.221319+0000 mgr.y (mgr.25421) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[77892]: audit 2023-10-09T08:46:29.263893+0000 mon.a (mon.0) 360 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: cephadm 2023-10-09T08:46:28.679464+0000 mgr.y (mgr.25421) 246 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: audit 2023-10-09T08:46:28.688681+0000 mon.a (mon.0) 356 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:29.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: audit 2023-10-09T08:46:28.690410+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:46:29.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: audit 2023-10-09T08:46:28.691250+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:29.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: cephadm 2023-10-09T08:46:28.693452+0000 mgr.y (mgr.25421) 247 : cephadm 1 Deploying daemon osd.2 on smithi016 2023-10-09T08:46:29.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: audit 2023-10-09T08:46:28.985340+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.16:0/2181558421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:29.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: cluster 2023-10-09T08:46:29.221319+0000 mgr.y (mgr.25421) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:29.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:29 smithi016 bash[75179]: audit 2023-10-09T08:46:29.263893+0000 mon.a (mon.0) 360 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: cephadm 2023-10-09T08:46:28.679464+0000 mgr.y (mgr.25421) 246 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T08:46:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: audit 2023-10-09T08:46:28.688681+0000 mon.a (mon.0) 356 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: audit 2023-10-09T08:46:28.690410+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:46:29.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: audit 2023-10-09T08:46:28.691250+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:29.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: cephadm 2023-10-09T08:46:28.693452+0000 mgr.y (mgr.25421) 247 : cephadm 1 Deploying daemon osd.2 on smithi016 2023-10-09T08:46:29.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: audit 2023-10-09T08:46:28.985340+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.16:0/2181558421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:29.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: cluster 2023-10-09T08:46:29.221319+0000 mgr.y (mgr.25421) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:29.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:29 smithi094 bash[59677]: audit 2023-10-09T08:46:29.263893+0000 mon.a (mon.0) 360 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:32.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:32 smithi016 bash[75179]: cluster 2023-10-09T08:46:31.222325+0000 mgr.y (mgr.25421) 249 : cluster 0 pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:32.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:32 smithi016 bash[75179]: audit 2023-10-09T08:46:31.349632+0000 mon.a (mon.0) 361 : audit 0 from='client.? 172.21.15.16:0/245440000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:32.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:32 smithi016 bash[77892]: cluster 2023-10-09T08:46:31.222325+0000 mgr.y (mgr.25421) 249 : cluster 0 pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:32.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:32 smithi016 bash[77892]: audit 2023-10-09T08:46:31.349632+0000 mon.a (mon.0) 361 : audit 0 from='client.? 172.21.15.16:0/245440000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:32.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:32 smithi094 bash[59677]: cluster 2023-10-09T08:46:31.222325+0000 mgr.y (mgr.25421) 249 : cluster 0 pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:32.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:32 smithi094 bash[59677]: audit 2023-10-09T08:46:31.349632+0000 mon.a (mon.0) 361 : audit 0 from='client.? 172.21.15.16:0/245440000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:33.636 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:33 smithi016 systemd[1]: Stopping Ceph osd.2 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:46:34.003 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:33 smithi016 bash[45289]: debug 2023-10-09T08:46:33.629+0000 7f1d9aa8a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T08:46:34.003 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:33 smithi016 bash[45289]: debug 2023-10-09T08:46:33.629+0000 7f1d9aa8a700 -1 osd.2 84 *** Got signal Terminated *** 2023-10-09T08:46:34.003 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:33 smithi016 bash[45289]: debug 2023-10-09T08:46:33.629+0000 7f1d9aa8a700 -1 osd.2 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T08:46:34.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[75179]: cluster 2023-10-09T08:46:33.223034+0000 mgr.y (mgr.25421) 250 : cluster 0 pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:34.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[75179]: cluster 2023-10-09T08:46:33.632909+0000 mon.a (mon.0) 362 : cluster 1 osd.2 marked itself down and dead 2023-10-09T08:46:34.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[75179]: audit 2023-10-09T08:46:33.719416+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.16:0/4019193503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:34.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[75179]: audit 2023-10-09T08:46:34.205732+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:34.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[75179]: audit 2023-10-09T08:46:34.207238+0000 mon.a (mon.0) 364 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:34.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[77892]: cluster 2023-10-09T08:46:33.223034+0000 mgr.y (mgr.25421) 250 : cluster 0 pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:34.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[77892]: cluster 2023-10-09T08:46:33.632909+0000 mon.a (mon.0) 362 : cluster 1 osd.2 marked itself down and dead 2023-10-09T08:46:34.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[77892]: audit 2023-10-09T08:46:33.719416+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.16:0/4019193503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:34.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[77892]: audit 2023-10-09T08:46:34.205732+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:34.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:34 smithi016 bash[77892]: audit 2023-10-09T08:46:34.207238+0000 mon.a (mon.0) 364 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:34 smithi094 bash[59677]: cluster 2023-10-09T08:46:33.223034+0000 mgr.y (mgr.25421) 250 : cluster 0 pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:34 smithi094 bash[59677]: cluster 2023-10-09T08:46:33.632909+0000 mon.a (mon.0) 362 : cluster 1 osd.2 marked itself down and dead 2023-10-09T08:46:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:34 smithi094 bash[59677]: audit 2023-10-09T08:46:33.719416+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.16:0/4019193503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:34 smithi094 bash[59677]: audit 2023-10-09T08:46:34.205732+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:34 smithi094 bash[59677]: audit 2023-10-09T08:46:34.207238+0000 mon.a (mon.0) 364 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:35.305 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[88952]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T08:46:35.305 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[89019]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T08:46:35.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.280310+0000 mon.a (mon.0) 365 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:46:35.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.283801+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.288903+0000 mon.a (mon.0) 366 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T08:46:35.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.289842+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.289965+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.290989+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.291166+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.292169+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[75179]: cluster 2023-10-09T08:46:34.295168+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.280310+0000 mon.a (mon.0) 365 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.283801+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.288903+0000 mon.a (mon.0) 366 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.289842+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.289965+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.290989+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.291166+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.292169+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[77892]: cluster 2023-10-09T08:46:34.295168+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.280310+0000 mon.a (mon.0) 365 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.283801+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.288903+0000 mon.a (mon.0) 366 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T08:46:35.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.289842+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.289965+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.290989+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.291166+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.292169+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:35.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:35 smithi094 bash[59677]: cluster 2023-10-09T08:46:34.295168+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T08:46:36.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:46:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:46:35] "GET /metrics HTTP/1.1" 200 33819 "" "Prometheus/2.33.4" 2023-10-09T08:46:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.223719+0000 mgr.y (mgr.25421) 251 : cluster 0 pgmap v187: 73 pgs: 13 peering, 1 stale+active+clean, 59 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.300635+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.302585+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.305758+0000 mon.a (mon.0) 367 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T08:46:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.306229+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.307659+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.307700+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.308538+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.308563+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.310185+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: cluster 2023-10-09T08:46:35.311536+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:36 smithi094 bash[59677]: audit 2023-10-09T08:46:36.084258+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.16:0/2386627065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.223719+0000 mgr.y (mgr.25421) 251 : cluster 0 pgmap v187: 73 pgs: 13 peering, 1 stale+active+clean, 59 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.300635+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.302585+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.305758+0000 mon.a (mon.0) 367 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.306229+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.307659+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.307700+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.308538+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.308563+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.310185+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: cluster 2023-10-09T08:46:35.311536+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[77892]: audit 2023-10-09T08:46:36.084258+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.16:0/2386627065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:36.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.223719+0000 mgr.y (mgr.25421) 251 : cluster 0 pgmap v187: 73 pgs: 13 peering, 1 stale+active+clean, 59 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T08:46:36.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.300635+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.302585+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.305758+0000 mon.a (mon.0) 367 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T08:46:36.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.306229+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.307659+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.307700+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.308538+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.308563+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.310185+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: cluster 2023-10-09T08:46:35.311536+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T08:46:36.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:36 smithi016 bash[75179]: audit 2023-10-09T08:46:36.084258+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.16:0/2386627065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:37.643 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:37 smithi016 bash[77892]: cluster 2023-10-09T08:46:37.224718+0000 mgr.y (mgr.25421) 252 : cluster 0 pgmap v189: 73 pgs: 1 active+undersized+degraded, 13 peering, 5 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:37.643 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:37 smithi016 bash[77892]: cluster 2023-10-09T08:46:37.301147+0000 mon.a (mon.0) 368 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:37.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:37 smithi016 bash[75179]: cluster 2023-10-09T08:46:37.224718+0000 mgr.y (mgr.25421) 252 : cluster 0 pgmap v189: 73 pgs: 1 active+undersized+degraded, 13 peering, 5 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:37.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:37 smithi016 bash[75179]: cluster 2023-10-09T08:46:37.301147+0000 mon.a (mon.0) 368 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:37 smithi094 bash[59677]: cluster 2023-10-09T08:46:37.224718+0000 mgr.y (mgr.25421) 252 : cluster 0 pgmap v189: 73 pgs: 1 active+undersized+degraded, 13 peering, 5 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:37 smithi094 bash[59677]: cluster 2023-10-09T08:46:37.301147+0000 mon.a (mon.0) 368 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:37.905 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:37 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T08:46:37.905 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:37 smithi016 systemd[1]: Stopped Ceph osd.2 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:46:38.252 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:37 smithi016 systemd[1]: Started Ceph osd.2 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:46:39.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:38 smithi094 bash[59677]: audit 2023-10-09T08:46:37.945335+0000 mon.a (mon.0) 369 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:39.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:38 smithi094 bash[59677]: audit 2023-10-09T08:46:37.953698+0000 mon.a (mon.0) 370 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:39.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:38 smithi094 bash[59677]: audit 2023-10-09T08:46:37.980746+0000 mon.a (mon.0) 371 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:39.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:38 smithi094 bash[59677]: audit 2023-10-09T08:46:38.440263+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.16:0/836263948' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:39.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[75179]: audit 2023-10-09T08:46:37.945335+0000 mon.a (mon.0) 369 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:39.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[75179]: audit 2023-10-09T08:46:37.953698+0000 mon.a (mon.0) 370 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:39.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[75179]: audit 2023-10-09T08:46:37.980746+0000 mon.a (mon.0) 371 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:39.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[75179]: audit 2023-10-09T08:46:38.440263+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.16:0/836263948' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:39.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[77892]: audit 2023-10-09T08:46:37.945335+0000 mon.a (mon.0) 369 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:39.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[77892]: audit 2023-10-09T08:46:37.953698+0000 mon.a (mon.0) 370 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:39.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[77892]: audit 2023-10-09T08:46:37.980746+0000 mon.a (mon.0) 371 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:39.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:38 smithi016 bash[77892]: audit 2023-10-09T08:46:38.440263+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.16:0/836263948' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:39.911 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T08:46:39.911 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T08:46:39.911 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T08:46:39.912 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T08:46:39.912 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T08:46:39.912 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T08:46:39.912 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T08:46:39.912 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[89198]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T08:46:40.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:39 smithi094 bash[59677]: cluster 2023-10-09T08:46:39.225512+0000 mgr.y (mgr.25421) 253 : cluster 0 pgmap v190: 73 pgs: 1 active+undersized+degraded, 13 peering, 5 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:40.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[75179]: cluster 2023-10-09T08:46:39.225512+0000 mgr.y (mgr.25421) 253 : cluster 0 pgmap v190: 73 pgs: 1 active+undersized+degraded, 13 peering, 5 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:40.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:39 smithi016 bash[77892]: cluster 2023-10-09T08:46:39.225512+0000 mgr.y (mgr.25421) 253 : cluster 0 pgmap v190: 73 pgs: 1 active+undersized+degraded, 13 peering, 5 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:41.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:40 smithi094 bash[59677]: audit 2023-10-09T08:46:40.818838+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.16:0/1264862625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:41.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:40 smithi016 bash[75179]: audit 2023-10-09T08:46:40.818838+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.16:0/1264862625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:41.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:40 smithi016 bash[77892]: audit 2023-10-09T08:46:40.818838+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.16:0/1264862625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:41.752 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:41 smithi016 bash[89522]: debug 2023-10-09T08:46:41.589+0000 7f467b72e700 1 -- 172.21.15.16:0/576595865 <== mon.0 v2:172.21.15.16:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d0572a44e0 con 0x55d0572fe800 2023-10-09T08:46:42.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:41 smithi094 bash[59677]: cluster 2023-10-09T08:46:41.226684+0000 mgr.y (mgr.25421) 254 : cluster 0 pgmap v191: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:42.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:41 smithi016 bash[75179]: cluster 2023-10-09T08:46:41.226684+0000 mgr.y (mgr.25421) 254 : cluster 0 pgmap v191: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:42.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:41 smithi016 bash[77892]: cluster 2023-10-09T08:46:41.226684+0000 mgr.y (mgr.25421) 254 : cluster 0 pgmap v191: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:42.253 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:42 smithi016 bash[89522]: debug 2023-10-09T08:46:42.141+0000 7f46848b4640 -1 Falling back to public interface 2023-10-09T08:46:43.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:43 smithi016 bash[75179]: audit 2023-10-09T08:46:43.187156+0000 mon.a (mon.0) 372 : audit 0 from='client.? 172.21.15.16:0/1048664034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:43.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:43 smithi016 bash[77892]: audit 2023-10-09T08:46:43.187156+0000 mon.a (mon.0) 372 : audit 0 from='client.? 172.21.15.16:0/1048664034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:43.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:43 smithi094 bash[59677]: audit 2023-10-09T08:46:43.187156+0000 mon.a (mon.0) 372 : audit 0 from='client.? 172.21.15.16:0/1048664034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:44.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:44 smithi094 bash[59677]: cluster 2023-10-09T08:46:43.227721+0000 mgr.y (mgr.25421) 255 : cluster 0 pgmap v192: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:44.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:44 smithi094 bash[59677]: cluster 2023-10-09T08:46:43.813581+0000 mon.a (mon.0) 373 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:46:44.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:44 smithi094 bash[59677]: audit 2023-10-09T08:46:44.224557+0000 mon.a (mon.0) 374 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:44.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:44 smithi094 bash[59677]: audit 2023-10-09T08:46:44.231820+0000 mon.a (mon.0) 375 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:44.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[77892]: cluster 2023-10-09T08:46:43.227721+0000 mgr.y (mgr.25421) 255 : cluster 0 pgmap v192: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:44.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[77892]: cluster 2023-10-09T08:46:43.813581+0000 mon.a (mon.0) 373 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:46:44.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[77892]: audit 2023-10-09T08:46:44.224557+0000 mon.a (mon.0) 374 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:44.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[77892]: audit 2023-10-09T08:46:44.231820+0000 mon.a (mon.0) 375 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:44.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[75179]: cluster 2023-10-09T08:46:43.227721+0000 mgr.y (mgr.25421) 255 : cluster 0 pgmap v192: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:44.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[75179]: cluster 2023-10-09T08:46:43.813581+0000 mon.a (mon.0) 373 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:46:44.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[75179]: audit 2023-10-09T08:46:44.224557+0000 mon.a (mon.0) 374 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:44.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:44 smithi016 bash[75179]: audit 2023-10-09T08:46:44.231820+0000 mon.a (mon.0) 375 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:46:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:46:45] "GET /metrics HTTP/1.1" 200 33819 "" "Prometheus/2.33.4" 2023-10-09T08:46:46.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:46 smithi094 bash[59677]: audit 2023-10-09T08:46:45.182874+0000 mon.a (mon.0) 376 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:46 smithi094 bash[59677]: audit 2023-10-09T08:46:45.190539+0000 mon.a (mon.0) 377 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:46 smithi094 bash[59677]: cluster 2023-10-09T08:46:45.228471+0000 mgr.y (mgr.25421) 256 : cluster 0 pgmap v193: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:46.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:46 smithi094 bash[59677]: audit 2023-10-09T08:46:45.552229+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.16:0/1224583192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:46.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:46 smithi094 bash[59677]: audit 2023-10-09T08:46:46.039145+0000 mon.a (mon.0) 378 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:46 smithi094 bash[59677]: audit 2023-10-09T08:46:46.046237+0000 mon.a (mon.0) 379 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[75179]: audit 2023-10-09T08:46:45.182874+0000 mon.a (mon.0) 376 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[75179]: audit 2023-10-09T08:46:45.190539+0000 mon.a (mon.0) 377 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[75179]: cluster 2023-10-09T08:46:45.228471+0000 mgr.y (mgr.25421) 256 : cluster 0 pgmap v193: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[75179]: audit 2023-10-09T08:46:45.552229+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.16:0/1224583192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[75179]: audit 2023-10-09T08:46:46.039145+0000 mon.a (mon.0) 378 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[75179]: audit 2023-10-09T08:46:46.046237+0000 mon.a (mon.0) 379 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[77892]: audit 2023-10-09T08:46:45.182874+0000 mon.a (mon.0) 376 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[77892]: audit 2023-10-09T08:46:45.190539+0000 mon.a (mon.0) 377 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[77892]: cluster 2023-10-09T08:46:45.228471+0000 mgr.y (mgr.25421) 256 : cluster 0 pgmap v193: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[77892]: audit 2023-10-09T08:46:45.552229+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.16:0/1224583192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:46.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[77892]: audit 2023-10-09T08:46:46.039145+0000 mon.a (mon.0) 378 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:46.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[77892]: audit 2023-10-09T08:46:46.046237+0000 mon.a (mon.0) 379 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:47.002 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:46 smithi016 bash[89522]: debug 2023-10-09T08:46:46.681+0000 7f46848b4640 -1 osd.2 84 log_to_monitors true 2023-10-09T08:46:47.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:47 smithi094 bash[59677]: audit 2023-10-09T08:46:46.692344+0000 mon.a (mon.0) 380 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:46:47.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:47 smithi016 bash[75179]: audit 2023-10-09T08:46:46.692344+0000 mon.a (mon.0) 380 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:46:47.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:47 smithi016 bash[77892]: audit 2023-10-09T08:46:46.692344+0000 mon.a (mon.0) 380 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:46:47.503 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 08:46:47 smithi016 bash[89522]: debug 2023-10-09T08:46:47.233+0000 7f4672519700 -1 osd.2 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T08:46:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:48 smithi094 bash[59677]: cluster 2023-10-09T08:46:47.199677+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T08:46:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:48 smithi094 bash[59677]: audit 2023-10-09T08:46:47.201425+0000 mon.a (mon.0) 381 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:46:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:48 smithi094 bash[59677]: cluster 2023-10-09T08:46:47.201528+0000 mon.a (mon.0) 382 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T08:46:48.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:48 smithi094 bash[59677]: audit 2023-10-09T08:46:47.202154+0000 mon.a (mon.0) 383 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:48.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:48 smithi094 bash[59677]: cluster 2023-10-09T08:46:47.229184+0000 mgr.y (mgr.25421) 257 : cluster 0 pgmap v195: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:48.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:48 smithi094 bash[59677]: audit 2023-10-09T08:46:47.913969+0000 mon.a (mon.0) 384 : audit 0 from='client.? 172.21.15.16:0/1973744319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:48.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[75179]: cluster 2023-10-09T08:46:47.199677+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T08:46:48.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[75179]: audit 2023-10-09T08:46:47.201425+0000 mon.a (mon.0) 381 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[75179]: cluster 2023-10-09T08:46:47.201528+0000 mon.a (mon.0) 382 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[75179]: audit 2023-10-09T08:46:47.202154+0000 mon.a (mon.0) 383 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[75179]: cluster 2023-10-09T08:46:47.229184+0000 mgr.y (mgr.25421) 257 : cluster 0 pgmap v195: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[75179]: audit 2023-10-09T08:46:47.913969+0000 mon.a (mon.0) 384 : audit 0 from='client.? 172.21.15.16:0/1973744319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[77892]: cluster 2023-10-09T08:46:47.199677+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[77892]: audit 2023-10-09T08:46:47.201425+0000 mon.a (mon.0) 381 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[77892]: cluster 2023-10-09T08:46:47.201528+0000 mon.a (mon.0) 382 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[77892]: audit 2023-10-09T08:46:47.202154+0000 mon.a (mon.0) 383 : audit 1 from='osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:46:48.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[77892]: cluster 2023-10-09T08:46:47.229184+0000 mgr.y (mgr.25421) 257 : cluster 0 pgmap v195: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:48.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:48 smithi016 bash[77892]: audit 2023-10-09T08:46:47.913969+0000 mon.a (mon.0) 384 : audit 0 from='client.? 172.21.15.16:0/1973744319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:49 smithi094 bash[59677]: cluster 2023-10-09T08:46:48.198563+0000 mon.a (mon.0) 385 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:46:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:49 smithi094 bash[59677]: cluster 2023-10-09T08:46:48.206529+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T08:46:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:49 smithi094 bash[59677]: cluster 2023-10-09T08:46:48.208495+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T08:46:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:49 smithi094 bash[59677]: cluster 2023-10-09T08:46:48.209833+0000 mon.a (mon.0) 386 : cluster 1 osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489] boot 2023-10-09T08:46:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:49 smithi094 bash[59677]: cluster 2023-10-09T08:46:48.209914+0000 mon.a (mon.0) 387 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T08:46:49.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:49 smithi094 bash[59677]: audit 2023-10-09T08:46:48.210724+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[75179]: cluster 2023-10-09T08:46:48.198563+0000 mon.a (mon.0) 385 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[75179]: cluster 2023-10-09T08:46:48.206529+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[75179]: cluster 2023-10-09T08:46:48.208495+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[75179]: cluster 2023-10-09T08:46:48.209833+0000 mon.a (mon.0) 386 : cluster 1 osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489] boot 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[75179]: cluster 2023-10-09T08:46:48.209914+0000 mon.a (mon.0) 387 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[75179]: audit 2023-10-09T08:46:48.210724+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[77892]: cluster 2023-10-09T08:46:48.198563+0000 mon.a (mon.0) 385 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[77892]: cluster 2023-10-09T08:46:48.206529+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T08:46:49.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[77892]: cluster 2023-10-09T08:46:48.208495+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T08:46:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[77892]: cluster 2023-10-09T08:46:48.209833+0000 mon.a (mon.0) 386 : cluster 1 osd.2 [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489] boot 2023-10-09T08:46:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[77892]: cluster 2023-10-09T08:46:48.209914+0000 mon.a (mon.0) 387 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T08:46:49.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:49 smithi016 bash[77892]: audit 2023-10-09T08:46:48.210724+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:46:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: audit 2023-10-09T08:46:49.206240+0000 mon.a (mon.0) 389 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: audit 2023-10-09T08:46:49.213799+0000 mon.a (mon.0) 390 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.215582+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.217234+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.218974+0000 mon.a (mon.0) 391 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T08:46:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.219584+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.220668+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.220915+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.220933+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.220999+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.226731+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:50 smithi094 bash[59677]: cluster 2023-10-09T08:46:49.229936+0000 mgr.y (mgr.25421) 258 : cluster 0 pgmap v198: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:50.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: audit 2023-10-09T08:46:49.206240+0000 mon.a (mon.0) 389 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:50.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: audit 2023-10-09T08:46:49.213799+0000 mon.a (mon.0) 390 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:50.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.215582+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.217234+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.218974+0000 mon.a (mon.0) 391 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T08:46:50.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.219584+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.220668+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.220915+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.220933+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.220999+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.226731+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[75179]: cluster 2023-10-09T08:46:49.229936+0000 mgr.y (mgr.25421) 258 : cluster 0 pgmap v198: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: audit 2023-10-09T08:46:49.206240+0000 mon.a (mon.0) 389 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: audit 2023-10-09T08:46:49.213799+0000 mon.a (mon.0) 390 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.215582+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.217234+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.218974+0000 mon.a (mon.0) 391 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T08:46:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.219584+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.220668+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.220915+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.220933+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.220999+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.226731+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T08:46:50.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:50 smithi016 bash[77892]: cluster 2023-10-09T08:46:49.229936+0000 mgr.y (mgr.25421) 258 : cluster 0 pgmap v198: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:46:51.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:51 smithi094 bash[59677]: audit 2023-10-09T08:46:50.283111+0000 mon.a (mon.0) 392 : audit 0 from='client.? 172.21.15.16:0/842166803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:51 smithi016 bash[75179]: audit 2023-10-09T08:46:50.283111+0000 mon.a (mon.0) 392 : audit 0 from='client.? 172.21.15.16:0/842166803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:51 smithi016 bash[77892]: audit 2023-10-09T08:46:50.283111+0000 mon.a (mon.0) 392 : audit 0 from='client.? 172.21.15.16:0/842166803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:52.425 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:52 smithi016 bash[75179]: cluster 2023-10-09T08:46:51.230519+0000 mgr.y (mgr.25421) 259 : cluster 0 pgmap v199: 73 pgs: 1 active+undersized+degraded, 6 active+undersized, 66 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:52.426 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:52 smithi016 bash[77892]: cluster 2023-10-09T08:46:51.230519+0000 mgr.y (mgr.25421) 259 : cluster 0 pgmap v199: 73 pgs: 1 active+undersized+degraded, 6 active+undersized, 66 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:52.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:52 smithi094 bash[59677]: cluster 2023-10-09T08:46:51.230519+0000 mgr.y (mgr.25421) 259 : cluster 0 pgmap v199: 73 pgs: 1 active+undersized+degraded, 6 active+undersized, 66 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:46:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:53 smithi094 bash[59677]: cluster 2023-10-09T08:46:52.220104+0000 mon.a (mon.0) 393 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:53.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:53 smithi094 bash[59677]: audit 2023-10-09T08:46:52.641889+0000 mon.a (mon.0) 394 : audit 0 from='client.? 172.21.15.16:0/1476155200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:53.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:53 smithi016 bash[75179]: cluster 2023-10-09T08:46:52.220104+0000 mon.a (mon.0) 393 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:53.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:53 smithi016 bash[75179]: audit 2023-10-09T08:46:52.641889+0000 mon.a (mon.0) 394 : audit 0 from='client.? 172.21.15.16:0/1476155200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:53.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:53 smithi016 bash[77892]: cluster 2023-10-09T08:46:52.220104+0000 mon.a (mon.0) 393 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:46:53.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:53 smithi016 bash[77892]: audit 2023-10-09T08:46:52.641889+0000 mon.a (mon.0) 394 : audit 0 from='client.? 172.21.15.16:0/1476155200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:54.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:54 smithi094 bash[59677]: cluster 2023-10-09T08:46:53.231259+0000 mgr.y (mgr.25421) 260 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:54.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:54 smithi016 bash[75179]: cluster 2023-10-09T08:46:53.231259+0000 mgr.y (mgr.25421) 260 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:54.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:54 smithi016 bash[77892]: cluster 2023-10-09T08:46:53.231259+0000 mgr.y (mgr.25421) 260 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:55.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:55 smithi016 bash[75179]: cluster 2023-10-09T08:46:54.229104+0000 mon.a (mon.0) 395 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T08:46:55.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:55 smithi016 bash[75179]: cluster 2023-10-09T08:46:54.229165+0000 mon.a (mon.0) 396 : cluster 1 Cluster is now healthy 2023-10-09T08:46:55.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:55 smithi016 bash[75179]: audit 2023-10-09T08:46:55.006562+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.16:0/321253486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:55.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:55 smithi016 bash[77892]: cluster 2023-10-09T08:46:54.229104+0000 mon.a (mon.0) 395 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T08:46:55.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:55 smithi016 bash[77892]: cluster 2023-10-09T08:46:54.229165+0000 mon.a (mon.0) 396 : cluster 1 Cluster is now healthy 2023-10-09T08:46:55.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:55 smithi016 bash[77892]: audit 2023-10-09T08:46:55.006562+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.16:0/321253486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:55 smithi094 bash[59677]: cluster 2023-10-09T08:46:54.229104+0000 mon.a (mon.0) 395 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T08:46:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:55 smithi094 bash[59677]: cluster 2023-10-09T08:46:54.229165+0000 mon.a (mon.0) 396 : cluster 1 Cluster is now healthy 2023-10-09T08:46:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:55 smithi094 bash[59677]: audit 2023-10-09T08:46:55.006562+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.16:0/321253486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:56.049 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:46:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:46:55] "GET /metrics HTTP/1.1" 200 33672 "" "Prometheus/2.33.4" 2023-10-09T08:46:56.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: cluster 2023-10-09T08:46:55.231920+0000 mgr.y (mgr.25421) 261 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:56.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.550025+0000 mon.a (mon.0) 397 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.557513+0000 mon.a (mon.0) 398 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.559407+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:56.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.560974+0000 mon.a (mon.0) 400 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.568775+0000 mon.a (mon.0) 401 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.652059+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.656742+0000 mon.a (mon.0) 403 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.659512+0000 mon.a (mon.0) 404 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.661439+0000 mon.a (mon.0) 405 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[75179]: audit 2023-10-09T08:46:55.663703+0000 mon.a (mon.0) 406 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: cluster 2023-10-09T08:46:55.231920+0000 mgr.y (mgr.25421) 261 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.550025+0000 mon.a (mon.0) 397 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.557513+0000 mon.a (mon.0) 398 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.559407+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.560974+0000 mon.a (mon.0) 400 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.568775+0000 mon.a (mon.0) 401 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.652059+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.656742+0000 mon.a (mon.0) 403 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.659512+0000 mon.a (mon.0) 404 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.661439+0000 mon.a (mon.0) 405 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:56 smithi016 bash[77892]: audit 2023-10-09T08:46:55.663703+0000 mon.a (mon.0) 406 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T08:46:56.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: cluster 2023-10-09T08:46:55.231920+0000 mgr.y (mgr.25421) 261 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:56.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.550025+0000 mon.a (mon.0) 397 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.557513+0000 mon.a (mon.0) 398 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.559407+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:56.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.560974+0000 mon.a (mon.0) 400 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:46:56.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.568775+0000 mon.a (mon.0) 401 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:56.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.652059+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:46:56.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.656742+0000 mon.a (mon.0) 403 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.659512+0000 mon.a (mon.0) 404 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.661439+0000 mon.a (mon.0) 405 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:46:56.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:56 smithi094 bash[59677]: audit 2023-10-09T08:46:55.663703+0000 mon.a (mon.0) 406 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T08:46:57.402 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:57 smithi016 bash[75179]: audit 2023-10-09T08:46:55.664005+0000 mgr.y (mgr.25421) 262 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T08:46:57.402 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:57 smithi016 bash[75179]: cephadm 2023-10-09T08:46:55.665642+0000 mgr.y (mgr.25421) 263 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T08:46:57.402 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:57 smithi016 bash[77892]: audit 2023-10-09T08:46:55.664005+0000 mgr.y (mgr.25421) 262 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T08:46:57.403 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:57 smithi016 bash[77892]: cephadm 2023-10-09T08:46:55.665642+0000 mgr.y (mgr.25421) 263 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T08:46:57.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:57 smithi094 bash[59677]: audit 2023-10-09T08:46:55.664005+0000 mgr.y (mgr.25421) 262 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T08:46:57.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:57 smithi094 bash[59677]: cephadm 2023-10-09T08:46:55.665642+0000 mgr.y (mgr.25421) 263 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T08:46:58.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:58 smithi094 bash[59677]: cluster 2023-10-09T08:46:57.232783+0000 mgr.y (mgr.25421) 264 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:58.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:58 smithi094 bash[59677]: audit 2023-10-09T08:46:57.380253+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.16:0/3578202434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:58.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:58 smithi094 bash[59677]: audit 2023-10-09T08:46:58.146730+0000 mon.a (mon.0) 408 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:58.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:58 smithi094 bash[59677]: audit 2023-10-09T08:46:58.148586+0000 mon.a (mon.0) 409 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:46:58.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:58 smithi094 bash[59677]: audit 2023-10-09T08:46:58.149975+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[77892]: cluster 2023-10-09T08:46:57.232783+0000 mgr.y (mgr.25421) 264 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[77892]: audit 2023-10-09T08:46:57.380253+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.16:0/3578202434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:58.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[77892]: audit 2023-10-09T08:46:58.146730+0000 mon.a (mon.0) 408 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:58.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[77892]: audit 2023-10-09T08:46:58.148586+0000 mon.a (mon.0) 409 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:46:58.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[77892]: audit 2023-10-09T08:46:58.149975+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:58.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[75179]: cluster 2023-10-09T08:46:57.232783+0000 mgr.y (mgr.25421) 264 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:46:58.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[75179]: audit 2023-10-09T08:46:57.380253+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.16:0/3578202434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:46:58.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[75179]: audit 2023-10-09T08:46:58.146730+0000 mon.a (mon.0) 408 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:46:58.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[75179]: audit 2023-10-09T08:46:58.148586+0000 mon.a (mon.0) 409 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:46:58.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:58 smithi016 bash[75179]: audit 2023-10-09T08:46:58.149975+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:46:59.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:59 smithi016 bash[77892]: cephadm 2023-10-09T08:46:58.135603+0000 mgr.y (mgr.25421) 265 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T08:46:59.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:46:59 smithi016 bash[77892]: cephadm 2023-10-09T08:46:58.153556+0000 mgr.y (mgr.25421) 266 : cephadm 1 Deploying daemon osd.3 on smithi016 2023-10-09T08:46:59.595 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:59 smithi016 bash[75179]: cephadm 2023-10-09T08:46:58.135603+0000 mgr.y (mgr.25421) 265 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T08:46:59.595 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:46:59 smithi016 bash[75179]: cephadm 2023-10-09T08:46:58.153556+0000 mgr.y (mgr.25421) 266 : cephadm 1 Deploying daemon osd.3 on smithi016 2023-10-09T08:46:59.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:59 smithi094 bash[59677]: cephadm 2023-10-09T08:46:58.135603+0000 mgr.y (mgr.25421) 265 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T08:46:59.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:46:59 smithi094 bash[59677]: cephadm 2023-10-09T08:46:58.153556+0000 mgr.y (mgr.25421) 266 : cephadm 1 Deploying daemon osd.3 on smithi016 2023-10-09T08:47:00.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:00 smithi094 bash[59677]: cluster 2023-10-09T08:46:59.233449+0000 mgr.y (mgr.25421) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:00.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:00 smithi094 bash[59677]: audit 2023-10-09T08:46:59.289013+0000 mon.a (mon.0) 411 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:00.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:00 smithi094 bash[59677]: audit 2023-10-09T08:46:59.753697+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.16:0/659299104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:00.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:00 smithi016 bash[75179]: cluster 2023-10-09T08:46:59.233449+0000 mgr.y (mgr.25421) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:00.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:00 smithi016 bash[75179]: audit 2023-10-09T08:46:59.289013+0000 mon.a (mon.0) 411 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:00.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:00 smithi016 bash[75179]: audit 2023-10-09T08:46:59.753697+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.16:0/659299104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:00.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:00 smithi016 bash[77892]: cluster 2023-10-09T08:46:59.233449+0000 mgr.y (mgr.25421) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:00.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:00 smithi016 bash[77892]: audit 2023-10-09T08:46:59.289013+0000 mon.a (mon.0) 411 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:00.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:00 smithi016 bash[77892]: audit 2023-10-09T08:46:59.753697+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.16:0/659299104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:02.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:02 smithi016 bash[75179]: cluster 2023-10-09T08:47:01.234524+0000 mgr.y (mgr.25421) 268 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:02.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:02 smithi016 bash[75179]: audit 2023-10-09T08:47:02.108052+0000 mon.a (mon.0) 412 : audit 0 from='client.? 172.21.15.16:0/1767782859' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:02.616 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:02 smithi016 bash[77892]: cluster 2023-10-09T08:47:01.234524+0000 mgr.y (mgr.25421) 268 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:02.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:02 smithi016 bash[77892]: audit 2023-10-09T08:47:02.108052+0000 mon.a (mon.0) 412 : audit 0 from='client.? 172.21.15.16:0/1767782859' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:02 smithi094 bash[59677]: cluster 2023-10-09T08:47:01.234524+0000 mgr.y (mgr.25421) 268 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:02 smithi094 bash[59677]: audit 2023-10-09T08:47:02.108052+0000 mon.a (mon.0) 412 : audit 0 from='client.? 172.21.15.16:0/1767782859' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:03.002 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:02 smithi016 systemd[1]: Stopping Ceph osd.3 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T08:47:03.304 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:03 smithi016 bash[49535]: debug 2023-10-09T08:47:03.009+0000 7ff687f58700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T08:47:03.304 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:03 smithi016 bash[49535]: debug 2023-10-09T08:47:03.009+0000 7ff687f58700 -1 osd.3 89 *** Got signal Terminated *** 2023-10-09T08:47:03.304 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:03 smithi016 bash[49535]: debug 2023-10-09T08:47:03.009+0000 7ff687f58700 -1 osd.3 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T08:47:03.592 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:03 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.013796+0000 mon.a (mon.0) 413 : cluster 1 osd.3 marked itself down and dead 2023-10-09T08:47:03.592 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:03 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.013796+0000 mon.a (mon.0) 413 : cluster 1 osd.3 marked itself down and dead 2023-10-09T08:47:03.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:03 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.013796+0000 mon.a (mon.0) 413 : cluster 1 osd.3 marked itself down and dead 2023-10-09T08:47:04.313 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[91713]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T08:47:04.314 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[91781]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T08:47:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.235387+0000 mgr.y (mgr.25421) 269 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.292102+0000 mon.a (mon.0) 414 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.301325+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.304498+0000 mon.a (mon.0) 415 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.305836+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.306057+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.306283+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.309005+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.312109+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: cluster 2023-10-09T08:47:03.316876+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: audit 2023-10-09T08:47:04.204545+0000 mon.a (mon.0) 416 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[77892]: audit 2023-10-09T08:47:04.205986+0000 mon.a (mon.0) 417 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.235387+0000 mgr.y (mgr.25421) 269 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.292102+0000 mon.a (mon.0) 414 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.301325+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.304498+0000 mon.a (mon.0) 415 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.305836+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.306057+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.306283+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.309005+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.312109+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: cluster 2023-10-09T08:47:03.316876+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: audit 2023-10-09T08:47:04.204545+0000 mon.a (mon.0) 416 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:04.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:04 smithi016 bash[75179]: audit 2023-10-09T08:47:04.205986+0000 mon.a (mon.0) 417 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.235387+0000 mgr.y (mgr.25421) 269 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.292102+0000 mon.a (mon.0) 414 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T08:47:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.301325+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.304498+0000 mon.a (mon.0) 415 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.305836+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.306057+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.306283+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.309005+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.312109+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: cluster 2023-10-09T08:47:03.316876+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: audit 2023-10-09T08:47:04.204545+0000 mon.a (mon.0) 416 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:04.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:04 smithi094 bash[59677]: audit 2023-10-09T08:47:04.205986+0000 mon.a (mon.0) 417 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:05.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.302830+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.303784+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.304452+0000 mon.a (mon.0) 418 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T08:47:05.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.306279+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.308139+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.308396+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.309119+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.310304+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.310315+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.310841+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:04.311362+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: audit 2023-10-09T08:47:04.480068+0000 mon.a (mon.0) 419 : audit 0 from='client.? 172.21.15.16:0/262968494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[75179]: cluster 2023-10-09T08:47:05.236220+0000 mgr.y (mgr.25421) 270 : cluster 0 pgmap v208: 73 pgs: 2 active+undersized+degraded, 7 stale+active+clean, 13 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:47:05.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.302830+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.303784+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.304452+0000 mon.a (mon.0) 418 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.306279+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.308139+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.308396+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.309119+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.310304+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.310315+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.310841+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:04.311362+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: audit 2023-10-09T08:47:04.480068+0000 mon.a (mon.0) 419 : audit 0 from='client.? 172.21.15.16:0/262968494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:05.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[77892]: cluster 2023-10-09T08:47:05.236220+0000 mgr.y (mgr.25421) 270 : cluster 0 pgmap v208: 73 pgs: 2 active+undersized+degraded, 7 stale+active+clean, 13 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:47:05.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.302830+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.303784+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.304452+0000 mon.a (mon.0) 418 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T08:47:05.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.306279+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.308139+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.308396+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.309119+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.310304+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.310315+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.310841+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:04.311362+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T08:47:05.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: audit 2023-10-09T08:47:04.480068+0000 mon.a (mon.0) 419 : audit 0 from='client.? 172.21.15.16:0/262968494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:05.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:05 smithi094 bash[59677]: cluster 2023-10-09T08:47:05.236220+0000 mgr.y (mgr.25421) 270 : cluster 0 pgmap v208: 73 pgs: 2 active+undersized+degraded, 7 stale+active+clean, 13 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T08:47:06.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:47:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:47:05] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T08:47:06.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:06 smithi094 bash[59677]: cluster 2023-10-09T08:47:05.310661+0000 mon.a (mon.0) 420 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:47:06.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:06 smithi016 bash[77892]: cluster 2023-10-09T08:47:05.310661+0000 mon.a (mon.0) 420 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:47:06.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:06 smithi016 bash[75179]: cluster 2023-10-09T08:47:05.310661+0000 mon.a (mon.0) 420 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:47:06.752 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:06 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T08:47:06.752 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:06 smithi016 systemd[1]: Stopped Ceph osd.3 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:47:07.252 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:06 smithi016 systemd[1]: Started Ceph osd.3 for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T08:47:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:07 smithi094 bash[59677]: audit 2023-10-09T08:47:06.818393+0000 mon.a (mon.0) 421 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:07 smithi094 bash[59677]: audit 2023-10-09T08:47:06.825951+0000 mon.a (mon.0) 422 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:07 smithi094 bash[59677]: audit 2023-10-09T08:47:06.837848+0000 mon.a (mon.0) 423 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:47:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:07 smithi094 bash[59677]: audit 2023-10-09T08:47:06.872516+0000 mon.a (mon.0) 424 : audit 0 from='client.? 172.21.15.16:0/457178138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:08.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:07 smithi094 bash[59677]: cluster 2023-10-09T08:47:07.237105+0000 mgr.y (mgr.25421) 271 : cluster 0 pgmap v209: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:08.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[77892]: audit 2023-10-09T08:47:06.818393+0000 mon.a (mon.0) 421 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:08.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[77892]: audit 2023-10-09T08:47:06.825951+0000 mon.a (mon.0) 422 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:08.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[77892]: audit 2023-10-09T08:47:06.837848+0000 mon.a (mon.0) 423 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:47:08.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[77892]: audit 2023-10-09T08:47:06.872516+0000 mon.a (mon.0) 424 : audit 0 from='client.? 172.21.15.16:0/457178138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:08.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[77892]: cluster 2023-10-09T08:47:07.237105+0000 mgr.y (mgr.25421) 271 : cluster 0 pgmap v209: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:08.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[75179]: audit 2023-10-09T08:47:06.818393+0000 mon.a (mon.0) 421 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:08.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[75179]: audit 2023-10-09T08:47:06.825951+0000 mon.a (mon.0) 422 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:08.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[75179]: audit 2023-10-09T08:47:06.837848+0000 mon.a (mon.0) 423 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:47:08.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[75179]: audit 2023-10-09T08:47:06.872516+0000 mon.a (mon.0) 424 : audit 0 from='client.? 172.21.15.16:0/457178138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:08.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:07 smithi016 bash[75179]: cluster 2023-10-09T08:47:07.237105+0000 mgr.y (mgr.25421) 271 : cluster 0 pgmap v209: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T08:47:08.658 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:08 smithi016 bash[91984]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T08:47:10.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:09 smithi016 bash[77892]: audit 2023-10-09T08:47:09.235151+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.16:0/1574773279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:10.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:09 smithi016 bash[77892]: cluster 2023-10-09T08:47:09.237651+0000 mgr.y (mgr.25421) 272 : cluster 0 pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:10.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:09 smithi016 bash[75179]: audit 2023-10-09T08:47:09.235151+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.16:0/1574773279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:10.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:09 smithi016 bash[75179]: cluster 2023-10-09T08:47:09.237651+0000 mgr.y (mgr.25421) 272 : cluster 0 pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:09 smithi094 bash[59677]: audit 2023-10-09T08:47:09.235151+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.16:0/1574773279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:09 smithi094 bash[59677]: cluster 2023-10-09T08:47:09.237651+0000 mgr.y (mgr.25421) 272 : cluster 0 pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:10.502 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:10 smithi016 bash[92306]: debug 2023-10-09T08:47:10.161+0000 7fe38dd79700 1 -- 172.21.15.16:0/3646025163 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f1808384e0 con 0x55f180892c00 2023-10-09T08:47:11.002 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:10 smithi016 bash[92306]: debug 2023-10-09T08:47:10.701+0000 7fe396eff640 -1 Falling back to public interface 2023-10-09T08:47:12.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:12 smithi094 bash[59677]: cluster 2023-10-09T08:47:11.238666+0000 mgr.y (mgr.25421) 273 : cluster 0 pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:12.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:12 smithi094 bash[59677]: audit 2023-10-09T08:47:11.608045+0000 mon.a (mon.0) 426 : audit 0 from='client.? 172.21.15.16:0/3937831737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:12.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:12 smithi016 bash[75179]: cluster 2023-10-09T08:47:11.238666+0000 mgr.y (mgr.25421) 273 : cluster 0 pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:12.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:12 smithi016 bash[75179]: audit 2023-10-09T08:47:11.608045+0000 mon.a (mon.0) 426 : audit 0 from='client.? 172.21.15.16:0/3937831737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:12.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:12 smithi016 bash[77892]: cluster 2023-10-09T08:47:11.238666+0000 mgr.y (mgr.25421) 273 : cluster 0 pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:12.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:12 smithi016 bash[77892]: audit 2023-10-09T08:47:11.608045+0000 mon.a (mon.0) 426 : audit 0 from='client.? 172.21.15.16:0/3937831737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:14.252 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:13 smithi016 bash[92306]: debug 2023-10-09T08:47:13.965+0000 7fe396eff640 -1 osd.3 89 log_to_monitors true 2023-10-09T08:47:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:14 smithi094 bash[59677]: cluster 2023-10-09T08:47:13.239468+0000 mgr.y (mgr.25421) 274 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:14 smithi094 bash[59677]: cluster 2023-10-09T08:47:13.822303+0000 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T08:47:14.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:14 smithi094 bash[59677]: audit 2023-10-09T08:47:13.930828+0000 mon.a (mon.0) 428 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:14.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:14 smithi094 bash[59677]: audit 2023-10-09T08:47:13.938313+0000 mon.a (mon.0) 429 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:14.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:14 smithi094 bash[59677]: audit 2023-10-09T08:47:13.972388+0000 mon.c (mon.2) 29 : audit 1 from='osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:47:14.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:14 smithi094 bash[59677]: audit 2023-10-09T08:47:13.972827+0000 mon.a (mon.0) 430 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:47:14.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:14 smithi094 bash[59677]: audit 2023-10-09T08:47:14.016242+0000 mon.a (mon.0) 431 : audit 0 from='client.? 172.21.15.16:0/1396897208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:14.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[75179]: cluster 2023-10-09T08:47:13.239468+0000 mgr.y (mgr.25421) 274 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:14.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[75179]: cluster 2023-10-09T08:47:13.822303+0000 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T08:47:14.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[75179]: audit 2023-10-09T08:47:13.930828+0000 mon.a (mon.0) 428 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:14.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[75179]: audit 2023-10-09T08:47:13.938313+0000 mon.a (mon.0) 429 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[75179]: audit 2023-10-09T08:47:13.972388+0000 mon.c (mon.2) 29 : audit 1 from='osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[75179]: audit 2023-10-09T08:47:13.972827+0000 mon.a (mon.0) 430 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[75179]: audit 2023-10-09T08:47:14.016242+0000 mon.a (mon.0) 431 : audit 0 from='client.? 172.21.15.16:0/1396897208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[77892]: cluster 2023-10-09T08:47:13.239468+0000 mgr.y (mgr.25421) 274 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[77892]: cluster 2023-10-09T08:47:13.822303+0000 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[77892]: audit 2023-10-09T08:47:13.930828+0000 mon.a (mon.0) 428 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[77892]: audit 2023-10-09T08:47:13.938313+0000 mon.a (mon.0) 429 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[77892]: audit 2023-10-09T08:47:13.972388+0000 mon.c (mon.2) 29 : audit 1 from='osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[77892]: audit 2023-10-09T08:47:13.972827+0000 mon.a (mon.0) 430 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:47:14.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[77892]: audit 2023-10-09T08:47:14.016242+0000 mon.a (mon.0) 431 : audit 0 from='client.? 172.21.15.16:0/1396897208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:15.252 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 08:47:14 smithi016 bash[92306]: debug 2023-10-09T08:47:14.981+0000 7fe38836b700 -1 osd.3 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T08:47:15.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:15 smithi094 bash[59677]: audit 2023-10-09T08:47:14.732592+0000 mon.a (mon.0) 432 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:15.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:15 smithi094 bash[59677]: audit 2023-10-09T08:47:14.740404+0000 mon.a (mon.0) 433 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:15.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:15 smithi094 bash[59677]: audit 2023-10-09T08:47:14.944461+0000 mon.a (mon.0) 434 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:47:15.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:15 smithi094 bash[59677]: cluster 2023-10-09T08:47:14.944578+0000 mon.a (mon.0) 435 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T08:47:15.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:15 smithi094 bash[59677]: audit 2023-10-09T08:47:14.945341+0000 mon.c (mon.2) 30 : audit 1 from='osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:47:15.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:15 smithi094 bash[59677]: audit 2023-10-09T08:47:14.945801+0000 mon.a (mon.0) 436 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:47:15.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:15 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.240467+0000 mgr.y (mgr.25421) 275 : cluster 0 pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:16.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:47:15] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T08:47:16.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[77892]: audit 2023-10-09T08:47:14.732592+0000 mon.a (mon.0) 432 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:16.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[77892]: audit 2023-10-09T08:47:14.740404+0000 mon.a (mon.0) 433 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:16.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[77892]: audit 2023-10-09T08:47:14.944461+0000 mon.a (mon.0) 434 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:47:16.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[77892]: cluster 2023-10-09T08:47:14.944578+0000 mon.a (mon.0) 435 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T08:47:16.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[77892]: audit 2023-10-09T08:47:14.945341+0000 mon.c (mon.2) 30 : audit 1 from='osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:47:16.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[77892]: audit 2023-10-09T08:47:14.945801+0000 mon.a (mon.0) 436 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.240467+0000 mgr.y (mgr.25421) 275 : cluster 0 pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[75179]: audit 2023-10-09T08:47:14.732592+0000 mon.a (mon.0) 432 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[75179]: audit 2023-10-09T08:47:14.740404+0000 mon.a (mon.0) 433 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[75179]: audit 2023-10-09T08:47:14.944461+0000 mon.a (mon.0) 434 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[75179]: cluster 2023-10-09T08:47:14.944578+0000 mon.a (mon.0) 435 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[75179]: audit 2023-10-09T08:47:14.945341+0000 mon.c (mon.2) 30 : audit 1 from='osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[75179]: audit 2023-10-09T08:47:14.945801+0000 mon.a (mon.0) 436 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-10-09T08:47:16.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:15 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.240467+0000 mgr.y (mgr.25421) 275 : cluster 0 pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:16.408 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T08:47:16.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.798619+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:16.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.798691+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:16.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.798963+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:16.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.798980+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:16.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.938763+0000 mon.a (mon.0) 437 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:47:16.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.943299+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T08:47:16.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.946631+0000 mon.a (mon.0) 438 : cluster 1 osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985] boot 2023-10-09T08:47:16.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: cluster 2023-10-09T08:47:15.946706+0000 mon.a (mon.0) 439 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T08:47:16.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: audit 2023-10-09T08:47:15.947299+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:47:16.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:16 smithi094 bash[59677]: audit 2023-10-09T08:47:16.387994+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.16:0/2299563149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:17.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.798619+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.798691+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.798963+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.798980+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.938763+0000 mon.a (mon.0) 437 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:47:17.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.943299+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T08:47:17.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.946631+0000 mon.a (mon.0) 438 : cluster 1 osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985] boot 2023-10-09T08:47:17.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: cluster 2023-10-09T08:47:15.946706+0000 mon.a (mon.0) 439 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T08:47:17.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: audit 2023-10-09T08:47:15.947299+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:47:17.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[77892]: audit 2023-10-09T08:47:16.387994+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.16:0/2299563149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:17.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.798619+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.798691+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.798963+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.798980+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T08:47:17.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.938763+0000 mon.a (mon.0) 437 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T08:47:17.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.943299+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T08:47:17.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.946631+0000 mon.a (mon.0) 438 : cluster 1 osd.3 [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985] boot 2023-10-09T08:47:17.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: cluster 2023-10-09T08:47:15.946706+0000 mon.a (mon.0) 439 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T08:47:17.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: audit 2023-10-09T08:47:15.947299+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:47:17.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:16 smithi016 bash[75179]: audit 2023-10-09T08:47:16.387994+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.16:0/2299563149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T08:47:17.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T08:47:18.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.955874+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.956994+0000 mon.a (mon.0) 442 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T08:47:18.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.958672+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.958708+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.959413+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.961768+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.963766+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:16.966017+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:17 smithi094 bash[59677]: cluster 2023-10-09T08:47:17.241366+0000 mgr.y (mgr.25421) 276 : cluster 0 pgmap v217: 73 pgs: 2 peering, 3 active+undersized+degraded, 19 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:18.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.955874+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.956994+0000 mon.a (mon.0) 442 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T08:47:18.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.958672+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.958708+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.959413+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.961768+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.963766+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:16.966017+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[77892]: cluster 2023-10-09T08:47:17.241366+0000 mgr.y (mgr.25421) 276 : cluster 0 pgmap v217: 73 pgs: 2 peering, 3 active+undersized+degraded, 19 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.955874+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.956994+0000 mon.a (mon.0) 442 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T08:47:18.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.958672+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.958708+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.959413+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.961768+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.963766+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:16.966017+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T08:47:18.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:17 smithi016 bash[75179]: cluster 2023-10-09T08:47:17.241366+0000 mgr.y (mgr.25421) 276 : cluster 0 pgmap v217: 73 pgs: 2 peering, 3 active+undersized+degraded, 19 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:20 smithi094 bash[59677]: audit 2023-10-09T08:47:19.205531+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:20 smithi094 bash[59677]: audit 2023-10-09T08:47:19.207072+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:20.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:20 smithi094 bash[59677]: cluster 2023-10-09T08:47:19.241931+0000 mgr.y (mgr.25421) 277 : cluster 0 pgmap v218: 73 pgs: 2 peering, 3 active+undersized+degraded, 19 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:20.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:20 smithi016 bash[75179]: audit 2023-10-09T08:47:19.205531+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:20.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:20 smithi016 bash[75179]: audit 2023-10-09T08:47:19.207072+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:20.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:20 smithi016 bash[75179]: cluster 2023-10-09T08:47:19.241931+0000 mgr.y (mgr.25421) 277 : cluster 0 pgmap v218: 73 pgs: 2 peering, 3 active+undersized+degraded, 19 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:20.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:20 smithi016 bash[77892]: audit 2023-10-09T08:47:19.205531+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:20.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:20 smithi016 bash[77892]: audit 2023-10-09T08:47:19.207072+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:20.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:20 smithi016 bash[77892]: cluster 2023-10-09T08:47:19.241931+0000 mgr.y (mgr.25421) 277 : cluster 0 pgmap v218: 73 pgs: 2 peering, 3 active+undersized+degraded, 19 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T08:47:21.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:21 smithi016 bash[75179]: cluster 2023-10-09T08:47:21.242725+0000 mgr.y (mgr.25421) 278 : cluster 0 pgmap v219: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:21.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:21 smithi016 bash[75179]: cluster 2023-10-09T08:47:21.294039+0000 mon.a (mon.0) 445 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T08:47:21.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:21 smithi016 bash[75179]: cluster 2023-10-09T08:47:21.294070+0000 mon.a (mon.0) 446 : cluster 1 Cluster is now healthy 2023-10-09T08:47:21.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:21 smithi016 bash[77892]: cluster 2023-10-09T08:47:21.242725+0000 mgr.y (mgr.25421) 278 : cluster 0 pgmap v219: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:21.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:21 smithi016 bash[77892]: cluster 2023-10-09T08:47:21.294039+0000 mon.a (mon.0) 445 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T08:47:21.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:21 smithi016 bash[77892]: cluster 2023-10-09T08:47:21.294070+0000 mon.a (mon.0) 446 : cluster 1 Cluster is now healthy 2023-10-09T08:47:21.644 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:47:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:21 smithi094 bash[59677]: cluster 2023-10-09T08:47:21.242725+0000 mgr.y (mgr.25421) 278 : cluster 0 pgmap v219: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:21 smithi094 bash[59677]: cluster 2023-10-09T08:47:21.294039+0000 mon.a (mon.0) 445 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T08:47:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:21 smithi094 bash[59677]: cluster 2023-10-09T08:47:21.294070+0000 mon.a (mon.0) 446 : cluster 1 Cluster is now healthy 2023-10-09T08:47:22.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:22 smithi016 bash[75179]: audit 2023-10-09T08:47:21.632680+0000 mgr.y (mgr.25421) 279 : audit 0 from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:22.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:22 smithi016 bash[75179]: cephadm 2023-10-09T08:47:21.633359+0000 mgr.y (mgr.25421) 280 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:47:22.826 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:22 smithi016 bash[75179]: audit 2023-10-09T08:47:21.640052+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:22.826 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:22 smithi016 bash[77892]: audit 2023-10-09T08:47:21.632680+0000 mgr.y (mgr.25421) 279 : audit 0 from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:22.826 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:22 smithi016 bash[77892]: cephadm 2023-10-09T08:47:21.633359+0000 mgr.y (mgr.25421) 280 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:47:22.826 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:22 smithi016 bash[77892]: audit 2023-10-09T08:47:21.640052+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:22.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T08:47:22.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:22 smithi094 bash[59677]: audit 2023-10-09T08:47:21.632680+0000 mgr.y (mgr.25421) 279 : audit 0 from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:22.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:22 smithi094 bash[59677]: cephadm 2023-10-09T08:47:21.633359+0000 mgr.y (mgr.25421) 280 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T08:47:22.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:22 smithi094 bash[59677]: audit 2023-10-09T08:47:21.640052+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:23.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:23 smithi094 bash[59677]: cluster 2023-10-09T08:47:23.243580+0000 mgr.y (mgr.25421) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:24.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:23 smithi016 bash[75179]: cluster 2023-10-09T08:47:23.243580+0000 mgr.y (mgr.25421) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:24.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:23 smithi016 bash[77892]: cluster 2023-10-09T08:47:23.243580+0000 mgr.y (mgr.25421) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:25.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:25 smithi016 bash[75179]: audit 2023-10-09T08:47:24.309654+0000 mon.a (mon.0) 448 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:25.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:25 smithi016 bash[77892]: audit 2023-10-09T08:47:24.309654+0000 mon.a (mon.0) 448 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:25.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:25 smithi094 bash[59677]: audit 2023-10-09T08:47:24.309654+0000 mon.a (mon.0) 448 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:26.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:47:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:47:25] "GET /metrics HTTP/1.1" 200 33733 "" "Prometheus/2.33.4" 2023-10-09T08:47:26.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:26 smithi016 bash[75179]: cluster 2023-10-09T08:47:25.244602+0000 mgr.y (mgr.25421) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:26.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:26 smithi016 bash[77892]: cluster 2023-10-09T08:47:25.244602+0000 mgr.y (mgr.25421) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:26.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:26 smithi094 bash[59677]: cluster 2023-10-09T08:47:25.244602+0000 mgr.y (mgr.25421) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:27.829 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T08:47:27.829 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (8m) - 14m 13.6M - 0.23.0 ba2b418f427c b9989d9bd43e 2023-10-09T08:47:27.829 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi094 *:3000 running (8m) - 14m 40.3M - 8.3.5 dad864ee21e9 f1a12efb3a56 2023-10-09T08:47:27.829 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.ccdktz smithi016 running (12m) - 12m 17.3M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 17a3e2c7a98b 2023-10-09T08:47:27.829 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi094.zfssci smithi094 running (12m) - 12m 14.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a917ff5f77b 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi094 *:8443,9283,8765 running (7m) - 21m 428M - 18.2.0-598-gb4fa663f 3060897fdf97 f0fcc951b93f 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (8m) - 24m 483M - 18.2.0-598-gb4fa663f 3060897fdf97 cdf79c37f30d 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (6m) - 24m 74.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 a8c0ba734014 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi094 running (3m) - 22m 41.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7b9786ef21a9 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (5m) - 22m 47.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 320971806e89 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (8m) - 15m 9488k - 1.3.1 1dbe0e931976 6c4a45c1d04a 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi094 *:9100 running (8m) - 15m 9451k - 1.3.1 1dbe0e931976 59bcf67d7eb5 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (106s) - 21m 42.0M 2525M 18.2.0-598-gb4fa663f 3060897fdf97 be477be4c121 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (76s) - 20m 40.6M 2525M 18.2.0-598-gb4fa663f 3060897fdf97 167f69e5a075 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (47s) - 19m 41.8M 2525M 18.2.0-598-gb4fa663f 3060897fdf97 5f2845aee315 2023-10-09T08:47:27.830 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (18s) - 19m - 2525M 18.2.0-598-gb4fa663f 3060897fdf97 bc12eb2c50ee 2023-10-09T08:47:27.831 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi094 running (18m) - 18m 48.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1a20cacc6df3 2023-10-09T08:47:27.831 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi094 running (17m) - 17m 51.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 381d98a4283c 2023-10-09T08:47:27.831 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi094 running (16m) - 16m 45.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d8307333e14e 2023-10-09T08:47:27.831 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi094 running (16m) - 16m 55.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 04152ef43de4 2023-10-09T08:47:27.831 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi094 *:9095 running (7m) - 15m 58.2M - 2.33.4 514e6a882f6e 4aee8e4f263b 2023-10-09T08:47:28.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.050482+0000 mon.a (mon.0) 449 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.057314+0000 mon.a (mon.0) 450 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.058905+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.060026+0000 mon.a (mon.0) 452 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.067142+0000 mon.a (mon.0) 453 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.151270+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.154148+0000 mon.a (mon.0) 455 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.155771+0000 mon.a (mon.0) 456 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: audit 2023-10-09T08:47:27.163271+0000 mon.a (mon.0) 457 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[75179]: cluster 2023-10-09T08:47:27.245126+0000 mgr.y (mgr.25421) 283 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.050482+0000 mon.a (mon.0) 449 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.057314+0000 mon.a (mon.0) 450 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.058905+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.060026+0000 mon.a (mon.0) 452 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.067142+0000 mon.a (mon.0) 453 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.151270+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.154148+0000 mon.a (mon.0) 455 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.155771+0000 mon.a (mon.0) 456 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: audit 2023-10-09T08:47:27.163271+0000 mon.a (mon.0) 457 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:28 smithi016 bash[77892]: cluster 2023-10-09T08:47:27.245126+0000 mgr.y (mgr.25421) 283 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:28.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.050482+0000 mon.a (mon.0) 449 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.057314+0000 mon.a (mon.0) 450 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.058905+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.060026+0000 mon.a (mon.0) 452 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.067142+0000 mon.a (mon.0) 453 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.151270+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.154148+0000 mon.a (mon.0) 455 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.155771+0000 mon.a (mon.0) 456 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: audit 2023-10-09T08:47:27.163271+0000 mon.a (mon.0) 457 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:28.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:28 smithi094 bash[59677]: cluster 2023-10-09T08:47:27.245126+0000 mgr.y (mgr.25421) 283 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:28.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T08:47:29.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:29 smithi094 bash[59677]: audit 2023-10-09T08:47:27.806155+0000 mgr.y (mgr.25421) 284 : audit 0 from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:29.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:29 smithi016 bash[75179]: audit 2023-10-09T08:47:27.806155+0000 mgr.y (mgr.25421) 284 : audit 0 from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:29.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:29 smithi016 bash[77892]: audit 2023-10-09T08:47:27.806155+0000 mgr.y (mgr.25421) 284 : audit 0 from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:30.429 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:30 smithi016 bash[75179]: cluster 2023-10-09T08:47:29.245648+0000 mgr.y (mgr.25421) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:30.429 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:30 smithi016 bash[77892]: cluster 2023-10-09T08:47:29.245648+0000 mgr.y (mgr.25421) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:30.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:30 smithi094 bash[59677]: cluster 2023-10-09T08:47:29.245648+0000 mgr.y (mgr.25421) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:32.465 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T08:47:32.465 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (8m) - 14m 13.6M - 0.23.0 ba2b418f427c b9989d9bd43e 2023-10-09T08:47:32.465 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi094 *:3000 running (8m) - 14m 40.3M - 8.3.5 dad864ee21e9 f1a12efb3a56 2023-10-09T08:47:32.465 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.ccdktz smithi016 running (13m) - 13m 17.3M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 17a3e2c7a98b 2023-10-09T08:47:32.465 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi094.zfssci smithi094 running (12m) - 12m 14.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a917ff5f77b 2023-10-09T08:47:32.465 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi094 *:8443,9283,8765 running (7m) - 22m 428M - 18.2.0-598-gb4fa663f 3060897fdf97 f0fcc951b93f 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (8m) - 24m 483M - 18.2.0-598-gb4fa663f 3060897fdf97 cdf79c37f30d 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (6m) - 24m 74.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 a8c0ba734014 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi094 running (3m) - 22m 41.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7b9786ef21a9 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (5m) - 22m 47.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 320971806e89 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (9m) - 15m 9488k - 1.3.1 1dbe0e931976 6c4a45c1d04a 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi094 *:9100 running (8m) - 15m 9451k - 1.3.1 1dbe0e931976 59bcf67d7eb5 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (111s) - 21m 42.0M 2525M 18.2.0-598-gb4fa663f 3060897fdf97 be477be4c121 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (81s) - 20m 40.6M 2525M 18.2.0-598-gb4fa663f 3060897fdf97 167f69e5a075 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (51s) - 20m 41.8M 2525M 18.2.0-598-gb4fa663f 3060897fdf97 5f2845aee315 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (23s) - 19m - 2525M 18.2.0-598-gb4fa663f 3060897fdf97 bc12eb2c50ee 2023-10-09T08:47:32.466 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi094 running (18m) - 18m 48.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1a20cacc6df3 2023-10-09T08:47:32.467 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi094 running (17m) - 17m 51.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 381d98a4283c 2023-10-09T08:47:32.467 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi094 running (16m) - 17m 45.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d8307333e14e 2023-10-09T08:47:32.467 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi094 running (16m) - 16m 55.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 04152ef43de4 2023-10-09T08:47:32.467 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi094 *:9095 running (7m) - 15m 58.2M - 2.33.4 514e6a882f6e 4aee8e4f263b 2023-10-09T08:47:32.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:32 smithi094 bash[59677]: cluster 2023-10-09T08:47:31.246702+0000 mgr.y (mgr.25421) 286 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:32.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:32 smithi016 bash[75179]: cluster 2023-10-09T08:47:31.246702+0000 mgr.y (mgr.25421) 286 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:32.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:32 smithi016 bash[77892]: cluster 2023-10-09T08:47:31.246702+0000 mgr.y (mgr.25421) 286 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:33.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T08:47:34.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[75179]: audit 2023-10-09T08:47:32.440999+0000 mgr.y (mgr.25421) 287 : audit 0 from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:34.563 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[75179]: cluster 2023-10-09T08:47:33.247578+0000 mgr.y (mgr.25421) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:34.563 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[75179]: audit 2023-10-09T08:47:34.208250+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:34.563 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[75179]: audit 2023-10-09T08:47:34.209570+0000 mon.a (mon.0) 459 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:34.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[77892]: audit 2023-10-09T08:47:32.440999+0000 mgr.y (mgr.25421) 287 : audit 0 from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:34.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[77892]: cluster 2023-10-09T08:47:33.247578+0000 mgr.y (mgr.25421) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:34.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[77892]: audit 2023-10-09T08:47:34.208250+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:34.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:34 smithi016 bash[77892]: audit 2023-10-09T08:47:34.209570+0000 mon.a (mon.0) 459 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:34 smithi094 bash[59677]: audit 2023-10-09T08:47:32.440999+0000 mgr.y (mgr.25421) 287 : audit 0 from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:47:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:34 smithi094 bash[59677]: cluster 2023-10-09T08:47:33.247578+0000 mgr.y (mgr.25421) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:34 smithi094 bash[59677]: audit 2023-10-09T08:47:34.208250+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:47:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:34 smithi094 bash[59677]: audit 2023-10-09T08:47:34.209570+0000 mon.a (mon.0) 459 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:36.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:47:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:47:35] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.33.4" 2023-10-09T08:47:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:36 smithi094 bash[59677]: cluster 2023-10-09T08:47:35.248532+0000 mgr.y (mgr.25421) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:36.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:36 smithi016 bash[75179]: cluster 2023-10-09T08:47:35.248532+0000 mgr.y (mgr.25421) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:36.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:36 smithi016 bash[77892]: cluster 2023-10-09T08:47:35.248532+0000 mgr.y (mgr.25421) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:36.864 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:47:36.864 INFO:teuthology.orchestra.run.smithi016.stdout: "mon": { 2023-10-09T08:47:36.864 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T08:47:36.864 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr": { 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "osd": { 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 4, 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "mds": { 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 2 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:47:36.865 INFO:teuthology.orchestra.run.smithi016.stdout: "overall": { 2023-10-09T08:47:36.866 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 6, 2023-10-09T08:47:36.866 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T08:47:36.866 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:47:36.866 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:47:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:37 smithi016 bash[77892]: audit 2023-10-09T08:47:36.859934+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.16:0/2549029479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:47:37.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:37 smithi016 bash[75179]: audit 2023-10-09T08:47:36.859934+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.16:0/2549029479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:47:37.706 DEBUG:teuthology.parallel:result is None 2023-10-09T08:47:37.707 INFO:teuthology.run_tasks:Running task print... 2023-10-09T08:47:37.721 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T08:47:37.721 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T08:47:37.748 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T08:47:37.749 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T08:47:37.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T08:47:37.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:37 smithi094 bash[59677]: audit 2023-10-09T08:47:36.859934+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.16:0/2549029479' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T08:47:38.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:38 smithi094 bash[59677]: cluster 2023-10-09T08:47:37.249223+0000 mgr.y (mgr.25421) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:38.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:38 smithi016 bash[75179]: cluster 2023-10-09T08:47:37.249223+0000 mgr.y (mgr.25421) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:38.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:38 smithi016 bash[77892]: cluster 2023-10-09T08:47:37.249223+0000 mgr.y (mgr.25421) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:40.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:39 smithi094 bash[59677]: cluster 2023-10-09T08:47:39.249871+0000 mgr.y (mgr.25421) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:40.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:39 smithi016 bash[75179]: cluster 2023-10-09T08:47:39.249871+0000 mgr.y (mgr.25421) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:40.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:39 smithi016 bash[77892]: cluster 2023-10-09T08:47:39.249871+0000 mgr.y (mgr.25421) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:42.531 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:47:42.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:42 smithi016 bash[75179]: cluster 2023-10-09T08:47:41.250890+0000 mgr.y (mgr.25421) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:42.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:42 smithi016 bash[77892]: cluster 2023-10-09T08:47:41.250890+0000 mgr.y (mgr.25421) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:42.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:42 smithi094 bash[59677]: cluster 2023-10-09T08:47:41.250890+0000 mgr.y (mgr.25421) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:44.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:44 smithi094 bash[59677]: cluster 2023-10-09T08:47:43.251705+0000 mgr.y (mgr.25421) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:44.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:44 smithi016 bash[75179]: cluster 2023-10-09T08:47:43.251705+0000 mgr.y (mgr.25421) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:44.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:44 smithi016 bash[77892]: cluster 2023-10-09T08:47:43.251705+0000 mgr.y (mgr.25421) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:45.993 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:47:46.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:47:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:47:45] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.33.4" 2023-10-09T08:47:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:46 smithi094 bash[59677]: cluster 2023-10-09T08:47:45.252634+0000 mgr.y (mgr.25421) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:46 smithi094 bash[59677]: audit 2023-10-09T08:47:45.975887+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.16:0/3124442843' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:47:46.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:46 smithi016 bash[75179]: cluster 2023-10-09T08:47:45.252634+0000 mgr.y (mgr.25421) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:46.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:46 smithi016 bash[75179]: audit 2023-10-09T08:47:45.975887+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.16:0/3124442843' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:47:46.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:46 smithi016 bash[77892]: cluster 2023-10-09T08:47:45.252634+0000 mgr.y (mgr.25421) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:46.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:46 smithi016 bash[77892]: audit 2023-10-09T08:47:45.975887+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.16:0/3124442843' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T08:47:46.827 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":37,"active_gid":25421,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6800","nonce":3077119964},{"type":"v1","addr":"172.21.15.16:6801","nonce":3077119964}]},"active_addr":"172.21.15.16:6801/3077119964","active_change":"2023-10-09T08:40:49.060119+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25240,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":74,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":4172388532}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":2454169267}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":2408130123}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":1989345438}]}]} 2023-10-09T08:47:46.832 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T08:47:46.832 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T08:47:46.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T08:47:48.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:48 smithi094 bash[59677]: cluster 2023-10-09T08:47:47.253465+0000 mgr.y (mgr.25421) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:48.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:48 smithi016 bash[75179]: cluster 2023-10-09T08:47:47.253465+0000 mgr.y (mgr.25421) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:48.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:48 smithi016 bash[77892]: cluster 2023-10-09T08:47:47.253465+0000 mgr.y (mgr.25421) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:50.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:49 smithi094 bash[59677]: audit 2023-10-09T08:47:49.202091+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:50.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:49 smithi094 bash[59677]: cluster 2023-10-09T08:47:49.254123+0000 mgr.y (mgr.25421) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:50.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:49 smithi016 bash[75179]: audit 2023-10-09T08:47:49.202091+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:50.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:49 smithi016 bash[75179]: cluster 2023-10-09T08:47:49.254123+0000 mgr.y (mgr.25421) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:50.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:49 smithi016 bash[77892]: audit 2023-10-09T08:47:49.202091+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:47:50.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:49 smithi016 bash[77892]: cluster 2023-10-09T08:47:49.254123+0000 mgr.y (mgr.25421) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:51.615 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:47:52.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:52 smithi016 bash[77892]: cluster 2023-10-09T08:47:51.255185+0000 mgr.y (mgr.25421) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:52.581 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:52 smithi016 bash[75179]: cluster 2023-10-09T08:47:51.255185+0000 mgr.y (mgr.25421) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:52.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:52 smithi094 bash[59677]: cluster 2023-10-09T08:47:51.255185+0000 mgr.y (mgr.25421) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:54 smithi094 bash[59677]: cluster 2023-10-09T08:47:53.255929+0000 mgr.y (mgr.25421) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:54.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:54 smithi016 bash[75179]: cluster 2023-10-09T08:47:53.255929+0000 mgr.y (mgr.25421) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:54.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:54 smithi016 bash[77892]: cluster 2023-10-09T08:47:53.255929+0000 mgr.y (mgr.25421) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:54.851 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:47:54.851 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":94,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:47:16.948445+0000","last_up_change":"2023-10-09T08:47:15.937734+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T08:34:26.406514+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T08:34:27.172366+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":74,"up_from":78,"up_thru":93,"down_at":75,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6803","nonce":2159748916}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6805","nonce":2159748916}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6809","nonce":2159748916}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6807","nonce":2159748916}]},"public_addr":"172.21.15.16:6803/2159748916","cluster_addr":"172.21.15.16:6805/2159748916","heartbeat_back_addr":"172.21.15.16:6809/2159748916","heartbeat_front_addr":"172.21.15.16:6807/2159748916","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":79,"up_from":83,"up_thru":93,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6811","nonce":3704674354}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6813","nonce":3704674354}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6817","nonce":3704674354}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6815","nonce":3704674354}]},"public_addr":"172.21.15.16:6811/3704674354","cluster_addr":"172.21.15.16:6813/3704674354","heartbeat_back_addr":"172.21.15.16:6817/3704674354","heartbeat_front_addr":"172.21.15.16:6815/3704674354","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":84,"up_from":88,"up_thru":93,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6819","nonce":1227549489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6821","nonce":1227549489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6825","nonce":1227549489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6823","nonce":1227549489}]},"public_addr":"172.21.15.16:6819/1227549489","cluster_addr":"172.21.15.16:6821/1227549489","heartbeat_back_addr":"172.21.15.16:6825/1227549489","heartbeat_front_addr":"172.21.15.16:6823/1227549489","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":89,"up_from":93,"up_thru":93,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6827","nonce":1617049985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6829","nonce":1617049985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6833","nonce":1617049985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6831","nonce":1617049985}]},"public_addr":"172.21.15.16:6827/1617049985","cluster_addr":"172.21.15.16:6829/1617049985","heartbeat_back_addr":"172.21.15.16:6833/1617049985","heartbeat_front_addr":"172.21.15.16:6831/1617049985","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":93,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":93,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":93,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":74},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":79},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":84},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":89},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/2419669477":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/3538653132":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/3387393375":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/3686891819":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/3915272728":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/1478513102":"2023-10-10T08:38:06.212642+0000","172.21.15.94:0/3558864062":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:6801/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.16:6801/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.16:0/1021312597":"2023-10-10T08:38:06.212642+0000","172.21.15.94:0/727051339":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.94:6834/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.94:0/2012987098":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.94:6835/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/2254945868":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.94:0/1375017130":"2023-10-10T08:39:43.331826+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:47:55.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:55 smithi016 bash[75179]: audit 2023-10-09T08:47:54.843804+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.16:0/682144921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:47:55.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:55 smithi016 bash[77892]: audit 2023-10-09T08:47:54.843804+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.16:0/682144921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:47:55.672 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T08:47:55.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T08:47:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:55 smithi094 bash[59677]: audit 2023-10-09T08:47:54.843804+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.16:0/682144921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:47:56.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:47:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:47:55] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-10-09T08:47:56.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:56 smithi094 bash[59677]: cluster 2023-10-09T08:47:55.256592+0000 mgr.y (mgr.25421) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:56.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:56 smithi016 bash[75179]: cluster 2023-10-09T08:47:55.256592+0000 mgr.y (mgr.25421) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:56.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:56 smithi016 bash[77892]: cluster 2023-10-09T08:47:55.256592+0000 mgr.y (mgr.25421) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:57.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:57 smithi094 bash[59677]: cluster 2023-10-09T08:47:57.257351+0000 mgr.y (mgr.25421) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:57.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:57 smithi016 bash[75179]: cluster 2023-10-09T08:47:57.257351+0000 mgr.y (mgr.25421) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:47:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:57 smithi016 bash[77892]: cluster 2023-10-09T08:47:57.257351+0000 mgr.y (mgr.25421) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:00.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:47:59 smithi094 bash[59677]: cluster 2023-10-09T08:47:59.257998+0000 mgr.y (mgr.25421) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:00.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:47:59 smithi016 bash[75179]: cluster 2023-10-09T08:47:59.257998+0000 mgr.y (mgr.25421) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:00.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:47:59 smithi016 bash[77892]: cluster 2023-10-09T08:47:59.257998+0000 mgr.y (mgr.25421) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:00.463 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:02.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:02 smithi094 bash[59677]: cluster 2023-10-09T08:48:01.258765+0000 mgr.y (mgr.25421) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:02.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:02 smithi016 bash[75179]: cluster 2023-10-09T08:48:01.258765+0000 mgr.y (mgr.25421) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:02.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:02 smithi016 bash[77892]: cluster 2023-10-09T08:48:01.258765+0000 mgr.y (mgr.25421) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:03.868 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:48:03.868 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":94,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T08:47:16.948445+0000","last_up_change":"2023-10-09T08:47:15.937734+0000","last_in_change":"2023-10-09T08:31:01.964465+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T08:34:26.406514+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T08:34:27.172366+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":74,"up_from":78,"up_thru":93,"down_at":75,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6803","nonce":2159748916}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6805","nonce":2159748916}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6809","nonce":2159748916}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6807","nonce":2159748916}]},"public_addr":"172.21.15.16:6803/2159748916","cluster_addr":"172.21.15.16:6805/2159748916","heartbeat_back_addr":"172.21.15.16:6809/2159748916","heartbeat_front_addr":"172.21.15.16:6807/2159748916","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":79,"up_from":83,"up_thru":93,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6811","nonce":3704674354}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6813","nonce":3704674354}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6817","nonce":3704674354}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6815","nonce":3704674354}]},"public_addr":"172.21.15.16:6811/3704674354","cluster_addr":"172.21.15.16:6813/3704674354","heartbeat_back_addr":"172.21.15.16:6817/3704674354","heartbeat_front_addr":"172.21.15.16:6815/3704674354","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":84,"up_from":88,"up_thru":93,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6819","nonce":1227549489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6821","nonce":1227549489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6825","nonce":1227549489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6823","nonce":1227549489}]},"public_addr":"172.21.15.16:6819/1227549489","cluster_addr":"172.21.15.16:6821/1227549489","heartbeat_back_addr":"172.21.15.16:6825/1227549489","heartbeat_front_addr":"172.21.15.16:6823/1227549489","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":89,"up_from":93,"up_thru":93,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6827","nonce":1617049985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6829","nonce":1617049985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6833","nonce":1617049985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6831","nonce":1617049985}]},"public_addr":"172.21.15.16:6827/1617049985","cluster_addr":"172.21.15.16:6829/1617049985","heartbeat_back_addr":"172.21.15.16:6833/1617049985","heartbeat_front_addr":"172.21.15.16:6831/1617049985","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":93,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":93,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":93,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":74},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":79},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":84},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":89},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2419669477":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/1478513102":"2023-10-10T08:38:06.212642+0000","172.21.15.16:0/3538653132":"2023-10-10T08:40:49.059520+0000","172.21.15.94:0/3558864062":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.94:0/1375017130":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/3686891819":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.94:0/2012987098":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.94:0/727051339":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/3915272728":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/1021312597":"2023-10-10T08:38:06.212642+0000","172.21.15.16:0/3387393375":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.16:6801/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.94:6834/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.94:6835/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/2254945868":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T08:48:04.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:04 smithi016 bash[75179]: cluster 2023-10-09T08:48:03.259580+0000 mgr.y (mgr.25421) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:04.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:04 smithi016 bash[75179]: audit 2023-10-09T08:48:03.861182+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.16:0/1968377897' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:48:04.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:04 smithi016 bash[75179]: audit 2023-10-09T08:48:04.202813+0000 mon.a (mon.0) 462 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:04.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:04 smithi016 bash[77892]: cluster 2023-10-09T08:48:03.259580+0000 mgr.y (mgr.25421) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:04 smithi016 bash[77892]: audit 2023-10-09T08:48:03.861182+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.16:0/1968377897' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:48:04.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:04 smithi016 bash[77892]: audit 2023-10-09T08:48:04.202813+0000 mon.a (mon.0) 462 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:04.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T08:48:04.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T08:48:04.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T08:48:04.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T08:48:04.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T08:48:04.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T08:48:04.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T08:48:04.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T08:48:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:04 smithi094 bash[59677]: cluster 2023-10-09T08:48:03.259580+0000 mgr.y (mgr.25421) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:04 smithi094 bash[59677]: audit 2023-10-09T08:48:03.861182+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.16:0/1968377897' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T08:48:04.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:04 smithi094 bash[59677]: audit 2023-10-09T08:48:04.202813+0000 mon.a (mon.0) 462 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:05.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:05 smithi016 bash[75179]: cluster 2023-10-09T08:48:05.260541+0000 mgr.y (mgr.25421) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:05.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:05 smithi016 bash[77892]: cluster 2023-10-09T08:48:05.260541+0000 mgr.y (mgr.25421) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:05.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:05 smithi094 bash[59677]: cluster 2023-10-09T08:48:05.260541+0000 mgr.y (mgr.25421) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:06.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:48:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:48:05] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T08:48:08.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:08 smithi094 bash[59677]: cluster 2023-10-09T08:48:07.261250+0000 mgr.y (mgr.25421) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:08.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:08 smithi016 bash[75179]: cluster 2023-10-09T08:48:07.261250+0000 mgr.y (mgr.25421) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:08.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:08 smithi016 bash[77892]: cluster 2023-10-09T08:48:07.261250+0000 mgr.y (mgr.25421) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:09.614 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:09.618 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:09.618 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:09.618 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:09.624 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:09.624 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:09.625 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:09.625 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:10.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:09 smithi016 bash[75179]: cluster 2023-10-09T08:48:09.261970+0000 mgr.y (mgr.25421) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:10.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:09 smithi016 bash[77892]: cluster 2023-10-09T08:48:09.261970+0000 mgr.y (mgr.25421) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:09 smithi094 bash[59677]: cluster 2023-10-09T08:48:09.261970+0000 mgr.y (mgr.25421) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:12.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:12 smithi016 bash[75179]: cluster 2023-10-09T08:48:11.262933+0000 mgr.y (mgr.25421) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:12.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:12 smithi016 bash[77892]: cluster 2023-10-09T08:48:11.262933+0000 mgr.y (mgr.25421) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:12.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:12 smithi094 bash[59677]: cluster 2023-10-09T08:48:11.262933+0000 mgr.y (mgr.25421) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:14 smithi094 bash[59677]: cluster 2023-10-09T08:48:13.263774+0000 mgr.y (mgr.25421) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:14.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:14 smithi016 bash[75179]: cluster 2023-10-09T08:48:13.263774+0000 mgr.y (mgr.25421) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:14.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:14 smithi016 bash[77892]: cluster 2023-10-09T08:48:13.263774+0000 mgr.y (mgr.25421) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:15.575 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:15 smithi016 bash[75179]: cluster 2023-10-09T08:48:15.264368+0000 mgr.y (mgr.25421) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:15.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:15 smithi016 bash[77892]: cluster 2023-10-09T08:48:15.264368+0000 mgr.y (mgr.25421) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:15 smithi094 bash[59677]: cluster 2023-10-09T08:48:15.264368+0000 mgr.y (mgr.25421) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:16.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:48:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:48:15] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T08:48:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:18 smithi094 bash[59677]: cluster 2023-10-09T08:48:17.264820+0000 mgr.y (mgr.25421) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:18.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:18 smithi016 bash[75179]: cluster 2023-10-09T08:48:17.264820+0000 mgr.y (mgr.25421) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:18.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:18 smithi016 bash[77892]: cluster 2023-10-09T08:48:17.264820+0000 mgr.y (mgr.25421) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:20.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:19 smithi016 bash[75179]: audit 2023-10-09T08:48:19.202537+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:20.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:19 smithi016 bash[75179]: cluster 2023-10-09T08:48:19.265354+0000 mgr.y (mgr.25421) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:20.220 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:19 smithi016 bash[77892]: audit 2023-10-09T08:48:19.202537+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:20.221 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:19 smithi016 bash[77892]: cluster 2023-10-09T08:48:19.265354+0000 mgr.y (mgr.25421) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:19 smithi094 bash[59677]: audit 2023-10-09T08:48:19.202537+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:19 smithi094 bash[59677]: cluster 2023-10-09T08:48:19.265354+0000 mgr.y (mgr.25421) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:22.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:22 smithi016 bash[75179]: cluster 2023-10-09T08:48:21.266387+0000 mgr.y (mgr.25421) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:22.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:22 smithi016 bash[77892]: cluster 2023-10-09T08:48:21.266387+0000 mgr.y (mgr.25421) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:22.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:22 smithi094 bash[59677]: cluster 2023-10-09T08:48:21.266387+0000 mgr.y (mgr.25421) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:23.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:23 smithi016 bash[75179]: cluster 2023-10-09T08:48:23.267164+0000 mgr.y (mgr.25421) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:23.635 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:23 smithi016 bash[77892]: cluster 2023-10-09T08:48:23.267164+0000 mgr.y (mgr.25421) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:23.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:23 smithi094 bash[59677]: cluster 2023-10-09T08:48:23.267164+0000 mgr.y (mgr.25421) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:24.265 INFO:teuthology.orchestra.run.smithi016.stdout:377957122068 2023-10-09T08:48:24.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T08:48:24.779 INFO:teuthology.orchestra.run.smithi016.stdout:356482285594 2023-10-09T08:48:24.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T08:48:26.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:48:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:48:25] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-10-09T08:48:26.582 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:26 smithi016 bash[75179]: cluster 2023-10-09T08:48:25.267714+0000 mgr.y (mgr.25421) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:26.582 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:26 smithi016 bash[77892]: cluster 2023-10-09T08:48:25.267714+0000 mgr.y (mgr.25421) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:26.597 INFO:teuthology.orchestra.run.smithi016.stdout:206158430414 2023-10-09T08:48:26.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T08:48:26.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:26 smithi094 bash[59677]: cluster 2023-10-09T08:48:25.267714+0000 mgr.y (mgr.25421) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:26.959 INFO:teuthology.orchestra.run.smithi016.stdout:180388626647 2023-10-09T08:48:26.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T08:48:27.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:27 smithi016 bash[75179]: audit 2023-10-09T08:48:27.241398+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:48:27.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:27 smithi016 bash[77892]: audit 2023-10-09T08:48:27.241398+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:48:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:27 smithi094 bash[59677]: audit 2023-10-09T08:48:27.241398+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:48:28.068 INFO:teuthology.orchestra.run.smithi016.stdout:154618822881 2023-10-09T08:48:28.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T08:48:28.693 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:28 smithi016 bash[75179]: cluster 2023-10-09T08:48:27.268503+0000 mgr.y (mgr.25421) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:28.693 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:28 smithi016 bash[77892]: cluster 2023-10-09T08:48:27.268503+0000 mgr.y (mgr.25421) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:28.694 INFO:teuthology.orchestra.run.smithi016.stdout:399431958544 2023-10-09T08:48:28.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T08:48:28.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:28 smithi094 bash[59677]: cluster 2023-10-09T08:48:27.268503+0000 mgr.y (mgr.25421) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:29.476 INFO:teuthology.orchestra.run.smithi016.stdout:128849019114 2023-10-09T08:48:29.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T08:48:29.489 INFO:teuthology.orchestra.run.smithi016.stdout:335007449121 2023-10-09T08:48:29.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T08:48:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:29 smithi094 bash[59677]: cluster 2023-10-09T08:48:29.269077+0000 mgr.y (mgr.25421) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:30.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:29 smithi016 bash[75179]: cluster 2023-10-09T08:48:29.269077+0000 mgr.y (mgr.25421) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:29 smithi016 bash[77892]: cluster 2023-10-09T08:48:29.269077+0000 mgr.y (mgr.25421) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:32.083 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:32.091 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:32.092 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:32.092 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:32.092 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:32.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:32 smithi016 bash[75179]: cluster 2023-10-09T08:48:31.270027+0000 mgr.y (mgr.25421) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:32.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:32 smithi016 bash[77892]: cluster 2023-10-09T08:48:31.270027+0000 mgr.y (mgr.25421) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:32.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:32 smithi094 bash[59677]: cluster 2023-10-09T08:48:31.270027+0000 mgr.y (mgr.25421) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:33.103 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:34.110 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:34.110 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: cluster 2023-10-09T08:48:33.270564+0000 mgr.y (mgr.25421) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:33.281234+0000 mon.a (mon.0) 465 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:33.288368+0000 mon.a (mon.0) 466 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:33.433463+0000 mon.a (mon.0) 467 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:33.439924+0000 mon.a (mon.0) 468 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:33.916151+0000 mon.a (mon.0) 469 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:33.917676+0000 mon.a (mon.0) 470 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:33.925974+0000 mon.a (mon.0) 471 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[77892]: audit 2023-10-09T08:48:34.202970+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: cluster 2023-10-09T08:48:33.270564+0000 mgr.y (mgr.25421) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:33.281234+0000 mon.a (mon.0) 465 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:33.288368+0000 mon.a (mon.0) 466 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:33.433463+0000 mon.a (mon.0) 467 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:33.439924+0000 mon.a (mon.0) 468 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:33.916151+0000 mon.a (mon.0) 469 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:48:34.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:33.917676+0000 mon.a (mon.0) 470 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:48:34.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:33.925974+0000 mon.a (mon.0) 471 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:34 smithi016 bash[75179]: audit 2023-10-09T08:48:34.202970+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: cluster 2023-10-09T08:48:33.270564+0000 mgr.y (mgr.25421) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:33.281234+0000 mon.a (mon.0) 465 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:33.288368+0000 mon.a (mon.0) 466 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:33.433463+0000 mon.a (mon.0) 467 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:33.439924+0000 mon.a (mon.0) 468 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:33.916151+0000 mon.a (mon.0) 469 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:48:34.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:33.917676+0000 mon.a (mon.0) 470 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:48:34.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:33.925974+0000 mon.a (mon.0) 471 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:48:34.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:34 smithi094 bash[59677]: audit 2023-10-09T08:48:34.202970+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:35.751 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:48:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:48:35] "GET /metrics HTTP/1.1" 200 33877 "" "Prometheus/2.33.4" 2023-10-09T08:48:36.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:36 smithi016 bash[77892]: cluster 2023-10-09T08:48:35.271224+0000 mgr.y (mgr.25421) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:36.547 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:36 smithi016 bash[75179]: cluster 2023-10-09T08:48:35.271224+0000 mgr.y (mgr.25421) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:36.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:36 smithi094 bash[59677]: cluster 2023-10-09T08:48:35.271224+0000 mgr.y (mgr.25421) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:38.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:38 smithi016 bash[75179]: cluster 2023-10-09T08:48:37.272011+0000 mgr.y (mgr.25421) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:38.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:38 smithi016 bash[77892]: cluster 2023-10-09T08:48:37.272011+0000 mgr.y (mgr.25421) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:38.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:38 smithi094 bash[59677]: cluster 2023-10-09T08:48:37.272011+0000 mgr.y (mgr.25421) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:40.180 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:39 smithi016 bash[75179]: cluster 2023-10-09T08:48:39.272623+0000 mgr.y (mgr.25421) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:40.180 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:39 smithi016 bash[77892]: cluster 2023-10-09T08:48:39.272623+0000 mgr.y (mgr.25421) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:40.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:39 smithi094 bash[59677]: cluster 2023-10-09T08:48:39.272623+0000 mgr.y (mgr.25421) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:41.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:41 smithi094 bash[59677]: cluster 2023-10-09T08:48:41.273457+0000 mgr.y (mgr.25421) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:41.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:41 smithi016 bash[77892]: cluster 2023-10-09T08:48:41.273457+0000 mgr.y (mgr.25421) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:41.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:41 smithi016 bash[75179]: cluster 2023-10-09T08:48:41.273457+0000 mgr.y (mgr.25421) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:43.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:43 smithi016 bash[77892]: cluster 2023-10-09T08:48:43.274008+0000 mgr.y (mgr.25421) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:43.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:43 smithi016 bash[75179]: cluster 2023-10-09T08:48:43.274008+0000 mgr.y (mgr.25421) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:43.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:43 smithi094 bash[59677]: cluster 2023-10-09T08:48:43.274008+0000 mgr.y (mgr.25421) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:45.205 INFO:teuthology.orchestra.run.smithi016.stdout:154618822885 2023-10-09T08:48:45.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:45 smithi016 bash[75179]: audit 2023-10-09T08:48:45.201925+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.16:0/1023421391' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:48:45.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:45 smithi016 bash[77892]: audit 2023-10-09T08:48:45.201925+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.16:0/1023421391' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:48:45.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:45 smithi094 bash[59677]: audit 2023-10-09T08:48:45.201925+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.16:0/1023421391' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T08:48:45.999 INFO:teuthology.orchestra.run.smithi016.stdout:206158430418 2023-10-09T08:48:46.038 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:48:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:48:45] "GET /metrics HTTP/1.1" 200 33877 "" "Prometheus/2.33.4" 2023-10-09T08:48:46.114 INFO:teuthology.orchestra.run.smithi016.stdout:356482285599 2023-10-09T08:48:46.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:46 smithi016 bash[75179]: cluster 2023-10-09T08:48:45.274968+0000 mgr.y (mgr.25421) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:46.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:46 smithi016 bash[75179]: audit 2023-10-09T08:48:45.996688+0000 mon.a (mon.0) 473 : audit 0 from='client.? 172.21.15.16:0/1455231411' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:48:46.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:46 smithi016 bash[75179]: audit 2023-10-09T08:48:46.111697+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.16:0/4087110695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:48:46.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:46 smithi016 bash[77892]: cluster 2023-10-09T08:48:45.274968+0000 mgr.y (mgr.25421) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:46.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:46 smithi016 bash[77892]: audit 2023-10-09T08:48:45.996688+0000 mon.a (mon.0) 473 : audit 0 from='client.? 172.21.15.16:0/1455231411' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:48:46.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:46 smithi016 bash[77892]: audit 2023-10-09T08:48:46.111697+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.16:0/4087110695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:48:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:46 smithi094 bash[59677]: cluster 2023-10-09T08:48:45.274968+0000 mgr.y (mgr.25421) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:46 smithi094 bash[59677]: audit 2023-10-09T08:48:45.996688+0000 mon.a (mon.0) 473 : audit 0 from='client.? 172.21.15.16:0/1455231411' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T08:48:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:46 smithi094 bash[59677]: audit 2023-10-09T08:48:46.111697+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.16:0/4087110695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T08:48:47.198 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822881 got 154618822885 for osd.5 2023-10-09T08:48:47.198 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:47.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:47 smithi016 bash[75179]: cluster 2023-10-09T08:48:47.275711+0000 mgr.y (mgr.25421) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:47.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:47 smithi016 bash[77892]: cluster 2023-10-09T08:48:47.275711+0000 mgr.y (mgr.25421) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:47.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:47 smithi094 bash[59677]: cluster 2023-10-09T08:48:47.275711+0000 mgr.y (mgr.25421) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:47.760 INFO:teuthology.orchestra.run.smithi016.stdout:180388626652 2023-10-09T08:48:48.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:48 smithi016 bash[77892]: audit 2023-10-09T08:48:47.757027+0000 mon.a (mon.0) 474 : audit 0 from='client.? 172.21.15.16:0/3999113427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:48:48.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:48 smithi016 bash[75179]: audit 2023-10-09T08:48:47.757027+0000 mon.a (mon.0) 474 : audit 0 from='client.? 172.21.15.16:0/3999113427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:48:48.477 INFO:tasks.cephadm.ceph_manager.ceph:need seq 356482285594 got 356482285599 for osd.1 2023-10-09T08:48:48.477 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:48.628 INFO:teuthology.orchestra.run.smithi016.stdout:335007449126 2023-10-09T08:48:48.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:48 smithi094 bash[59677]: audit 2023-10-09T08:48:47.757027+0000 mon.a (mon.0) 474 : audit 0 from='client.? 172.21.15.16:0/3999113427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T08:48:49.081 INFO:teuthology.orchestra.run.smithi016.stdout:377957122074 2023-10-09T08:48:49.216 INFO:teuthology.orchestra.run.smithi016.stdout:399431958548 2023-10-09T08:48:49.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:49 smithi094 bash[59677]: audit 2023-10-09T08:48:48.625263+0000 mon.a (mon.0) 475 : audit 0 from='client.? 172.21.15.16:0/745615477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:48:49.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:49 smithi094 bash[59677]: audit 2023-10-09T08:48:49.078536+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.16:0/414965086' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:48:49.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:49 smithi094 bash[59677]: audit 2023-10-09T08:48:49.203429+0000 mon.a (mon.0) 477 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:49.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:49 smithi094 bash[59677]: audit 2023-10-09T08:48:49.212738+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.16:0/3094548858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:48:49.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:49 smithi094 bash[59677]: cluster 2023-10-09T08:48:49.276100+0000 mgr.y (mgr.25421) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[75179]: audit 2023-10-09T08:48:48.625263+0000 mon.a (mon.0) 475 : audit 0 from='client.? 172.21.15.16:0/745615477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[75179]: audit 2023-10-09T08:48:49.078536+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.16:0/414965086' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[75179]: audit 2023-10-09T08:48:49.203429+0000 mon.a (mon.0) 477 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[75179]: audit 2023-10-09T08:48:49.212738+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.16:0/3094548858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[75179]: cluster 2023-10-09T08:48:49.276100+0000 mgr.y (mgr.25421) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[77892]: audit 2023-10-09T08:48:48.625263+0000 mon.a (mon.0) 475 : audit 0 from='client.? 172.21.15.16:0/745615477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[77892]: audit 2023-10-09T08:48:49.078536+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.16:0/414965086' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T08:48:49.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[77892]: audit 2023-10-09T08:48:49.203429+0000 mon.a (mon.0) 477 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:48:49.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[77892]: audit 2023-10-09T08:48:49.212738+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.16:0/3094548858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T08:48:49.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:49 smithi016 bash[77892]: cluster 2023-10-09T08:48:49.276100+0000 mgr.y (mgr.25421) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:50.024 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430414 got 206158430418 for osd.7 2023-10-09T08:48:50.025 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:50.458 INFO:teuthology.orchestra.run.smithi016.stdout:128849019118 2023-10-09T08:48:50.910 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:50 smithi016 bash[75179]: audit 2023-10-09T08:48:50.454904+0000 mon.a (mon.0) 478 : audit 0 from='client.? 172.21.15.16:0/2119448859' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:48:51.156 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:50 smithi016 bash[77892]: audit 2023-10-09T08:48:50.454904+0000 mon.a (mon.0) 478 : audit 0 from='client.? 172.21.15.16:0/2119448859' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:48:51.157 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626647 got 180388626652 for osd.6 2023-10-09T08:48:51.157 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:51.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:50 smithi094 bash[59677]: audit 2023-10-09T08:48:50.454904+0000 mon.a (mon.0) 478 : audit 0 from='client.? 172.21.15.16:0/2119448859' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T08:48:52.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:51 smithi094 bash[59677]: cluster 2023-10-09T08:48:51.276822+0000 mgr.y (mgr.25421) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:52.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:51 smithi016 bash[75179]: cluster 2023-10-09T08:48:51.276822+0000 mgr.y (mgr.25421) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:52.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:51 smithi016 bash[77892]: cluster 2023-10-09T08:48:51.276822+0000 mgr.y (mgr.25421) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:52.319 INFO:tasks.cephadm.ceph_manager.ceph:need seq 335007449121 got 335007449126 for osd.0 2023-10-09T08:48:52.319 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:52.469 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958544 got 399431958548 for osd.3 2023-10-09T08:48:52.470 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:52.916 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122068 got 377957122074 for osd.2 2023-10-09T08:48:52.917 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:52.926 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019114 got 128849019118 for osd.4 2023-10-09T08:48:52.926 DEBUG:teuthology.parallel:result is None 2023-10-09T08:48:52.926 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T08:48:52.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:48:53.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:53 smithi094 bash[59677]: cluster 2023-10-09T08:48:53.277603+0000 mgr.y (mgr.25421) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:53 smithi016 bash[75179]: cluster 2023-10-09T08:48:53.277603+0000 mgr.y (mgr.25421) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:53.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:53 smithi016 bash[77892]: cluster 2023-10-09T08:48:53.277603+0000 mgr.y (mgr.25421) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:55 smithi016 bash[75179]: cluster 2023-10-09T08:48:55.278595+0000 mgr.y (mgr.25421) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:55 smithi016 bash[77892]: cluster 2023-10-09T08:48:55.278595+0000 mgr.y (mgr.25421) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:55.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:55 smithi094 bash[59677]: cluster 2023-10-09T08:48:55.278595+0000 mgr.y (mgr.25421) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:56.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:48:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:48:55] "GET /metrics HTTP/1.1" 200 33880 "" "Prometheus/2.33.4" 2023-10-09T08:48:57.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:57 smithi094 bash[59677]: cluster 2023-10-09T08:48:57.279333+0000 mgr.y (mgr.25421) 330 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:57.746 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:48:57.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:57 smithi016 bash[75179]: cluster 2023-10-09T08:48:57.279333+0000 mgr.y (mgr.25421) 330 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:48:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:57 smithi016 bash[77892]: cluster 2023-10-09T08:48:57.279333+0000 mgr.y (mgr.25421) 330 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:00.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:48:59 smithi094 bash[59677]: cluster 2023-10-09T08:48:59.280019+0000 mgr.y (mgr.25421) 331 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:00.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:48:59 smithi016 bash[75179]: cluster 2023-10-09T08:48:59.280019+0000 mgr.y (mgr.25421) 331 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:00.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:48:59 smithi016 bash[77892]: cluster 2023-10-09T08:48:59.280019+0000 mgr.y (mgr.25421) 331 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:00.834 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:49:00.836 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:49:01.582 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":268,"stamp":"2023-10-09T08:48:59.279636+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1352296,"kb_used_data":9320,"kb_used_omap":126,"kb_used_meta":1342785,"kb_avail":748608920,"statfs":{"total":767960285184,"available":766575534080,"internally_reserved":0,"allocated":9543680,"data_stored":5876632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1375012475},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004391"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950127+0000","last_change":"2023-10-09T08:46:18.676602+0000","last_active":"2023-10-09T08:47:15.950127+0000","last_peered":"2023-10-09T08:47:15.950127+0000","last_clean":"2023-10-09T08:47:15.950127+0000","last_became_active":"2023-10-09T08:46:18.676122+0000","last_became_peered":"2023-10-09T08:46:18.676122+0000","last_unstale":"2023-10-09T08:47:15.950127+0000","last_undegraded":"2023-10-09T08:47:15.950127+0000","last_fullsized":"2023-10-09T08:47:15.950127+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953679+0000","last_change":"2023-10-09T08:46:49.233211+0000","last_active":"2023-10-09T08:47:16.953679+0000","last_peered":"2023-10-09T08:47:16.953679+0000","last_clean":"2023-10-09T08:47:16.953679+0000","last_became_active":"2023-10-09T08:46:49.232775+0000","last_became_peered":"2023-10-09T08:46:49.232775+0000","last_unstale":"2023-10-09T08:47:16.953679+0000","last_undegraded":"2023-10-09T08:47:16.953679+0000","last_fullsized":"2023-10-09T08:47:16.953679+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:59.423857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.966115+0000","last_change":"2023-10-09T08:47:16.966115+0000","last_active":"2023-10-09T08:47:16.966115+0000","last_peered":"2023-10-09T08:47:16.966115+0000","last_clean":"2023-10-09T08:47:16.966115+0000","last_became_active":"2023-10-09T08:47:16.965795+0000","last_became_peered":"2023-10-09T08:47:16.965795+0000","last_unstale":"2023-10-09T08:47:16.966115+0000","last_undegraded":"2023-10-09T08:47:16.966115+0000","last_fullsized":"2023-10-09T08:47:16.966115+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:40.817840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.970132+0000","last_change":"2023-10-09T08:47:16.970132+0000","last_active":"2023-10-09T08:47:16.970132+0000","last_peered":"2023-10-09T08:47:16.970132+0000","last_clean":"2023-10-09T08:47:16.970132+0000","last_became_active":"2023-10-09T08:47:16.969760+0000","last_became_peered":"2023-10-09T08:47:16.969760+0000","last_unstale":"2023-10-09T08:47:16.970132+0000","last_undegraded":"2023-10-09T08:47:16.970132+0000","last_fullsized":"2023-10-09T08:47:16.970132+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.801830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969367+0000","last_change":"2023-10-09T08:47:16.969367+0000","last_active":"2023-10-09T08:47:16.969367+0000","last_peered":"2023-10-09T08:47:16.969367+0000","last_clean":"2023-10-09T08:47:16.969367+0000","last_became_active":"2023-10-09T08:47:16.969000+0000","last_became_peered":"2023-10-09T08:47:16.969000+0000","last_unstale":"2023-10-09T08:47:16.969367+0000","last_undegraded":"2023-10-09T08:47:16.969367+0000","last_fullsized":"2023-10-09T08:47:16.969367+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961355+0000","last_change":"2023-10-09T08:34:27.739113+0000","last_active":"2023-10-09T08:47:15.961355+0000","last_peered":"2023-10-09T08:47:15.961355+0000","last_clean":"2023-10-09T08:47:15.961355+0000","last_became_active":"2023-10-09T08:34:27.738645+0000","last_became_peered":"2023-10-09T08:34:27.738645+0000","last_unstale":"2023-10-09T08:47:15.961355+0000","last_undegraded":"2023-10-09T08:47:15.961355+0000","last_fullsized":"2023-10-09T08:47:15.961355+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968742+0000","last_change":"2023-10-09T08:47:16.968742+0000","last_active":"2023-10-09T08:47:16.968742+0000","last_peered":"2023-10-09T08:47:16.968742+0000","last_clean":"2023-10-09T08:47:16.968742+0000","last_became_active":"2023-10-09T08:47:16.968317+0000","last_became_peered":"2023-10-09T08:47:16.968317+0000","last_unstale":"2023-10-09T08:47:16.968742+0000","last_undegraded":"2023-10-09T08:47:16.968742+0000","last_fullsized":"2023-10-09T08:47:16.968742+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962594+0000","last_change":"2023-10-09T08:46:18.677157+0000","last_active":"2023-10-09T08:47:15.962594+0000","last_peered":"2023-10-09T08:47:15.962594+0000","last_clean":"2023-10-09T08:47:15.962594+0000","last_became_active":"2023-10-09T08:46:18.676750+0000","last_became_peered":"2023-10-09T08:46:18.676750+0000","last_unstale":"2023-10-09T08:47:15.962594+0000","last_undegraded":"2023-10-09T08:47:15.962594+0000","last_fullsized":"2023-10-09T08:47:15.962594+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.964981+0000","last_change":"2023-10-09T08:47:16.964981+0000","last_active":"2023-10-09T08:47:16.964981+0000","last_peered":"2023-10-09T08:47:16.964981+0000","last_clean":"2023-10-09T08:47:16.964981+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.964981+0000","last_undegraded":"2023-10-09T08:47:16.964981+0000","last_fullsized":"2023-10-09T08:47:16.964981+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:04.336414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953808+0000","last_change":"2023-10-09T08:45:49.184830+0000","last_active":"2023-10-09T08:47:16.953808+0000","last_peered":"2023-10-09T08:47:16.953808+0000","last_clean":"2023-10-09T08:47:16.953808+0000","last_became_active":"2023-10-09T08:45:49.184434+0000","last_became_peered":"2023-10-09T08:45:49.184434+0000","last_unstale":"2023-10-09T08:47:16.953808+0000","last_undegraded":"2023-10-09T08:47:16.953808+0000","last_fullsized":"2023-10-09T08:47:16.953808+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:41.880170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950161+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:47:15.950161+0000","last_peered":"2023-10-09T08:47:15.950161+0000","last_clean":"2023-10-09T08:47:15.950161+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:47:15.950161+0000","last_undegraded":"2023-10-09T08:47:15.950161+0000","last_fullsized":"2023-10-09T08:47:15.950161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954285+0000","last_change":"2023-10-09T08:46:18.683082+0000","last_active":"2023-10-09T08:47:16.954285+0000","last_peered":"2023-10-09T08:47:16.954285+0000","last_clean":"2023-10-09T08:47:16.954285+0000","last_became_active":"2023-10-09T08:46:18.682657+0000","last_became_peered":"2023-10-09T08:46:18.682657+0000","last_unstale":"2023-10-09T08:47:16.954285+0000","last_undegraded":"2023-10-09T08:47:16.954285+0000","last_fullsized":"2023-10-09T08:47:16.954285+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:45.531585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956212+0000","last_change":"2023-10-09T08:46:49.232591+0000","last_active":"2023-10-09T08:47:15.956212+0000","last_peered":"2023-10-09T08:47:15.956212+0000","last_clean":"2023-10-09T08:47:15.956212+0000","last_became_active":"2023-10-09T08:46:49.232216+0000","last_became_peered":"2023-10-09T08:46:49.232216+0000","last_unstale":"2023-10-09T08:47:15.956212+0000","last_undegraded":"2023-10-09T08:47:15.956212+0000","last_fullsized":"2023-10-09T08:47:15.956212+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968372+0000","last_change":"2023-10-09T08:47:16.968372+0000","last_active":"2023-10-09T08:47:16.968372+0000","last_peered":"2023-10-09T08:47:16.968372+0000","last_clean":"2023-10-09T08:47:16.968372+0000","last_became_active":"2023-10-09T08:47:16.968024+0000","last_became_peered":"2023-10-09T08:47:16.968024+0000","last_unstale":"2023-10-09T08:47:16.968372+0000","last_undegraded":"2023-10-09T08:47:16.968372+0000","last_fullsized":"2023-10-09T08:47:16.968372+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:07.642684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962677+0000","last_change":"2023-10-09T08:46:18.678094+0000","last_active":"2023-10-09T08:47:16.962677+0000","last_peered":"2023-10-09T08:47:16.962677+0000","last_clean":"2023-10-09T08:47:16.962677+0000","last_became_active":"2023-10-09T08:46:18.677705+0000","last_became_peered":"2023-10-09T08:46:18.677705+0000","last_unstale":"2023-10-09T08:47:16.962677+0000","last_undegraded":"2023-10-09T08:47:16.962677+0000","last_fullsized":"2023-10-09T08:47:16.962677+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:31.647828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974508+0000","last_change":"2023-10-09T08:47:16.974508+0000","last_active":"2023-10-09T08:47:16.974508+0000","last_peered":"2023-10-09T08:47:16.974508+0000","last_clean":"2023-10-09T08:47:16.974508+0000","last_became_active":"2023-10-09T08:47:16.974255+0000","last_became_peered":"2023-10-09T08:47:16.974255+0000","last_unstale":"2023-10-09T08:47:16.974508+0000","last_undegraded":"2023-10-09T08:47:16.974508+0000","last_fullsized":"2023-10-09T08:47:16.974508+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:40.049955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953435+0000","last_change":"2023-10-09T08:45:49.182610+0000","last_active":"2023-10-09T08:47:16.953435+0000","last_peered":"2023-10-09T08:47:16.953435+0000","last_clean":"2023-10-09T08:47:16.953435+0000","last_became_active":"2023-10-09T08:45:49.182013+0000","last_became_peered":"2023-10-09T08:45:49.182013+0000","last_unstale":"2023-10-09T08:47:16.953435+0000","last_undegraded":"2023-10-09T08:47:16.953435+0000","last_fullsized":"2023-10-09T08:47:16.953435+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:44.316894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962365+0000","last_change":"2023-10-09T08:46:49.233112+0000","last_active":"2023-10-09T08:47:16.962365+0000","last_peered":"2023-10-09T08:47:16.962365+0000","last_clean":"2023-10-09T08:47:16.962365+0000","last_became_active":"2023-10-09T08:46:49.232724+0000","last_became_peered":"2023-10-09T08:46:49.232724+0000","last_unstale":"2023-10-09T08:47:16.962365+0000","last_undegraded":"2023-10-09T08:47:16.962365+0000","last_fullsized":"2023-10-09T08:47:16.962365+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.285159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961023+0000","last_change":"2023-10-09T08:45:49.181115+0000","last_active":"2023-10-09T08:47:15.961023+0000","last_peered":"2023-10-09T08:47:15.961023+0000","last_clean":"2023-10-09T08:47:15.961023+0000","last_became_active":"2023-10-09T08:45:49.180736+0000","last_became_peered":"2023-10-09T08:45:49.180736+0000","last_unstale":"2023-10-09T08:47:15.961023+0000","last_undegraded":"2023-10-09T08:47:15.961023+0000","last_fullsized":"2023-10-09T08:47:15.961023+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954452+0000","last_change":"2023-10-09T08:46:49.232279+0000","last_active":"2023-10-09T08:47:16.954452+0000","last_peered":"2023-10-09T08:47:16.954452+0000","last_clean":"2023-10-09T08:47:16.954452+0000","last_became_active":"2023-10-09T08:46:49.231911+0000","last_became_peered":"2023-10-09T08:46:49.231911+0000","last_unstale":"2023-10-09T08:47:16.954452+0000","last_undegraded":"2023-10-09T08:47:16.954452+0000","last_fullsized":"2023-10-09T08:47:16.954452+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:38.497218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965912+0000","last_change":"2023-10-09T08:47:16.965497+0000","last_active":"2023-10-09T08:47:16.965912+0000","last_peered":"2023-10-09T08:47:16.965912+0000","last_clean":"2023-10-09T08:47:16.965912+0000","last_became_active":"2023-10-09T08:47:16.965181+0000","last_became_peered":"2023-10-09T08:47:16.965181+0000","last_unstale":"2023-10-09T08:47:16.965912+0000","last_undegraded":"2023-10-09T08:47:16.965912+0000","last_fullsized":"2023-10-09T08:47:16.965912+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:52.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956282+0000","last_change":"2023-10-09T08:46:18.680004+0000","last_active":"2023-10-09T08:47:15.956282+0000","last_peered":"2023-10-09T08:47:15.956282+0000","last_clean":"2023-10-09T08:47:15.956282+0000","last_became_active":"2023-10-09T08:46:18.679617+0000","last_became_peered":"2023-10-09T08:46:18.679617+0000","last_unstale":"2023-10-09T08:47:15.956282+0000","last_undegraded":"2023-10-09T08:47:15.956282+0000","last_fullsized":"2023-10-09T08:47:15.956282+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956609+0000","last_change":"2023-10-09T08:34:27.860287+0000","last_active":"2023-10-09T08:47:15.956609+0000","last_peered":"2023-10-09T08:47:15.956609+0000","last_clean":"2023-10-09T08:47:15.956609+0000","last_became_active":"2023-10-09T08:34:27.859851+0000","last_became_peered":"2023-10-09T08:34:27.859851+0000","last_unstale":"2023-10-09T08:47:15.956609+0000","last_undegraded":"2023-10-09T08:47:15.956609+0000","last_fullsized":"2023-10-09T08:47:15.956609+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973286+0000","last_change":"2023-10-09T08:47:16.973286+0000","last_active":"2023-10-09T08:47:16.973286+0000","last_peered":"2023-10-09T08:47:16.973286+0000","last_clean":"2023-10-09T08:47:16.973286+0000","last_became_active":"2023-10-09T08:47:16.972839+0000","last_became_peered":"2023-10-09T08:47:16.972839+0000","last_unstale":"2023-10-09T08:47:16.973286+0000","last_undegraded":"2023-10-09T08:47:16.973286+0000","last_fullsized":"2023-10-09T08:47:16.973286+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:56.524537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":296,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956174+0000","last_change":"2023-10-09T08:45:49.183161+0000","last_active":"2023-10-09T08:47:15.956174+0000","last_peered":"2023-10-09T08:47:15.956174+0000","last_clean":"2023-10-09T08:47:15.956174+0000","last_became_active":"2023-10-09T08:45:49.182560+0000","last_became_peered":"2023-10-09T08:45:49.182560+0000","last_unstale":"2023-10-09T08:47:15.956174+0000","last_undegraded":"2023-10-09T08:47:15.956174+0000","last_fullsized":"2023-10-09T08:47:15.956174+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960367+0000","last_change":"2023-10-09T08:46:49.234616+0000","last_active":"2023-10-09T08:47:15.960367+0000","last_peered":"2023-10-09T08:47:15.960367+0000","last_clean":"2023-10-09T08:47:15.960367+0000","last_became_active":"2023-10-09T08:46:49.234217+0000","last_became_peered":"2023-10-09T08:46:49.234217+0000","last_unstale":"2023-10-09T08:47:15.960367+0000","last_undegraded":"2023-10-09T08:47:15.960367+0000","last_fullsized":"2023-10-09T08:47:15.960367+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962773+0000","last_change":"2023-10-09T08:46:18.681375+0000","last_active":"2023-10-09T08:47:16.962773+0000","last_peered":"2023-10-09T08:47:16.962773+0000","last_clean":"2023-10-09T08:47:16.962773+0000","last_became_active":"2023-10-09T08:46:18.680713+0000","last_became_peered":"2023-10-09T08:46:18.680713+0000","last_unstale":"2023-10-09T08:47:16.962773+0000","last_undegraded":"2023-10-09T08:47:16.962773+0000","last_fullsized":"2023-10-09T08:47:16.962773+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:59.671965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963096+0000","last_change":"2023-10-09T08:46:49.231894+0000","last_active":"2023-10-09T08:47:16.963096+0000","last_peered":"2023-10-09T08:47:16.963096+0000","last_clean":"2023-10-09T08:47:16.963096+0000","last_became_active":"2023-10-09T08:46:49.231531+0000","last_became_peered":"2023-10-09T08:46:49.231531+0000","last_unstale":"2023-10-09T08:47:16.963096+0000","last_undegraded":"2023-10-09T08:47:16.963096+0000","last_fullsized":"2023-10-09T08:47:16.963096+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:50.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968243+0000","last_change":"2023-10-09T08:47:16.966880+0000","last_active":"2023-10-09T08:47:16.968243+0000","last_peered":"2023-10-09T08:47:16.968243+0000","last_clean":"2023-10-09T08:47:16.968243+0000","last_became_active":"2023-10-09T08:47:16.966539+0000","last_became_peered":"2023-10-09T08:47:16.966539+0000","last_unstale":"2023-10-09T08:47:16.968243+0000","last_undegraded":"2023-10-09T08:47:16.968243+0000","last_fullsized":"2023-10-09T08:47:16.968243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:55.047737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975638+0000","last_change":"2023-10-09T08:47:16.975638+0000","last_active":"2023-10-09T08:47:16.975638+0000","last_peered":"2023-10-09T08:47:16.975638+0000","last_clean":"2023-10-09T08:47:16.975638+0000","last_became_active":"2023-10-09T08:47:16.975248+0000","last_became_peered":"2023-10-09T08:47:16.975248+0000","last_unstale":"2023-10-09T08:47:16.975638+0000","last_undegraded":"2023-10-09T08:47:16.975638+0000","last_fullsized":"2023-10-09T08:47:16.975638+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950601+0000","last_change":"2023-10-09T08:45:49.184088+0000","last_active":"2023-10-09T08:47:15.950601+0000","last_peered":"2023-10-09T08:47:15.950601+0000","last_clean":"2023-10-09T08:47:15.950601+0000","last_became_active":"2023-10-09T08:45:49.183674+0000","last_became_peered":"2023-10-09T08:45:49.183674+0000","last_unstale":"2023-10-09T08:47:15.950601+0000","last_undegraded":"2023-10-09T08:47:15.950601+0000","last_fullsized":"2023-10-09T08:47:15.950601+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962059+0000","last_change":"2023-10-09T08:46:18.680020+0000","last_active":"2023-10-09T08:47:15.962059+0000","last_peered":"2023-10-09T08:47:15.962059+0000","last_clean":"2023-10-09T08:47:15.962059+0000","last_became_active":"2023-10-09T08:46:18.679652+0000","last_became_peered":"2023-10-09T08:46:18.679652+0000","last_unstale":"2023-10-09T08:47:15.962059+0000","last_undegraded":"2023-10-09T08:47:15.962059+0000","last_fullsized":"2023-10-09T08:47:15.962059+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963009+0000","last_change":"2023-10-09T08:46:18.684501+0000","last_active":"2023-10-09T08:47:16.963009+0000","last_peered":"2023-10-09T08:47:16.963009+0000","last_clean":"2023-10-09T08:47:16.963009+0000","last_became_active":"2023-10-09T08:46:18.684159+0000","last_became_peered":"2023-10-09T08:46:18.684159+0000","last_unstale":"2023-10-09T08:47:16.963009+0000","last_undegraded":"2023-10-09T08:47:16.963009+0000","last_fullsized":"2023-10-09T08:47:16.963009+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:21.270259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.976238+0000","last_change":"2023-10-09T08:47:16.976238+0000","last_active":"2023-10-09T08:47:16.976238+0000","last_peered":"2023-10-09T08:47:16.976238+0000","last_clean":"2023-10-09T08:47:16.976238+0000","last_became_active":"2023-10-09T08:47:16.975840+0000","last_became_peered":"2023-10-09T08:47:16.975840+0000","last_unstale":"2023-10-09T08:47:16.976238+0000","last_undegraded":"2023-10-09T08:47:16.976238+0000","last_fullsized":"2023-10-09T08:47:16.976238+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962626+0000","last_change":"2023-10-09T08:46:18.680373+0000","last_active":"2023-10-09T08:47:16.962626+0000","last_peered":"2023-10-09T08:47:16.962626+0000","last_clean":"2023-10-09T08:47:16.962626+0000","last_became_active":"2023-10-09T08:46:18.679999+0000","last_became_peered":"2023-10-09T08:46:18.679999+0000","last_unstale":"2023-10-09T08:47:16.962626+0000","last_undegraded":"2023-10-09T08:47:16.962626+0000","last_fullsized":"2023-10-09T08:47:16.962626+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:32.374511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965963+0000","last_change":"2023-10-09T08:47:16.965479+0000","last_active":"2023-10-09T08:47:16.965963+0000","last_peered":"2023-10-09T08:47:16.965963+0000","last_clean":"2023-10-09T08:47:16.965963+0000","last_became_active":"2023-10-09T08:47:16.965141+0000","last_became_peered":"2023-10-09T08:47:16.965141+0000","last_unstale":"2023-10-09T08:47:16.965963+0000","last_undegraded":"2023-10-09T08:47:16.965963+0000","last_fullsized":"2023-10-09T08:47:16.965963+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:50.145649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968443+0000","last_change":"2023-10-09T08:47:16.968443+0000","last_active":"2023-10-09T08:47:16.968443+0000","last_peered":"2023-10-09T08:47:16.968443+0000","last_clean":"2023-10-09T08:47:16.968443+0000","last_became_active":"2023-10-09T08:47:16.967631+0000","last_became_peered":"2023-10-09T08:47:16.967631+0000","last_unstale":"2023-10-09T08:47:16.968443+0000","last_undegraded":"2023-10-09T08:47:16.968443+0000","last_fullsized":"2023-10-09T08:47:16.968443+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:19.412588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951852+0000","last_change":"2023-10-09T08:46:18.676843+0000","last_active":"2023-10-09T08:47:15.951852+0000","last_peered":"2023-10-09T08:47:15.951852+0000","last_clean":"2023-10-09T08:47:15.951852+0000","last_became_active":"2023-10-09T08:46:18.676468+0000","last_became_peered":"2023-10-09T08:46:18.676468+0000","last_unstale":"2023-10-09T08:47:15.951852+0000","last_undegraded":"2023-10-09T08:47:15.951852+0000","last_fullsized":"2023-10-09T08:47:15.951852+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950381+0000","last_change":"2023-10-09T08:46:49.235099+0000","last_active":"2023-10-09T08:47:15.950381+0000","last_peered":"2023-10-09T08:47:15.950381+0000","last_clean":"2023-10-09T08:47:15.950381+0000","last_became_active":"2023-10-09T08:46:49.234136+0000","last_became_peered":"2023-10-09T08:46:49.234136+0000","last_unstale":"2023-10-09T08:47:15.950381+0000","last_undegraded":"2023-10-09T08:47:15.950381+0000","last_fullsized":"2023-10-09T08:47:15.950381+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.952239+0000","last_change":"2023-10-09T08:46:49.234714+0000","last_active":"2023-10-09T08:47:15.952239+0000","last_peered":"2023-10-09T08:47:15.952239+0000","last_clean":"2023-10-09T08:47:15.952239+0000","last_became_active":"2023-10-09T08:46:49.234307+0000","last_became_peered":"2023-10-09T08:46:49.234307+0000","last_unstale":"2023-10-09T08:47:15.952239+0000","last_undegraded":"2023-10-09T08:47:15.952239+0000","last_fullsized":"2023-10-09T08:47:15.952239+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957582+0000","last_change":"2023-10-09T08:46:18.681222+0000","last_active":"2023-10-09T08:47:15.957582+0000","last_peered":"2023-10-09T08:47:15.957582+0000","last_clean":"2023-10-09T08:47:15.957582+0000","last_became_active":"2023-10-09T08:46:18.680710+0000","last_became_peered":"2023-10-09T08:46:18.680710+0000","last_unstale":"2023-10-09T08:47:15.957582+0000","last_undegraded":"2023-10-09T08:47:15.957582+0000","last_fullsized":"2023-10-09T08:47:15.957582+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972109+0000","last_change":"2023-10-09T08:47:16.972109+0000","last_active":"2023-10-09T08:47:16.972109+0000","last_peered":"2023-10-09T08:47:16.972109+0000","last_clean":"2023-10-09T08:47:16.972109+0000","last_became_active":"2023-10-09T08:47:16.971716+0000","last_became_peered":"2023-10-09T08:47:16.971716+0000","last_unstale":"2023-10-09T08:47:16.972109+0000","last_undegraded":"2023-10-09T08:47:16.972109+0000","last_fullsized":"2023-10-09T08:47:16.972109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965045+0000","last_change":"2023-10-09T08:47:16.965045+0000","last_active":"2023-10-09T08:47:16.965045+0000","last_peered":"2023-10-09T08:47:16.965045+0000","last_clean":"2023-10-09T08:47:16.965045+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.965045+0000","last_undegraded":"2023-10-09T08:47:16.965045+0000","last_fullsized":"2023-10-09T08:47:16.965045+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:49.126847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953696+0000","last_change":"2023-10-09T08:46:18.683057+0000","last_active":"2023-10-09T08:47:16.953696+0000","last_peered":"2023-10-09T08:47:16.953696+0000","last_clean":"2023-10-09T08:47:16.953696+0000","last_became_active":"2023-10-09T08:46:18.682672+0000","last_became_peered":"2023-10-09T08:46:18.682672+0000","last_unstale":"2023-10-09T08:47:16.953696+0000","last_undegraded":"2023-10-09T08:47:16.953696+0000","last_fullsized":"2023-10-09T08:47:16.953696+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:42.975534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975514+0000","last_change":"2023-10-09T08:47:16.975514+0000","last_active":"2023-10-09T08:47:16.975514+0000","last_peered":"2023-10-09T08:47:16.975514+0000","last_clean":"2023-10-09T08:47:16.975514+0000","last_became_active":"2023-10-09T08:47:16.975184+0000","last_became_peered":"2023-10-09T08:47:16.975184+0000","last_unstale":"2023-10-09T08:47:16.975514+0000","last_undegraded":"2023-10-09T08:47:16.975514+0000","last_fullsized":"2023-10-09T08:47:16.975514+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957336+0000","last_change":"2023-10-09T08:46:49.231597+0000","last_active":"2023-10-09T08:47:15.957336+0000","last_peered":"2023-10-09T08:47:15.957336+0000","last_clean":"2023-10-09T08:47:15.957336+0000","last_became_active":"2023-10-09T08:46:49.231198+0000","last_became_peered":"2023-10-09T08:46:49.231198+0000","last_unstale":"2023-10-09T08:47:15.957336+0000","last_undegraded":"2023-10-09T08:47:15.957336+0000","last_fullsized":"2023-10-09T08:47:15.957336+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961042+0000","last_change":"2023-10-09T08:34:27.733891+0000","last_active":"2023-10-09T08:47:15.961042+0000","last_peered":"2023-10-09T08:47:15.961042+0000","last_clean":"2023-10-09T08:47:15.961042+0000","last_became_active":"2023-10-09T08:34:27.733379+0000","last_became_peered":"2023-10-09T08:34:27.733379+0000","last_unstale":"2023-10-09T08:47:15.961042+0000","last_undegraded":"2023-10-09T08:47:15.961042+0000","last_fullsized":"2023-10-09T08:47:15.961042+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972571+0000","last_change":"2023-10-09T08:47:16.972571+0000","last_active":"2023-10-09T08:47:16.972571+0000","last_peered":"2023-10-09T08:47:16.972571+0000","last_clean":"2023-10-09T08:47:16.972571+0000","last_became_active":"2023-10-09T08:47:16.972035+0000","last_became_peered":"2023-10-09T08:47:16.972035+0000","last_unstale":"2023-10-09T08:47:16.972571+0000","last_undegraded":"2023-10-09T08:47:16.972571+0000","last_fullsized":"2023-10-09T08:47:16.972571+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957950+0000","last_change":"2023-10-09T08:46:18.674774+0000","last_active":"2023-10-09T08:47:15.957950+0000","last_peered":"2023-10-09T08:47:15.957950+0000","last_clean":"2023-10-09T08:47:15.957950+0000","last_became_active":"2023-10-09T08:46:18.674334+0000","last_became_peered":"2023-10-09T08:46:18.674334+0000","last_unstale":"2023-10-09T08:47:15.957950+0000","last_undegraded":"2023-10-09T08:47:15.957950+0000","last_fullsized":"2023-10-09T08:47:15.957950+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960548+0000","last_change":"2023-10-09T08:34:29.490932+0000","last_active":"2023-10-09T08:47:15.960548+0000","last_peered":"2023-10-09T08:47:15.960548+0000","last_clean":"2023-10-09T08:47:15.960548+0000","last_became_active":"2023-10-09T08:34:29.489841+0000","last_became_peered":"2023-10-09T08:34:29.489841+0000","last_unstale":"2023-10-09T08:47:15.960548+0000","last_undegraded":"2023-10-09T08:47:15.960548+0000","last_fullsized":"2023-10-09T08:47:15.960548+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958369+0000","last_change":"2023-10-09T08:45:49.182558+0000","last_active":"2023-10-09T08:47:15.958369+0000","last_peered":"2023-10-09T08:47:15.958369+0000","last_clean":"2023-10-09T08:47:15.958369+0000","last_became_active":"2023-10-09T08:45:49.182145+0000","last_became_peered":"2023-10-09T08:45:49.182145+0000","last_unstale":"2023-10-09T08:47:15.958369+0000","last_undegraded":"2023-10-09T08:47:15.958369+0000","last_fullsized":"2023-10-09T08:47:15.958369+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974455+0000","last_change":"2023-10-09T08:47:16.974455+0000","last_active":"2023-10-09T08:47:16.974455+0000","last_peered":"2023-10-09T08:47:16.974455+0000","last_clean":"2023-10-09T08:47:16.974455+0000","last_became_active":"2023-10-09T08:47:16.973803+0000","last_became_peered":"2023-10-09T08:47:16.973803+0000","last_unstale":"2023-10-09T08:47:16.974455+0000","last_undegraded":"2023-10-09T08:47:16.974455+0000","last_fullsized":"2023-10-09T08:47:16.974455+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:02.753047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954654+0000","last_change":"2023-10-09T08:45:49.188401+0000","last_active":"2023-10-09T08:47:15.954654+0000","last_peered":"2023-10-09T08:47:15.954654+0000","last_clean":"2023-10-09T08:47:15.954654+0000","last_became_active":"2023-10-09T08:45:49.188014+0000","last_became_peered":"2023-10-09T08:45:49.188014+0000","last_unstale":"2023-10-09T08:47:15.954654+0000","last_undegraded":"2023-10-09T08:47:15.954654+0000","last_fullsized":"2023-10-09T08:47:15.954654+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972320+0000","last_change":"2023-10-09T08:47:16.971847+0000","last_active":"2023-10-09T08:47:16.972320+0000","last_peered":"2023-10-09T08:47:16.972320+0000","last_clean":"2023-10-09T08:47:16.972320+0000","last_became_active":"2023-10-09T08:47:16.970986+0000","last_became_peered":"2023-10-09T08:47:16.970986+0000","last_unstale":"2023-10-09T08:47:16.972320+0000","last_undegraded":"2023-10-09T08:47:16.972320+0000","last_fullsized":"2023-10-09T08:47:16.972320+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:06.084956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954882+0000","last_change":"2023-10-09T08:34:27.726666+0000","last_active":"2023-10-09T08:47:15.954882+0000","last_peered":"2023-10-09T08:47:15.954882+0000","last_clean":"2023-10-09T08:47:15.954882+0000","last_became_active":"2023-10-09T08:34:27.726189+0000","last_became_peered":"2023-10-09T08:34:27.726189+0000","last_unstale":"2023-10-09T08:47:15.954882+0000","last_undegraded":"2023-10-09T08:47:15.954882+0000","last_fullsized":"2023-10-09T08:47:15.954882+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975645+0000","last_change":"2023-10-09T08:47:16.975645+0000","last_active":"2023-10-09T08:47:16.975645+0000","last_peered":"2023-10-09T08:47:16.975645+0000","last_clean":"2023-10-09T08:47:16.975645+0000","last_became_active":"2023-10-09T08:47:16.975277+0000","last_became_peered":"2023-10-09T08:47:16.975277+0000","last_unstale":"2023-10-09T08:47:16.975645+0000","last_undegraded":"2023-10-09T08:47:16.975645+0000","last_fullsized":"2023-10-09T08:47:16.975645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.971694+0000","last_change":"2023-10-09T08:47:16.971357+0000","last_active":"2023-10-09T08:47:16.971694+0000","last_peered":"2023-10-09T08:47:16.971694+0000","last_clean":"2023-10-09T08:47:16.971694+0000","last_became_active":"2023-10-09T08:47:16.971034+0000","last_became_peered":"2023-10-09T08:47:16.971034+0000","last_unstale":"2023-10-09T08:47:16.971694+0000","last_undegraded":"2023-10-09T08:47:16.971694+0000","last_fullsized":"2023-10-09T08:47:16.971694+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:05.892369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974125+0000","last_change":"2023-10-09T08:47:16.974125+0000","last_active":"2023-10-09T08:47:16.974125+0000","last_peered":"2023-10-09T08:47:16.974125+0000","last_clean":"2023-10-09T08:47:16.974125+0000","last_became_active":"2023-10-09T08:47:16.973763+0000","last_became_peered":"2023-10-09T08:47:16.973763+0000","last_unstale":"2023-10-09T08:47:16.974125+0000","last_undegraded":"2023-10-09T08:47:16.974125+0000","last_fullsized":"2023-10-09T08:47:16.974125+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:31.548547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958645+0000","last_change":"2023-10-09T08:46:49.233182+0000","last_active":"2023-10-09T08:47:15.958645+0000","last_peered":"2023-10-09T08:47:15.958645+0000","last_clean":"2023-10-09T08:47:15.958645+0000","last_became_active":"2023-10-09T08:46:49.232746+0000","last_became_peered":"2023-10-09T08:46:49.232746+0000","last_unstale":"2023-10-09T08:47:15.958645+0000","last_undegraded":"2023-10-09T08:47:15.958645+0000","last_fullsized":"2023-10-09T08:47:15.958645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951634+0000","last_change":"2023-10-09T08:46:18.676902+0000","last_active":"2023-10-09T08:47:15.951634+0000","last_peered":"2023-10-09T08:47:15.951634+0000","last_clean":"2023-10-09T08:47:15.951634+0000","last_became_active":"2023-10-09T08:46:18.676560+0000","last_became_peered":"2023-10-09T08:46:18.676560+0000","last_unstale":"2023-10-09T08:47:15.951634+0000","last_undegraded":"2023-10-09T08:47:15.951634+0000","last_fullsized":"2023-10-09T08:47:15.951634+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":68,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951207+0000","last_change":"2023-10-09T08:34:27.730961+0000","last_active":"2023-10-09T08:47:15.951207+0000","last_peered":"2023-10-09T08:47:15.951207+0000","last_clean":"2023-10-09T08:47:15.951207+0000","last_became_active":"2023-10-09T08:34:27.730198+0000","last_became_peered":"2023-10-09T08:34:27.730198+0000","last_unstale":"2023-10-09T08:47:15.951207+0000","last_undegraded":"2023-10-09T08:47:15.951207+0000","last_fullsized":"2023-10-09T08:47:15.951207+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963049+0000","last_change":"2023-10-09T08:46:49.231880+0000","last_active":"2023-10-09T08:47:16.963049+0000","last_peered":"2023-10-09T08:47:16.963049+0000","last_clean":"2023-10-09T08:47:16.963049+0000","last_became_active":"2023-10-09T08:46:49.231471+0000","last_became_peered":"2023-10-09T08:46:49.231471+0000","last_unstale":"2023-10-09T08:47:16.963049+0000","last_undegraded":"2023-10-09T08:47:16.963049+0000","last_fullsized":"2023-10-09T08:47:16.963049+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:31.742745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973891+0000","last_change":"2023-10-09T08:47:16.973891+0000","last_active":"2023-10-09T08:47:16.973891+0000","last_peered":"2023-10-09T08:47:16.973891+0000","last_clean":"2023-10-09T08:47:16.973891+0000","last_became_active":"2023-10-09T08:47:16.973405+0000","last_became_peered":"2023-10-09T08:47:16.973405+0000","last_unstale":"2023-10-09T08:47:16.973891+0000","last_undegraded":"2023-10-09T08:47:16.973891+0000","last_fullsized":"2023-10-09T08:47:16.973891+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972498+0000","last_change":"2023-10-09T08:47:16.972498+0000","last_active":"2023-10-09T08:47:16.972498+0000","last_peered":"2023-10-09T08:47:16.972498+0000","last_clean":"2023-10-09T08:47:16.972498+0000","last_became_active":"2023-10-09T08:47:16.972184+0000","last_became_peered":"2023-10-09T08:47:16.972184+0000","last_unstale":"2023-10-09T08:47:16.972498+0000","last_undegraded":"2023-10-09T08:47:16.972498+0000","last_fullsized":"2023-10-09T08:47:16.972498+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:03.559858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960484+0000","last_change":"2023-10-09T08:46:18.675538+0000","last_active":"2023-10-09T08:47:15.960484+0000","last_peered":"2023-10-09T08:47:15.960484+0000","last_clean":"2023-10-09T08:47:15.960484+0000","last_became_active":"2023-10-09T08:46:18.675149+0000","last_became_peered":"2023-10-09T08:46:18.675149+0000","last_unstale":"2023-10-09T08:47:15.960484+0000","last_undegraded":"2023-10-09T08:47:15.960484+0000","last_fullsized":"2023-10-09T08:47:15.960484+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969948+0000","last_change":"2023-10-09T08:47:16.969948+0000","last_active":"2023-10-09T08:47:16.969948+0000","last_peered":"2023-10-09T08:47:16.969948+0000","last_clean":"2023-10-09T08:47:16.969948+0000","last_became_active":"2023-10-09T08:47:16.969666+0000","last_became_peered":"2023-10-09T08:47:16.969666+0000","last_unstale":"2023-10-09T08:47:16.969948+0000","last_undegraded":"2023-10-09T08:47:16.969948+0000","last_fullsized":"2023-10-09T08:47:16.969948+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:25.089633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972317+0000","last_change":"2023-10-09T08:47:16.971965+0000","last_active":"2023-10-09T08:47:16.972317+0000","last_peered":"2023-10-09T08:47:16.972317+0000","last_clean":"2023-10-09T08:47:16.972317+0000","last_became_active":"2023-10-09T08:47:16.971651+0000","last_became_peered":"2023-10-09T08:47:16.971651+0000","last_unstale":"2023-10-09T08:47:16.972317+0000","last_undegraded":"2023-10-09T08:47:16.972317+0000","last_fullsized":"2023-10-09T08:47:16.972317+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:04.259603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956192+0000","last_change":"2023-10-09T08:46:18.680365+0000","last_active":"2023-10-09T08:47:15.956192+0000","last_peered":"2023-10-09T08:47:15.956192+0000","last_clean":"2023-10-09T08:47:15.956192+0000","last_became_active":"2023-10-09T08:46:18.679875+0000","last_became_peered":"2023-10-09T08:46:18.679875+0000","last_unstale":"2023-10-09T08:47:15.956192+0000","last_undegraded":"2023-10-09T08:47:15.956192+0000","last_fullsized":"2023-10-09T08:47:15.956192+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965460+0000","last_change":"2023-10-09T08:47:16.965460+0000","last_active":"2023-10-09T08:47:16.965460+0000","last_peered":"2023-10-09T08:47:16.965460+0000","last_clean":"2023-10-09T08:47:16.965460+0000","last_became_active":"2023-10-09T08:47:16.965040+0000","last_became_peered":"2023-10-09T08:47:16.965040+0000","last_unstale":"2023-10-09T08:47:16.965460+0000","last_undegraded":"2023-10-09T08:47:16.965460+0000","last_fullsized":"2023-10-09T08:47:16.965460+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.025735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972434+0000","last_change":"2023-10-09T08:47:16.972106+0000","last_active":"2023-10-09T08:47:16.972434+0000","last_peered":"2023-10-09T08:47:16.972434+0000","last_clean":"2023-10-09T08:47:16.972434+0000","last_became_active":"2023-10-09T08:47:16.971511+0000","last_became_peered":"2023-10-09T08:47:16.971511+0000","last_unstale":"2023-10-09T08:47:16.972434+0000","last_undegraded":"2023-10-09T08:47:16.972434+0000","last_fullsized":"2023-10-09T08:47:16.972434+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:12.674597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969696+0000","last_change":"2023-10-09T08:47:16.968800+0000","last_active":"2023-10-09T08:47:16.969696+0000","last_peered":"2023-10-09T08:47:16.969696+0000","last_clean":"2023-10-09T08:47:16.969696+0000","last_became_active":"2023-10-09T08:47:16.968430+0000","last_became_peered":"2023-10-09T08:47:16.968430+0000","last_unstale":"2023-10-09T08:47:16.969696+0000","last_undegraded":"2023-10-09T08:47:16.969696+0000","last_fullsized":"2023-10-09T08:47:16.969696+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:08.011498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965051+0000","last_change":"2023-10-09T08:47:16.965051+0000","last_active":"2023-10-09T08:47:16.965051+0000","last_peered":"2023-10-09T08:47:16.965051+0000","last_clean":"2023-10-09T08:47:16.965051+0000","last_became_active":"2023-10-09T08:47:16.964664+0000","last_became_peered":"2023-10-09T08:47:16.964664+0000","last_unstale":"2023-10-09T08:47:16.965051+0000","last_undegraded":"2023-10-09T08:47:16.965051+0000","last_fullsized":"2023-10-09T08:47:16.965051+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.377202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950228+0000","last_change":"2023-10-09T08:46:49.233483+0000","last_active":"2023-10-09T08:47:15.950228+0000","last_peered":"2023-10-09T08:47:15.950228+0000","last_clean":"2023-10-09T08:47:15.950228+0000","last_became_active":"2023-10-09T08:46:49.233033+0000","last_became_peered":"2023-10-09T08:46:49.233033+0000","last_unstale":"2023-10-09T08:47:15.950228+0000","last_undegraded":"2023-10-09T08:47:15.950228+0000","last_fullsized":"2023-10-09T08:47:15.950228+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430421,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310904,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434248,"statfs":{"total":95995035648,"available":95676669952,"internally_reserved":0,"allocated":1556480,"data_stored":1102308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.92900000000000005,"15min":0.92200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.2569999999999999,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.008},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.94199999999999995,"15min":0.94499999999999995},"min":{"1min":0.64300000000000002,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1679999999999999,"5min":1.262,"15min":1.262},"last":1.0669999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:48:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.99299999999999999,"15min":0.98899999999999999},"min":{"1min":0.72999999999999998,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.24,"5min":1.446,"15min":1.446},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.004,"5min":0.97899999999999998,"15min":0.97499999999999998},"min":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.181,"5min":1.319,"15min":1.319},"last":0.97099999999999997}]},{"osd":2,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.91400000000000003,"15min":0.92400000000000004},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.125,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":1.034},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.92700000000000005,"15min":0.93000000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.78200000000000003}]},{"osd":3,"last update":"Mon Oct 9 08:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":1.173}]},{"osd":4,"last update":"Mon Oct 9 08:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.88300000000000001,"15min":0.88400000000000001},"min":{"1min":0.69199999999999995,"5min":0.45500000000000002,"15min":0.40300000000000002},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.4630000000000001},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.96499999999999997,"15min":0.93799999999999994},"min":{"1min":0.69499999999999995,"5min":0.54800000000000004,"15min":0.40000000000000002},"max":{"1min":1.262,"5min":1.488,"15min":1.488},"last":1.0800000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97799999999999998,"15min":0.98899999999999999},"min":{"1min":0.69099999999999995,"5min":0.58799999999999997,"15min":0.501},"max":{"1min":1.131,"5min":1.329,"15min":1.6399999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.028,"5min":1.0129999999999999,"15min":0.999},"min":{"1min":0.67300000000000004,"5min":0.54900000000000004,"15min":0.46100000000000002},"max":{"1min":1.3400000000000001,"5min":1.3400000000000001,"15min":1.794},"last":0.68200000000000005}]},{"osd":6,"last update":"Mon Oct 9 08:48:09 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.04,"15min":1.032},"min":{"1min":0.76900000000000002,"5min":0.57499999999999996,"15min":0.44},"max":{"1min":1.409,"5min":1.4350000000000001,"15min":1.7170000000000001},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.0920000000000001,"15min":1.0649999999999999},"min":{"1min":0.79100000000000004,"5min":0.63,"15min":0.45000000000000001},"max":{"1min":1.3240000000000001,"5min":1.528,"15min":1.528},"last":1.218}]}]},{"osd":6,"up_from":42,"seq":180388626654,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1508,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1544192,"data_stored":1102112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92600000000000005,"15min":0.92700000000000005},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.1739999999999999,"5min":1.349,"15min":1.349},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.996,"15min":0.98999999999999999},"min":{"1min":0.78200000000000003,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.248,"5min":1.5,"15min":1.5},"last":1.111}]},{"osd":1,"last update":"Mon Oct 9 08:48:23 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.89100000000000001,"15min":0.89500000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":1.151},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.58099999999999996,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.2310000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:48:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.91400000000000003,"15min":0.91600000000000004},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.89500000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.94899999999999995,"15min":0.93999999999999995},"min":{"1min":0.76500000000000001,"5min":0.625,"15min":0.625},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.1859999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:48:16 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":0.94699999999999995}]},{"osd":4,"last update":"Mon Oct 9 08:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.873,"15min":0.89200000000000002},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.379},"max":{"1min":1.159,"5min":1.286,"15min":1.629},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.94299999999999995},"min":{"1min":0.70999999999999996,"5min":0.436,"15min":0.436},"max":{"1min":1.2549999999999999,"5min":1.373,"15min":1.671},"last":1.04}]},{"osd":5,"last update":"Mon Oct 9 08:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.96999999999999997,"15min":0.97499999999999998},"min":{"1min":0.80100000000000005,"5min":0.495,"15min":0.495},"max":{"1min":1.2190000000000001,"5min":1.3720000000000001,"15min":1.7410000000000001},"last":1.3109999999999999},{"interface":"front","average":{"1min":1.036,"5min":1.03,"15min":1.038},"min":{"1min":0.76600000000000001,"5min":0.66700000000000004,"15min":0.51600000000000001},"max":{"1min":1.3420000000000001,"5min":1.46,"15min":4.4420000000000002},"last":1.1319999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.034,"15min":1.046},"min":{"1min":0.69999999999999996,"5min":0.65800000000000003,"15min":0.45900000000000002},"max":{"1min":1.3,"5min":1.383,"15min":1.8220000000000001},"last":1.272},{"interface":"front","average":{"1min":1.052,"5min":1.0740000000000001,"15min":1.0760000000000001},"min":{"1min":0.75700000000000001,"5min":0.53300000000000003,"15min":0.52500000000000002},"max":{"1min":1.2529999999999999,"5min":1.556,"15min":2.262},"last":1.177}]}]},{"osd":1,"up_from":83,"seq":356482285602,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32396,"kb_used_data":964,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712756,"statfs":{"total":95995035648,"available":95961862144,"internally_reserved":0,"allocated":987136,"data_stored":516401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.626,"15min":0.625},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83399999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.68000000000000005,"15min":0.67400000000000004},"min":{"1min":0.48499999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.76100000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:47:52 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.747,"15min":0.747},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.70599999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.1699999999999999,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.996},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.90200000000000002}]},{"osd":4,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.96699999999999997,"15min":0.96499999999999997},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.4079999999999999,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":1.228},{"interface":"front","average":{"1min":1.006,"5min":0.97599999999999998,"15min":0.97099999999999997},"min":{"1min":0.72799999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.278,"5min":1.577,"15min":1.577},"last":0.83699999999999997}]},{"osd":5,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"min":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.2869999999999999,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.181},{"interface":"front","average":{"1min":1.024,"5min":1.0609999999999999,"15min":1.0669999999999999},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.298,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":1.349}]},{"osd":6,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.093,"15min":1.0860000000000001},"min":{"1min":0.68700000000000006,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.391,"5min":1.522,"15min":1.522},"last":0.94199999999999995},{"interface":"front","average":{"1min":1.127,"5min":1.1120000000000001,"15min":1.1100000000000001},"min":{"1min":0.81799999999999995,"5min":0.754,"15min":0.754},"max":{"1min":1.3560000000000001,"5min":1.577,"15min":1.577},"last":1.1200000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.046,"15min":1.0389999999999999},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.365,"5min":1.462,"15min":1.462},"last":1.294},{"interface":"front","average":{"1min":1.119,"5min":1.109,"15min":1.107},"min":{"1min":0.69999999999999996,"5min":0.624,"15min":0.624},"max":{"1min":1.427,"5min":1.625,"15min":1.625},"last":1.0589999999999999}]}]},{"osd":0,"up_from":78,"seq":335007449128,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33060,"kb_used_data":1564,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712092,"statfs":{"total":95995035648,"available":95961182208,"internally_reserved":0,"allocated":1601536,"data_stored":1109136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.71999999999999997,"15min":0.71899999999999997},"min":{"1min":0.45300000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.77500000000000002,"15min":0.78300000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.01,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.65300000000000002}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.83199999999999996,"15min":0.84799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.083,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.92900000000000005,"15min":0.95099999999999996},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":1.359}]},{"osd":3,"last update":"Mon Oct 9 08:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":1.085}]},{"osd":4,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0389999999999999,"15min":1.046},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":1.415},{"interface":"front","average":{"1min":0.873,"5min":0.93700000000000006,"15min":0.96199999999999997},"min":{"1min":0.60799999999999998,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.0469999999999999,"5min":1.528,"15min":1.528},"last":0.86899999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0189999999999999,"15min":1.018},"min":{"1min":0.63500000000000001,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.361,"5min":1.51,"15min":1.51},"last":1.22},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0980000000000001,"15min":1.115},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.1859999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.1359999999999999,"15min":1.1599999999999999},"min":{"1min":0.77200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3959999999999999,"5min":1.663,"15min":1.663},"last":1.127},{"interface":"front","average":{"1min":1.071,"5min":1.1950000000000001,"15min":1.2150000000000001},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.486,"5min":1.609,"15min":1.609},"last":0.96699999999999997}]},{"osd":7,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.214,"15min":1.2330000000000001},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.5880000000000001,"5min":1.679,"15min":1.679},"last":1.29},{"interface":"front","average":{"1min":1.028,"5min":1.117,"15min":1.1479999999999999},"min":{"1min":0.64600000000000002,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.3779999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.26}]}]},{"osd":2,"up_from":88,"seq":377957122076,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36616,"kb_used_data":960,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708536,"statfs":{"total":95995035648,"available":95957540864,"internally_reserved":0,"allocated":983040,"data_stored":514873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.70999999999999996,"15min":0.70199999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.79300000000000004,"15min":0.81000000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2629999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.69999999999999996}]},{"osd":1,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.0009999999999999,"5min":1.127,"15min":1.127},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.755,"5min":0.82899999999999996,"15min":0.84099999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.165,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.91300000000000003}]},{"osd":3,"last update":"Mon Oct 9 08:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.96999999999999997}]},{"osd":4,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.96599999999999997,"15min":0.97199999999999998},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.1919999999999999,"5min":1.24,"15min":1.24},"last":1.127},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0469999999999999,"15min":1.0580000000000001},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.085}]},{"osd":5,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.131,"15min":1.1579999999999999},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.464,"5min":1.49,"15min":1.49},"last":1.25},{"interface":"front","average":{"1min":1,"5min":1.0649999999999999,"15min":1.0760000000000001},"min":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"max":{"1min":1.2230000000000001,"5min":1.337,"15min":1.337},"last":1.2230000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.1259999999999999,"15min":1.137},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.3100000000000001,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.3009999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.1870000000000001,"15min":1.204},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.446,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.446}]},{"osd":7,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.1539999999999999,"15min":1.1599999999999999},"min":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"max":{"1min":1.4099999999999999,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":1.3919999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.214,"15min":1.2350000000000001},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.5169999999999999,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.3620000000000001}]}]},{"osd":3,"up_from":93,"seq":399431958550,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28276,"kb_used_data":940,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716876,"statfs":{"total":95995035648,"available":95966081024,"internally_reserved":0,"allocated":962560,"data_stored":511788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":1.0820000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":1.0569999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.873},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.91100000000000003}]},{"osd":4,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":1.119},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.98799999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.034,"15min":1.034},"min":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"min":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.141}]},{"osd":6,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"min":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"max":{"1min":1.369,"5min":1.369,"15min":1.369},"last":1.181},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"min":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"max":{"1min":1.482,"5min":1.482,"15min":1.482},"last":1.288}]},{"osd":7,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.163,"15min":1.163},"min":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"max":{"1min":1.52,"5min":1.52,"15min":1.52},"last":1.234},{"interface":"front","average":{"1min":1.167,"5min":1.167,"15min":1.167},"min":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"max":{"1min":1.581,"5min":1.581,"15min":1.581},"last":1.2070000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019120,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302132,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443020,"statfs":{"total":95995035648,"available":95685652480,"internally_reserved":0,"allocated":962560,"data_stored":512135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:47:56 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":0.86899999999999999,"15min":0.875},"min":{"1min":0.60699999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.228,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.90000000000000002,"15min":0.89600000000000002},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1779999999999999,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.64800000000000002}]},{"osd":1,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.85299999999999998,"15min":0.85099999999999998},"min":{"1min":0.64900000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.0349999999999999,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.96399999999999997,"15min":0.95999999999999996},"min":{"1min":0.73999999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":1.115}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93799999999999994,"15min":0.93600000000000005},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.03,"5min":0.96599999999999997,"15min":0.95599999999999996},"min":{"1min":0.80200000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.0620000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"min":{"1min":0.75,"5min":0.75,"15min":0.75},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.286}]},{"osd":5,"last update":"Mon Oct 9 08:48:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.89500000000000002,"15min":0.91400000000000003},"min":{"1min":0.73799999999999999,"5min":0.51200000000000001,"15min":0.39200000000000002},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.216},{"interface":"front","average":{"1min":1.111,"5min":0.99199999999999999,"15min":0.99399999999999999},"min":{"1min":0.96999999999999997,"5min":0.48399999999999999,"15min":0.47399999999999998},"max":{"1min":1.389,"5min":1.389,"15min":1.4730000000000001},"last":1.2030000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:48:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.95699999999999996,"15min":0.94399999999999995},"min":{"1min":0.71799999999999997,"5min":0.54300000000000004,"15min":0.42899999999999999},"max":{"1min":1.2909999999999999,"5min":1.4059999999999999,"15min":1.4179999999999999},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.089,"5min":1.0469999999999999,"15min":1.044},"min":{"1min":0.79600000000000004,"5min":0.57599999999999996,"15min":0.48399999999999999},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":1.3120000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:47:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.0529999999999999,"15min":1.069},"min":{"1min":0.84999999999999998,"5min":0.54400000000000004,"15min":0.44800000000000001},"max":{"1min":1.272,"5min":1.48,"15min":1.48},"last":1.3600000000000001},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.0840000000000001,"15min":1.071},"min":{"1min":0.87,"5min":0.64000000000000001,"15min":0.38},"max":{"1min":1.351,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.331}]}]},{"osd":5,"up_from":36,"seq":154618822888,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310308,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434844,"statfs":{"total":95995035648,"available":95677280256,"internally_reserved":0,"allocated":946176,"data_stored":507879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.876,"15min":0.876},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.1419999999999999,"5min":1.262,"15min":1.262},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.91100000000000003,"15min":0.90600000000000003},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.133,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.032}]},{"osd":1,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.91600000000000004,"15min":0.91300000000000003},"min":{"1min":0.72699999999999998,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"min":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":1.113}]},{"osd":2,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.92000000000000004,"15min":0.92700000000000005},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1639999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.98999999999999999,"15min":0.998},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.284,"5min":1.323,"15min":1.323},"last":1.284}]},{"osd":3,"last update":"Mon Oct 9 08:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.006,"15min":1.006},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"max":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.93999999999999995}]},{"osd":4,"last update":"Mon Oct 9 08:48:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.878,"15min":0.85999999999999999},"min":{"1min":0.68600000000000005,"5min":0.61299999999999999,"15min":0.46999999999999997},"max":{"1min":1.1890000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.01,"5min":0.95299999999999996,"15min":0.95099999999999996},"min":{"1min":0.60999999999999999,"5min":0.505,"15min":0.442},"max":{"1min":1.212,"5min":1.405,"15min":1.405},"last":1.2290000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.94999999999999996,"15min":0.96699999999999997},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.45200000000000001},"max":{"1min":1.2370000000000001,"5min":1.333,"15min":1.411},"last":0.84099999999999997},{"interface":"front","average":{"1min":1.083,"5min":1.02,"15min":1.01},"min":{"1min":0.69599999999999995,"5min":0.57899999999999996,"15min":0.40400000000000003},"max":{"1min":1.347,"5min":1.4750000000000001,"15min":3.2919999999999998},"last":1.1459999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:48:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.0640000000000001,"15min":1.042},"min":{"1min":0.78100000000000003,"5min":0.53300000000000003,"15min":0.36899999999999999},"max":{"1min":1.2849999999999999,"5min":1.4119999999999999,"15min":1.4910000000000001},"last":1.3260000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.071,"15min":1.0820000000000001},"min":{"1min":0.85399999999999998,"5min":0.59999999999999998,"15min":0.45000000000000001},"max":{"1min":1.3140000000000001,"5min":1.4950000000000001,"15min":1.782},"last":1.1819999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:49:01.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:49:01.637 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:01 smithi016 bash[75179]: audit 2023-10-09T08:49:00.812254+0000 mgr.y (mgr.25421) 332 : audit 0 from='client.44591 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:01.637 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:01 smithi016 bash[75179]: cluster 2023-10-09T08:49:01.280939+0000 mgr.y (mgr.25421) 333 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:01.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:01 smithi016 bash[77892]: audit 2023-10-09T08:49:00.812254+0000 mgr.y (mgr.25421) 332 : audit 0 from='client.44591 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:01.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:01 smithi016 bash[77892]: cluster 2023-10-09T08:49:01.280939+0000 mgr.y (mgr.25421) 333 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:01.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:01 smithi094 bash[59677]: audit 2023-10-09T08:49:00.812254+0000 mgr.y (mgr.25421) 332 : audit 0 from='client.44591 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:01.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:01 smithi094 bash[59677]: cluster 2023-10-09T08:49:01.280939+0000 mgr.y (mgr.25421) 333 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:03.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:03 smithi094 bash[59677]: cluster 2023-10-09T08:49:03.281774+0000 mgr.y (mgr.25421) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:03.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:03 smithi016 bash[75179]: cluster 2023-10-09T08:49:03.281774+0000 mgr.y (mgr.25421) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:03.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:03 smithi016 bash[77892]: cluster 2023-10-09T08:49:03.281774+0000 mgr.y (mgr.25421) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:05.246 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:04 smithi094 bash[59677]: audit 2023-10-09T08:49:04.204247+0000 mon.a (mon.0) 479 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:05.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:04 smithi016 bash[75179]: audit 2023-10-09T08:49:04.204247+0000 mon.a (mon.0) 479 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:05.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:04 smithi016 bash[77892]: audit 2023-10-09T08:49:04.204247+0000 mon.a (mon.0) 479 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:06.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:05 smithi016 bash[75179]: cluster 2023-10-09T08:49:05.282856+0000 mgr.y (mgr.25421) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:06.002 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:49:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:49:05] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.33.4" 2023-10-09T08:49:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:05 smithi016 bash[77892]: cluster 2023-10-09T08:49:05.282856+0000 mgr.y (mgr.25421) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:06.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:05 smithi094 bash[59677]: cluster 2023-10-09T08:49:05.282856+0000 mgr.y (mgr.25421) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:06.414 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:49:07.653 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:07 smithi016 bash[75179]: cluster 2023-10-09T08:49:07.283684+0000 mgr.y (mgr.25421) 336 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:07.653 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:07 smithi016 bash[77892]: cluster 2023-10-09T08:49:07.283684+0000 mgr.y (mgr.25421) 336 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:07.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:07 smithi094 bash[59677]: cluster 2023-10-09T08:49:07.283684+0000 mgr.y (mgr.25421) 336 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:09.808 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:49:09.810 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:49:10.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:09 smithi094 bash[59677]: cluster 2023-10-09T08:49:09.284388+0000 mgr.y (mgr.25421) 337 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:10.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:09 smithi016 bash[75179]: cluster 2023-10-09T08:49:09.284388+0000 mgr.y (mgr.25421) 337 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:10.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:09 smithi016 bash[77892]: cluster 2023-10-09T08:49:09.284388+0000 mgr.y (mgr.25421) 337 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:10.682 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":273,"stamp":"2023-10-09T08:49:09.284044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1352296,"kb_used_data":9320,"kb_used_omap":126,"kb_used_meta":1342785,"kb_avail":748608920,"statfs":{"total":767960285184,"available":766575534080,"internally_reserved":0,"allocated":9543680,"data_stored":5876632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1375012475},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005197"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950127+0000","last_change":"2023-10-09T08:46:18.676602+0000","last_active":"2023-10-09T08:47:15.950127+0000","last_peered":"2023-10-09T08:47:15.950127+0000","last_clean":"2023-10-09T08:47:15.950127+0000","last_became_active":"2023-10-09T08:46:18.676122+0000","last_became_peered":"2023-10-09T08:46:18.676122+0000","last_unstale":"2023-10-09T08:47:15.950127+0000","last_undegraded":"2023-10-09T08:47:15.950127+0000","last_fullsized":"2023-10-09T08:47:15.950127+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953679+0000","last_change":"2023-10-09T08:46:49.233211+0000","last_active":"2023-10-09T08:47:16.953679+0000","last_peered":"2023-10-09T08:47:16.953679+0000","last_clean":"2023-10-09T08:47:16.953679+0000","last_became_active":"2023-10-09T08:46:49.232775+0000","last_became_peered":"2023-10-09T08:46:49.232775+0000","last_unstale":"2023-10-09T08:47:16.953679+0000","last_undegraded":"2023-10-09T08:47:16.953679+0000","last_fullsized":"2023-10-09T08:47:16.953679+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:59.423857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.966115+0000","last_change":"2023-10-09T08:47:16.966115+0000","last_active":"2023-10-09T08:47:16.966115+0000","last_peered":"2023-10-09T08:47:16.966115+0000","last_clean":"2023-10-09T08:47:16.966115+0000","last_became_active":"2023-10-09T08:47:16.965795+0000","last_became_peered":"2023-10-09T08:47:16.965795+0000","last_unstale":"2023-10-09T08:47:16.966115+0000","last_undegraded":"2023-10-09T08:47:16.966115+0000","last_fullsized":"2023-10-09T08:47:16.966115+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:40.817840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.970132+0000","last_change":"2023-10-09T08:47:16.970132+0000","last_active":"2023-10-09T08:47:16.970132+0000","last_peered":"2023-10-09T08:47:16.970132+0000","last_clean":"2023-10-09T08:47:16.970132+0000","last_became_active":"2023-10-09T08:47:16.969760+0000","last_became_peered":"2023-10-09T08:47:16.969760+0000","last_unstale":"2023-10-09T08:47:16.970132+0000","last_undegraded":"2023-10-09T08:47:16.970132+0000","last_fullsized":"2023-10-09T08:47:16.970132+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.801830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969367+0000","last_change":"2023-10-09T08:47:16.969367+0000","last_active":"2023-10-09T08:47:16.969367+0000","last_peered":"2023-10-09T08:47:16.969367+0000","last_clean":"2023-10-09T08:47:16.969367+0000","last_became_active":"2023-10-09T08:47:16.969000+0000","last_became_peered":"2023-10-09T08:47:16.969000+0000","last_unstale":"2023-10-09T08:47:16.969367+0000","last_undegraded":"2023-10-09T08:47:16.969367+0000","last_fullsized":"2023-10-09T08:47:16.969367+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961355+0000","last_change":"2023-10-09T08:34:27.739113+0000","last_active":"2023-10-09T08:47:15.961355+0000","last_peered":"2023-10-09T08:47:15.961355+0000","last_clean":"2023-10-09T08:47:15.961355+0000","last_became_active":"2023-10-09T08:34:27.738645+0000","last_became_peered":"2023-10-09T08:34:27.738645+0000","last_unstale":"2023-10-09T08:47:15.961355+0000","last_undegraded":"2023-10-09T08:47:15.961355+0000","last_fullsized":"2023-10-09T08:47:15.961355+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968742+0000","last_change":"2023-10-09T08:47:16.968742+0000","last_active":"2023-10-09T08:47:16.968742+0000","last_peered":"2023-10-09T08:47:16.968742+0000","last_clean":"2023-10-09T08:47:16.968742+0000","last_became_active":"2023-10-09T08:47:16.968317+0000","last_became_peered":"2023-10-09T08:47:16.968317+0000","last_unstale":"2023-10-09T08:47:16.968742+0000","last_undegraded":"2023-10-09T08:47:16.968742+0000","last_fullsized":"2023-10-09T08:47:16.968742+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962594+0000","last_change":"2023-10-09T08:46:18.677157+0000","last_active":"2023-10-09T08:47:15.962594+0000","last_peered":"2023-10-09T08:47:15.962594+0000","last_clean":"2023-10-09T08:47:15.962594+0000","last_became_active":"2023-10-09T08:46:18.676750+0000","last_became_peered":"2023-10-09T08:46:18.676750+0000","last_unstale":"2023-10-09T08:47:15.962594+0000","last_undegraded":"2023-10-09T08:47:15.962594+0000","last_fullsized":"2023-10-09T08:47:15.962594+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.964981+0000","last_change":"2023-10-09T08:47:16.964981+0000","last_active":"2023-10-09T08:47:16.964981+0000","last_peered":"2023-10-09T08:47:16.964981+0000","last_clean":"2023-10-09T08:47:16.964981+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.964981+0000","last_undegraded":"2023-10-09T08:47:16.964981+0000","last_fullsized":"2023-10-09T08:47:16.964981+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:04.336414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953808+0000","last_change":"2023-10-09T08:45:49.184830+0000","last_active":"2023-10-09T08:47:16.953808+0000","last_peered":"2023-10-09T08:47:16.953808+0000","last_clean":"2023-10-09T08:47:16.953808+0000","last_became_active":"2023-10-09T08:45:49.184434+0000","last_became_peered":"2023-10-09T08:45:49.184434+0000","last_unstale":"2023-10-09T08:47:16.953808+0000","last_undegraded":"2023-10-09T08:47:16.953808+0000","last_fullsized":"2023-10-09T08:47:16.953808+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:41.880170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950161+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:47:15.950161+0000","last_peered":"2023-10-09T08:47:15.950161+0000","last_clean":"2023-10-09T08:47:15.950161+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:47:15.950161+0000","last_undegraded":"2023-10-09T08:47:15.950161+0000","last_fullsized":"2023-10-09T08:47:15.950161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954285+0000","last_change":"2023-10-09T08:46:18.683082+0000","last_active":"2023-10-09T08:47:16.954285+0000","last_peered":"2023-10-09T08:47:16.954285+0000","last_clean":"2023-10-09T08:47:16.954285+0000","last_became_active":"2023-10-09T08:46:18.682657+0000","last_became_peered":"2023-10-09T08:46:18.682657+0000","last_unstale":"2023-10-09T08:47:16.954285+0000","last_undegraded":"2023-10-09T08:47:16.954285+0000","last_fullsized":"2023-10-09T08:47:16.954285+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:45.531585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956212+0000","last_change":"2023-10-09T08:46:49.232591+0000","last_active":"2023-10-09T08:47:15.956212+0000","last_peered":"2023-10-09T08:47:15.956212+0000","last_clean":"2023-10-09T08:47:15.956212+0000","last_became_active":"2023-10-09T08:46:49.232216+0000","last_became_peered":"2023-10-09T08:46:49.232216+0000","last_unstale":"2023-10-09T08:47:15.956212+0000","last_undegraded":"2023-10-09T08:47:15.956212+0000","last_fullsized":"2023-10-09T08:47:15.956212+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968372+0000","last_change":"2023-10-09T08:47:16.968372+0000","last_active":"2023-10-09T08:47:16.968372+0000","last_peered":"2023-10-09T08:47:16.968372+0000","last_clean":"2023-10-09T08:47:16.968372+0000","last_became_active":"2023-10-09T08:47:16.968024+0000","last_became_peered":"2023-10-09T08:47:16.968024+0000","last_unstale":"2023-10-09T08:47:16.968372+0000","last_undegraded":"2023-10-09T08:47:16.968372+0000","last_fullsized":"2023-10-09T08:47:16.968372+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:07.642684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962677+0000","last_change":"2023-10-09T08:46:18.678094+0000","last_active":"2023-10-09T08:47:16.962677+0000","last_peered":"2023-10-09T08:47:16.962677+0000","last_clean":"2023-10-09T08:47:16.962677+0000","last_became_active":"2023-10-09T08:46:18.677705+0000","last_became_peered":"2023-10-09T08:46:18.677705+0000","last_unstale":"2023-10-09T08:47:16.962677+0000","last_undegraded":"2023-10-09T08:47:16.962677+0000","last_fullsized":"2023-10-09T08:47:16.962677+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:31.647828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974508+0000","last_change":"2023-10-09T08:47:16.974508+0000","last_active":"2023-10-09T08:47:16.974508+0000","last_peered":"2023-10-09T08:47:16.974508+0000","last_clean":"2023-10-09T08:47:16.974508+0000","last_became_active":"2023-10-09T08:47:16.974255+0000","last_became_peered":"2023-10-09T08:47:16.974255+0000","last_unstale":"2023-10-09T08:47:16.974508+0000","last_undegraded":"2023-10-09T08:47:16.974508+0000","last_fullsized":"2023-10-09T08:47:16.974508+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:40.049955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953435+0000","last_change":"2023-10-09T08:45:49.182610+0000","last_active":"2023-10-09T08:47:16.953435+0000","last_peered":"2023-10-09T08:47:16.953435+0000","last_clean":"2023-10-09T08:47:16.953435+0000","last_became_active":"2023-10-09T08:45:49.182013+0000","last_became_peered":"2023-10-09T08:45:49.182013+0000","last_unstale":"2023-10-09T08:47:16.953435+0000","last_undegraded":"2023-10-09T08:47:16.953435+0000","last_fullsized":"2023-10-09T08:47:16.953435+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:44.316894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962365+0000","last_change":"2023-10-09T08:46:49.233112+0000","last_active":"2023-10-09T08:47:16.962365+0000","last_peered":"2023-10-09T08:47:16.962365+0000","last_clean":"2023-10-09T08:47:16.962365+0000","last_became_active":"2023-10-09T08:46:49.232724+0000","last_became_peered":"2023-10-09T08:46:49.232724+0000","last_unstale":"2023-10-09T08:47:16.962365+0000","last_undegraded":"2023-10-09T08:47:16.962365+0000","last_fullsized":"2023-10-09T08:47:16.962365+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.285159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961023+0000","last_change":"2023-10-09T08:45:49.181115+0000","last_active":"2023-10-09T08:47:15.961023+0000","last_peered":"2023-10-09T08:47:15.961023+0000","last_clean":"2023-10-09T08:47:15.961023+0000","last_became_active":"2023-10-09T08:45:49.180736+0000","last_became_peered":"2023-10-09T08:45:49.180736+0000","last_unstale":"2023-10-09T08:47:15.961023+0000","last_undegraded":"2023-10-09T08:47:15.961023+0000","last_fullsized":"2023-10-09T08:47:15.961023+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954452+0000","last_change":"2023-10-09T08:46:49.232279+0000","last_active":"2023-10-09T08:47:16.954452+0000","last_peered":"2023-10-09T08:47:16.954452+0000","last_clean":"2023-10-09T08:47:16.954452+0000","last_became_active":"2023-10-09T08:46:49.231911+0000","last_became_peered":"2023-10-09T08:46:49.231911+0000","last_unstale":"2023-10-09T08:47:16.954452+0000","last_undegraded":"2023-10-09T08:47:16.954452+0000","last_fullsized":"2023-10-09T08:47:16.954452+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:38.497218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965912+0000","last_change":"2023-10-09T08:47:16.965497+0000","last_active":"2023-10-09T08:47:16.965912+0000","last_peered":"2023-10-09T08:47:16.965912+0000","last_clean":"2023-10-09T08:47:16.965912+0000","last_became_active":"2023-10-09T08:47:16.965181+0000","last_became_peered":"2023-10-09T08:47:16.965181+0000","last_unstale":"2023-10-09T08:47:16.965912+0000","last_undegraded":"2023-10-09T08:47:16.965912+0000","last_fullsized":"2023-10-09T08:47:16.965912+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:52.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956282+0000","last_change":"2023-10-09T08:46:18.680004+0000","last_active":"2023-10-09T08:47:15.956282+0000","last_peered":"2023-10-09T08:47:15.956282+0000","last_clean":"2023-10-09T08:47:15.956282+0000","last_became_active":"2023-10-09T08:46:18.679617+0000","last_became_peered":"2023-10-09T08:46:18.679617+0000","last_unstale":"2023-10-09T08:47:15.956282+0000","last_undegraded":"2023-10-09T08:47:15.956282+0000","last_fullsized":"2023-10-09T08:47:15.956282+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956609+0000","last_change":"2023-10-09T08:34:27.860287+0000","last_active":"2023-10-09T08:47:15.956609+0000","last_peered":"2023-10-09T08:47:15.956609+0000","last_clean":"2023-10-09T08:47:15.956609+0000","last_became_active":"2023-10-09T08:34:27.859851+0000","last_became_peered":"2023-10-09T08:34:27.859851+0000","last_unstale":"2023-10-09T08:47:15.956609+0000","last_undegraded":"2023-10-09T08:47:15.956609+0000","last_fullsized":"2023-10-09T08:47:15.956609+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973286+0000","last_change":"2023-10-09T08:47:16.973286+0000","last_active":"2023-10-09T08:47:16.973286+0000","last_peered":"2023-10-09T08:47:16.973286+0000","last_clean":"2023-10-09T08:47:16.973286+0000","last_became_active":"2023-10-09T08:47:16.972839+0000","last_became_peered":"2023-10-09T08:47:16.972839+0000","last_unstale":"2023-10-09T08:47:16.973286+0000","last_undegraded":"2023-10-09T08:47:16.973286+0000","last_fullsized":"2023-10-09T08:47:16.973286+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:56.524537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":296,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956174+0000","last_change":"2023-10-09T08:45:49.183161+0000","last_active":"2023-10-09T08:47:15.956174+0000","last_peered":"2023-10-09T08:47:15.956174+0000","last_clean":"2023-10-09T08:47:15.956174+0000","last_became_active":"2023-10-09T08:45:49.182560+0000","last_became_peered":"2023-10-09T08:45:49.182560+0000","last_unstale":"2023-10-09T08:47:15.956174+0000","last_undegraded":"2023-10-09T08:47:15.956174+0000","last_fullsized":"2023-10-09T08:47:15.956174+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960367+0000","last_change":"2023-10-09T08:46:49.234616+0000","last_active":"2023-10-09T08:47:15.960367+0000","last_peered":"2023-10-09T08:47:15.960367+0000","last_clean":"2023-10-09T08:47:15.960367+0000","last_became_active":"2023-10-09T08:46:49.234217+0000","last_became_peered":"2023-10-09T08:46:49.234217+0000","last_unstale":"2023-10-09T08:47:15.960367+0000","last_undegraded":"2023-10-09T08:47:15.960367+0000","last_fullsized":"2023-10-09T08:47:15.960367+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962773+0000","last_change":"2023-10-09T08:46:18.681375+0000","last_active":"2023-10-09T08:47:16.962773+0000","last_peered":"2023-10-09T08:47:16.962773+0000","last_clean":"2023-10-09T08:47:16.962773+0000","last_became_active":"2023-10-09T08:46:18.680713+0000","last_became_peered":"2023-10-09T08:46:18.680713+0000","last_unstale":"2023-10-09T08:47:16.962773+0000","last_undegraded":"2023-10-09T08:47:16.962773+0000","last_fullsized":"2023-10-09T08:47:16.962773+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:59.671965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963096+0000","last_change":"2023-10-09T08:46:49.231894+0000","last_active":"2023-10-09T08:47:16.963096+0000","last_peered":"2023-10-09T08:47:16.963096+0000","last_clean":"2023-10-09T08:47:16.963096+0000","last_became_active":"2023-10-09T08:46:49.231531+0000","last_became_peered":"2023-10-09T08:46:49.231531+0000","last_unstale":"2023-10-09T08:47:16.963096+0000","last_undegraded":"2023-10-09T08:47:16.963096+0000","last_fullsized":"2023-10-09T08:47:16.963096+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:50.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968243+0000","last_change":"2023-10-09T08:47:16.966880+0000","last_active":"2023-10-09T08:47:16.968243+0000","last_peered":"2023-10-09T08:47:16.968243+0000","last_clean":"2023-10-09T08:47:16.968243+0000","last_became_active":"2023-10-09T08:47:16.966539+0000","last_became_peered":"2023-10-09T08:47:16.966539+0000","last_unstale":"2023-10-09T08:47:16.968243+0000","last_undegraded":"2023-10-09T08:47:16.968243+0000","last_fullsized":"2023-10-09T08:47:16.968243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:55.047737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975638+0000","last_change":"2023-10-09T08:47:16.975638+0000","last_active":"2023-10-09T08:47:16.975638+0000","last_peered":"2023-10-09T08:47:16.975638+0000","last_clean":"2023-10-09T08:47:16.975638+0000","last_became_active":"2023-10-09T08:47:16.975248+0000","last_became_peered":"2023-10-09T08:47:16.975248+0000","last_unstale":"2023-10-09T08:47:16.975638+0000","last_undegraded":"2023-10-09T08:47:16.975638+0000","last_fullsized":"2023-10-09T08:47:16.975638+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950601+0000","last_change":"2023-10-09T08:45:49.184088+0000","last_active":"2023-10-09T08:47:15.950601+0000","last_peered":"2023-10-09T08:47:15.950601+0000","last_clean":"2023-10-09T08:47:15.950601+0000","last_became_active":"2023-10-09T08:45:49.183674+0000","last_became_peered":"2023-10-09T08:45:49.183674+0000","last_unstale":"2023-10-09T08:47:15.950601+0000","last_undegraded":"2023-10-09T08:47:15.950601+0000","last_fullsized":"2023-10-09T08:47:15.950601+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962059+0000","last_change":"2023-10-09T08:46:18.680020+0000","last_active":"2023-10-09T08:47:15.962059+0000","last_peered":"2023-10-09T08:47:15.962059+0000","last_clean":"2023-10-09T08:47:15.962059+0000","last_became_active":"2023-10-09T08:46:18.679652+0000","last_became_peered":"2023-10-09T08:46:18.679652+0000","last_unstale":"2023-10-09T08:47:15.962059+0000","last_undegraded":"2023-10-09T08:47:15.962059+0000","last_fullsized":"2023-10-09T08:47:15.962059+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963009+0000","last_change":"2023-10-09T08:46:18.684501+0000","last_active":"2023-10-09T08:47:16.963009+0000","last_peered":"2023-10-09T08:47:16.963009+0000","last_clean":"2023-10-09T08:47:16.963009+0000","last_became_active":"2023-10-09T08:46:18.684159+0000","last_became_peered":"2023-10-09T08:46:18.684159+0000","last_unstale":"2023-10-09T08:47:16.963009+0000","last_undegraded":"2023-10-09T08:47:16.963009+0000","last_fullsized":"2023-10-09T08:47:16.963009+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:21.270259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.976238+0000","last_change":"2023-10-09T08:47:16.976238+0000","last_active":"2023-10-09T08:47:16.976238+0000","last_peered":"2023-10-09T08:47:16.976238+0000","last_clean":"2023-10-09T08:47:16.976238+0000","last_became_active":"2023-10-09T08:47:16.975840+0000","last_became_peered":"2023-10-09T08:47:16.975840+0000","last_unstale":"2023-10-09T08:47:16.976238+0000","last_undegraded":"2023-10-09T08:47:16.976238+0000","last_fullsized":"2023-10-09T08:47:16.976238+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962626+0000","last_change":"2023-10-09T08:46:18.680373+0000","last_active":"2023-10-09T08:47:16.962626+0000","last_peered":"2023-10-09T08:47:16.962626+0000","last_clean":"2023-10-09T08:47:16.962626+0000","last_became_active":"2023-10-09T08:46:18.679999+0000","last_became_peered":"2023-10-09T08:46:18.679999+0000","last_unstale":"2023-10-09T08:47:16.962626+0000","last_undegraded":"2023-10-09T08:47:16.962626+0000","last_fullsized":"2023-10-09T08:47:16.962626+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:32.374511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965963+0000","last_change":"2023-10-09T08:47:16.965479+0000","last_active":"2023-10-09T08:47:16.965963+0000","last_peered":"2023-10-09T08:47:16.965963+0000","last_clean":"2023-10-09T08:47:16.965963+0000","last_became_active":"2023-10-09T08:47:16.965141+0000","last_became_peered":"2023-10-09T08:47:16.965141+0000","last_unstale":"2023-10-09T08:47:16.965963+0000","last_undegraded":"2023-10-09T08:47:16.965963+0000","last_fullsized":"2023-10-09T08:47:16.965963+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:50.145649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968443+0000","last_change":"2023-10-09T08:47:16.968443+0000","last_active":"2023-10-09T08:47:16.968443+0000","last_peered":"2023-10-09T08:47:16.968443+0000","last_clean":"2023-10-09T08:47:16.968443+0000","last_became_active":"2023-10-09T08:47:16.967631+0000","last_became_peered":"2023-10-09T08:47:16.967631+0000","last_unstale":"2023-10-09T08:47:16.968443+0000","last_undegraded":"2023-10-09T08:47:16.968443+0000","last_fullsized":"2023-10-09T08:47:16.968443+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:19.412588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951852+0000","last_change":"2023-10-09T08:46:18.676843+0000","last_active":"2023-10-09T08:47:15.951852+0000","last_peered":"2023-10-09T08:47:15.951852+0000","last_clean":"2023-10-09T08:47:15.951852+0000","last_became_active":"2023-10-09T08:46:18.676468+0000","last_became_peered":"2023-10-09T08:46:18.676468+0000","last_unstale":"2023-10-09T08:47:15.951852+0000","last_undegraded":"2023-10-09T08:47:15.951852+0000","last_fullsized":"2023-10-09T08:47:15.951852+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950381+0000","last_change":"2023-10-09T08:46:49.235099+0000","last_active":"2023-10-09T08:47:15.950381+0000","last_peered":"2023-10-09T08:47:15.950381+0000","last_clean":"2023-10-09T08:47:15.950381+0000","last_became_active":"2023-10-09T08:46:49.234136+0000","last_became_peered":"2023-10-09T08:46:49.234136+0000","last_unstale":"2023-10-09T08:47:15.950381+0000","last_undegraded":"2023-10-09T08:47:15.950381+0000","last_fullsized":"2023-10-09T08:47:15.950381+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.952239+0000","last_change":"2023-10-09T08:46:49.234714+0000","last_active":"2023-10-09T08:47:15.952239+0000","last_peered":"2023-10-09T08:47:15.952239+0000","last_clean":"2023-10-09T08:47:15.952239+0000","last_became_active":"2023-10-09T08:46:49.234307+0000","last_became_peered":"2023-10-09T08:46:49.234307+0000","last_unstale":"2023-10-09T08:47:15.952239+0000","last_undegraded":"2023-10-09T08:47:15.952239+0000","last_fullsized":"2023-10-09T08:47:15.952239+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957582+0000","last_change":"2023-10-09T08:46:18.681222+0000","last_active":"2023-10-09T08:47:15.957582+0000","last_peered":"2023-10-09T08:47:15.957582+0000","last_clean":"2023-10-09T08:47:15.957582+0000","last_became_active":"2023-10-09T08:46:18.680710+0000","last_became_peered":"2023-10-09T08:46:18.680710+0000","last_unstale":"2023-10-09T08:47:15.957582+0000","last_undegraded":"2023-10-09T08:47:15.957582+0000","last_fullsized":"2023-10-09T08:47:15.957582+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972109+0000","last_change":"2023-10-09T08:47:16.972109+0000","last_active":"2023-10-09T08:47:16.972109+0000","last_peered":"2023-10-09T08:47:16.972109+0000","last_clean":"2023-10-09T08:47:16.972109+0000","last_became_active":"2023-10-09T08:47:16.971716+0000","last_became_peered":"2023-10-09T08:47:16.971716+0000","last_unstale":"2023-10-09T08:47:16.972109+0000","last_undegraded":"2023-10-09T08:47:16.972109+0000","last_fullsized":"2023-10-09T08:47:16.972109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965045+0000","last_change":"2023-10-09T08:47:16.965045+0000","last_active":"2023-10-09T08:47:16.965045+0000","last_peered":"2023-10-09T08:47:16.965045+0000","last_clean":"2023-10-09T08:47:16.965045+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.965045+0000","last_undegraded":"2023-10-09T08:47:16.965045+0000","last_fullsized":"2023-10-09T08:47:16.965045+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:49.126847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953696+0000","last_change":"2023-10-09T08:46:18.683057+0000","last_active":"2023-10-09T08:47:16.953696+0000","last_peered":"2023-10-09T08:47:16.953696+0000","last_clean":"2023-10-09T08:47:16.953696+0000","last_became_active":"2023-10-09T08:46:18.682672+0000","last_became_peered":"2023-10-09T08:46:18.682672+0000","last_unstale":"2023-10-09T08:47:16.953696+0000","last_undegraded":"2023-10-09T08:47:16.953696+0000","last_fullsized":"2023-10-09T08:47:16.953696+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:42.975534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975514+0000","last_change":"2023-10-09T08:47:16.975514+0000","last_active":"2023-10-09T08:47:16.975514+0000","last_peered":"2023-10-09T08:47:16.975514+0000","last_clean":"2023-10-09T08:47:16.975514+0000","last_became_active":"2023-10-09T08:47:16.975184+0000","last_became_peered":"2023-10-09T08:47:16.975184+0000","last_unstale":"2023-10-09T08:47:16.975514+0000","last_undegraded":"2023-10-09T08:47:16.975514+0000","last_fullsized":"2023-10-09T08:47:16.975514+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957336+0000","last_change":"2023-10-09T08:46:49.231597+0000","last_active":"2023-10-09T08:47:15.957336+0000","last_peered":"2023-10-09T08:47:15.957336+0000","last_clean":"2023-10-09T08:47:15.957336+0000","last_became_active":"2023-10-09T08:46:49.231198+0000","last_became_peered":"2023-10-09T08:46:49.231198+0000","last_unstale":"2023-10-09T08:47:15.957336+0000","last_undegraded":"2023-10-09T08:47:15.957336+0000","last_fullsized":"2023-10-09T08:47:15.957336+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961042+0000","last_change":"2023-10-09T08:34:27.733891+0000","last_active":"2023-10-09T08:47:15.961042+0000","last_peered":"2023-10-09T08:47:15.961042+0000","last_clean":"2023-10-09T08:47:15.961042+0000","last_became_active":"2023-10-09T08:34:27.733379+0000","last_became_peered":"2023-10-09T08:34:27.733379+0000","last_unstale":"2023-10-09T08:47:15.961042+0000","last_undegraded":"2023-10-09T08:47:15.961042+0000","last_fullsized":"2023-10-09T08:47:15.961042+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972571+0000","last_change":"2023-10-09T08:47:16.972571+0000","last_active":"2023-10-09T08:47:16.972571+0000","last_peered":"2023-10-09T08:47:16.972571+0000","last_clean":"2023-10-09T08:47:16.972571+0000","last_became_active":"2023-10-09T08:47:16.972035+0000","last_became_peered":"2023-10-09T08:47:16.972035+0000","last_unstale":"2023-10-09T08:47:16.972571+0000","last_undegraded":"2023-10-09T08:47:16.972571+0000","last_fullsized":"2023-10-09T08:47:16.972571+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957950+0000","last_change":"2023-10-09T08:46:18.674774+0000","last_active":"2023-10-09T08:47:15.957950+0000","last_peered":"2023-10-09T08:47:15.957950+0000","last_clean":"2023-10-09T08:47:15.957950+0000","last_became_active":"2023-10-09T08:46:18.674334+0000","last_became_peered":"2023-10-09T08:46:18.674334+0000","last_unstale":"2023-10-09T08:47:15.957950+0000","last_undegraded":"2023-10-09T08:47:15.957950+0000","last_fullsized":"2023-10-09T08:47:15.957950+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960548+0000","last_change":"2023-10-09T08:34:29.490932+0000","last_active":"2023-10-09T08:47:15.960548+0000","last_peered":"2023-10-09T08:47:15.960548+0000","last_clean":"2023-10-09T08:47:15.960548+0000","last_became_active":"2023-10-09T08:34:29.489841+0000","last_became_peered":"2023-10-09T08:34:29.489841+0000","last_unstale":"2023-10-09T08:47:15.960548+0000","last_undegraded":"2023-10-09T08:47:15.960548+0000","last_fullsized":"2023-10-09T08:47:15.960548+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958369+0000","last_change":"2023-10-09T08:45:49.182558+0000","last_active":"2023-10-09T08:47:15.958369+0000","last_peered":"2023-10-09T08:47:15.958369+0000","last_clean":"2023-10-09T08:47:15.958369+0000","last_became_active":"2023-10-09T08:45:49.182145+0000","last_became_peered":"2023-10-09T08:45:49.182145+0000","last_unstale":"2023-10-09T08:47:15.958369+0000","last_undegraded":"2023-10-09T08:47:15.958369+0000","last_fullsized":"2023-10-09T08:47:15.958369+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974455+0000","last_change":"2023-10-09T08:47:16.974455+0000","last_active":"2023-10-09T08:47:16.974455+0000","last_peered":"2023-10-09T08:47:16.974455+0000","last_clean":"2023-10-09T08:47:16.974455+0000","last_became_active":"2023-10-09T08:47:16.973803+0000","last_became_peered":"2023-10-09T08:47:16.973803+0000","last_unstale":"2023-10-09T08:47:16.974455+0000","last_undegraded":"2023-10-09T08:47:16.974455+0000","last_fullsized":"2023-10-09T08:47:16.974455+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:02.753047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954654+0000","last_change":"2023-10-09T08:45:49.188401+0000","last_active":"2023-10-09T08:47:15.954654+0000","last_peered":"2023-10-09T08:47:15.954654+0000","last_clean":"2023-10-09T08:47:15.954654+0000","last_became_active":"2023-10-09T08:45:49.188014+0000","last_became_peered":"2023-10-09T08:45:49.188014+0000","last_unstale":"2023-10-09T08:47:15.954654+0000","last_undegraded":"2023-10-09T08:47:15.954654+0000","last_fullsized":"2023-10-09T08:47:15.954654+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972320+0000","last_change":"2023-10-09T08:47:16.971847+0000","last_active":"2023-10-09T08:47:16.972320+0000","last_peered":"2023-10-09T08:47:16.972320+0000","last_clean":"2023-10-09T08:47:16.972320+0000","last_became_active":"2023-10-09T08:47:16.970986+0000","last_became_peered":"2023-10-09T08:47:16.970986+0000","last_unstale":"2023-10-09T08:47:16.972320+0000","last_undegraded":"2023-10-09T08:47:16.972320+0000","last_fullsized":"2023-10-09T08:47:16.972320+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:06.084956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954882+0000","last_change":"2023-10-09T08:34:27.726666+0000","last_active":"2023-10-09T08:47:15.954882+0000","last_peered":"2023-10-09T08:47:15.954882+0000","last_clean":"2023-10-09T08:47:15.954882+0000","last_became_active":"2023-10-09T08:34:27.726189+0000","last_became_peered":"2023-10-09T08:34:27.726189+0000","last_unstale":"2023-10-09T08:47:15.954882+0000","last_undegraded":"2023-10-09T08:47:15.954882+0000","last_fullsized":"2023-10-09T08:47:15.954882+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975645+0000","last_change":"2023-10-09T08:47:16.975645+0000","last_active":"2023-10-09T08:47:16.975645+0000","last_peered":"2023-10-09T08:47:16.975645+0000","last_clean":"2023-10-09T08:47:16.975645+0000","last_became_active":"2023-10-09T08:47:16.975277+0000","last_became_peered":"2023-10-09T08:47:16.975277+0000","last_unstale":"2023-10-09T08:47:16.975645+0000","last_undegraded":"2023-10-09T08:47:16.975645+0000","last_fullsized":"2023-10-09T08:47:16.975645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.971694+0000","last_change":"2023-10-09T08:47:16.971357+0000","last_active":"2023-10-09T08:47:16.971694+0000","last_peered":"2023-10-09T08:47:16.971694+0000","last_clean":"2023-10-09T08:47:16.971694+0000","last_became_active":"2023-10-09T08:47:16.971034+0000","last_became_peered":"2023-10-09T08:47:16.971034+0000","last_unstale":"2023-10-09T08:47:16.971694+0000","last_undegraded":"2023-10-09T08:47:16.971694+0000","last_fullsized":"2023-10-09T08:47:16.971694+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:05.892369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974125+0000","last_change":"2023-10-09T08:47:16.974125+0000","last_active":"2023-10-09T08:47:16.974125+0000","last_peered":"2023-10-09T08:47:16.974125+0000","last_clean":"2023-10-09T08:47:16.974125+0000","last_became_active":"2023-10-09T08:47:16.973763+0000","last_became_peered":"2023-10-09T08:47:16.973763+0000","last_unstale":"2023-10-09T08:47:16.974125+0000","last_undegraded":"2023-10-09T08:47:16.974125+0000","last_fullsized":"2023-10-09T08:47:16.974125+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:31.548547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958645+0000","last_change":"2023-10-09T08:46:49.233182+0000","last_active":"2023-10-09T08:47:15.958645+0000","last_peered":"2023-10-09T08:47:15.958645+0000","last_clean":"2023-10-09T08:47:15.958645+0000","last_became_active":"2023-10-09T08:46:49.232746+0000","last_became_peered":"2023-10-09T08:46:49.232746+0000","last_unstale":"2023-10-09T08:47:15.958645+0000","last_undegraded":"2023-10-09T08:47:15.958645+0000","last_fullsized":"2023-10-09T08:47:15.958645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951634+0000","last_change":"2023-10-09T08:46:18.676902+0000","last_active":"2023-10-09T08:47:15.951634+0000","last_peered":"2023-10-09T08:47:15.951634+0000","last_clean":"2023-10-09T08:47:15.951634+0000","last_became_active":"2023-10-09T08:46:18.676560+0000","last_became_peered":"2023-10-09T08:46:18.676560+0000","last_unstale":"2023-10-09T08:47:15.951634+0000","last_undegraded":"2023-10-09T08:47:15.951634+0000","last_fullsized":"2023-10-09T08:47:15.951634+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":68,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951207+0000","last_change":"2023-10-09T08:34:27.730961+0000","last_active":"2023-10-09T08:47:15.951207+0000","last_peered":"2023-10-09T08:47:15.951207+0000","last_clean":"2023-10-09T08:47:15.951207+0000","last_became_active":"2023-10-09T08:34:27.730198+0000","last_became_peered":"2023-10-09T08:34:27.730198+0000","last_unstale":"2023-10-09T08:47:15.951207+0000","last_undegraded":"2023-10-09T08:47:15.951207+0000","last_fullsized":"2023-10-09T08:47:15.951207+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963049+0000","last_change":"2023-10-09T08:46:49.231880+0000","last_active":"2023-10-09T08:47:16.963049+0000","last_peered":"2023-10-09T08:47:16.963049+0000","last_clean":"2023-10-09T08:47:16.963049+0000","last_became_active":"2023-10-09T08:46:49.231471+0000","last_became_peered":"2023-10-09T08:46:49.231471+0000","last_unstale":"2023-10-09T08:47:16.963049+0000","last_undegraded":"2023-10-09T08:47:16.963049+0000","last_fullsized":"2023-10-09T08:47:16.963049+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:31.742745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973891+0000","last_change":"2023-10-09T08:47:16.973891+0000","last_active":"2023-10-09T08:47:16.973891+0000","last_peered":"2023-10-09T08:47:16.973891+0000","last_clean":"2023-10-09T08:47:16.973891+0000","last_became_active":"2023-10-09T08:47:16.973405+0000","last_became_peered":"2023-10-09T08:47:16.973405+0000","last_unstale":"2023-10-09T08:47:16.973891+0000","last_undegraded":"2023-10-09T08:47:16.973891+0000","last_fullsized":"2023-10-09T08:47:16.973891+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972498+0000","last_change":"2023-10-09T08:47:16.972498+0000","last_active":"2023-10-09T08:47:16.972498+0000","last_peered":"2023-10-09T08:47:16.972498+0000","last_clean":"2023-10-09T08:47:16.972498+0000","last_became_active":"2023-10-09T08:47:16.972184+0000","last_became_peered":"2023-10-09T08:47:16.972184+0000","last_unstale":"2023-10-09T08:47:16.972498+0000","last_undegraded":"2023-10-09T08:47:16.972498+0000","last_fullsized":"2023-10-09T08:47:16.972498+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:03.559858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960484+0000","last_change":"2023-10-09T08:46:18.675538+0000","last_active":"2023-10-09T08:47:15.960484+0000","last_peered":"2023-10-09T08:47:15.960484+0000","last_clean":"2023-10-09T08:47:15.960484+0000","last_became_active":"2023-10-09T08:46:18.675149+0000","last_became_peered":"2023-10-09T08:46:18.675149+0000","last_unstale":"2023-10-09T08:47:15.960484+0000","last_undegraded":"2023-10-09T08:47:15.960484+0000","last_fullsized":"2023-10-09T08:47:15.960484+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969948+0000","last_change":"2023-10-09T08:47:16.969948+0000","last_active":"2023-10-09T08:47:16.969948+0000","last_peered":"2023-10-09T08:47:16.969948+0000","last_clean":"2023-10-09T08:47:16.969948+0000","last_became_active":"2023-10-09T08:47:16.969666+0000","last_became_peered":"2023-10-09T08:47:16.969666+0000","last_unstale":"2023-10-09T08:47:16.969948+0000","last_undegraded":"2023-10-09T08:47:16.969948+0000","last_fullsized":"2023-10-09T08:47:16.969948+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:25.089633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972317+0000","last_change":"2023-10-09T08:47:16.971965+0000","last_active":"2023-10-09T08:47:16.972317+0000","last_peered":"2023-10-09T08:47:16.972317+0000","last_clean":"2023-10-09T08:47:16.972317+0000","last_became_active":"2023-10-09T08:47:16.971651+0000","last_became_peered":"2023-10-09T08:47:16.971651+0000","last_unstale":"2023-10-09T08:47:16.972317+0000","last_undegraded":"2023-10-09T08:47:16.972317+0000","last_fullsized":"2023-10-09T08:47:16.972317+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:04.259603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956192+0000","last_change":"2023-10-09T08:46:18.680365+0000","last_active":"2023-10-09T08:47:15.956192+0000","last_peered":"2023-10-09T08:47:15.956192+0000","last_clean":"2023-10-09T08:47:15.956192+0000","last_became_active":"2023-10-09T08:46:18.679875+0000","last_became_peered":"2023-10-09T08:46:18.679875+0000","last_unstale":"2023-10-09T08:47:15.956192+0000","last_undegraded":"2023-10-09T08:47:15.956192+0000","last_fullsized":"2023-10-09T08:47:15.956192+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965460+0000","last_change":"2023-10-09T08:47:16.965460+0000","last_active":"2023-10-09T08:47:16.965460+0000","last_peered":"2023-10-09T08:47:16.965460+0000","last_clean":"2023-10-09T08:47:16.965460+0000","last_became_active":"2023-10-09T08:47:16.965040+0000","last_became_peered":"2023-10-09T08:47:16.965040+0000","last_unstale":"2023-10-09T08:47:16.965460+0000","last_undegraded":"2023-10-09T08:47:16.965460+0000","last_fullsized":"2023-10-09T08:47:16.965460+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.025735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972434+0000","last_change":"2023-10-09T08:47:16.972106+0000","last_active":"2023-10-09T08:47:16.972434+0000","last_peered":"2023-10-09T08:47:16.972434+0000","last_clean":"2023-10-09T08:47:16.972434+0000","last_became_active":"2023-10-09T08:47:16.971511+0000","last_became_peered":"2023-10-09T08:47:16.971511+0000","last_unstale":"2023-10-09T08:47:16.972434+0000","last_undegraded":"2023-10-09T08:47:16.972434+0000","last_fullsized":"2023-10-09T08:47:16.972434+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:12.674597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969696+0000","last_change":"2023-10-09T08:47:16.968800+0000","last_active":"2023-10-09T08:47:16.969696+0000","last_peered":"2023-10-09T08:47:16.969696+0000","last_clean":"2023-10-09T08:47:16.969696+0000","last_became_active":"2023-10-09T08:47:16.968430+0000","last_became_peered":"2023-10-09T08:47:16.968430+0000","last_unstale":"2023-10-09T08:47:16.969696+0000","last_undegraded":"2023-10-09T08:47:16.969696+0000","last_fullsized":"2023-10-09T08:47:16.969696+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:08.011498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965051+0000","last_change":"2023-10-09T08:47:16.965051+0000","last_active":"2023-10-09T08:47:16.965051+0000","last_peered":"2023-10-09T08:47:16.965051+0000","last_clean":"2023-10-09T08:47:16.965051+0000","last_became_active":"2023-10-09T08:47:16.964664+0000","last_became_peered":"2023-10-09T08:47:16.964664+0000","last_unstale":"2023-10-09T08:47:16.965051+0000","last_undegraded":"2023-10-09T08:47:16.965051+0000","last_fullsized":"2023-10-09T08:47:16.965051+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.377202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950228+0000","last_change":"2023-10-09T08:46:49.233483+0000","last_active":"2023-10-09T08:47:15.950228+0000","last_peered":"2023-10-09T08:47:15.950228+0000","last_clean":"2023-10-09T08:47:15.950228+0000","last_became_active":"2023-10-09T08:46:49.233033+0000","last_became_peered":"2023-10-09T08:46:49.233033+0000","last_unstale":"2023-10-09T08:47:15.950228+0000","last_undegraded":"2023-10-09T08:47:15.950228+0000","last_fullsized":"2023-10-09T08:47:15.950228+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430423,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310904,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434248,"statfs":{"total":95995035648,"available":95676669952,"internally_reserved":0,"allocated":1556480,"data_stored":1102308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.92900000000000005,"15min":0.92200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.2569999999999999,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.94199999999999995,"15min":0.94499999999999995},"min":{"1min":0.64300000000000002,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1679999999999999,"5min":1.262,"15min":1.262},"last":0.98299999999999998}]},{"osd":1,"last update":"Mon Oct 9 08:48:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.99299999999999999,"15min":0.98899999999999999},"min":{"1min":0.72999999999999998,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.24,"5min":1.446,"15min":1.446},"last":0.874},{"interface":"front","average":{"1min":1.004,"5min":0.97899999999999998,"15min":0.97499999999999998},"min":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.181,"5min":1.319,"15min":1.319},"last":0.91200000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.91400000000000003,"15min":0.92400000000000004},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.125,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.92700000000000005,"15min":0.93000000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":1.1020000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.95699999999999996}]},{"osd":4,"last update":"Mon Oct 9 08:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.88300000000000001,"15min":0.88400000000000001},"min":{"1min":0.69199999999999995,"5min":0.45500000000000002,"15min":0.40300000000000002},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.4630000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.96499999999999997,"15min":0.93799999999999994},"min":{"1min":0.69499999999999995,"5min":0.54800000000000004,"15min":0.40000000000000002},"max":{"1min":1.262,"5min":1.488,"15min":1.488},"last":1.0640000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97799999999999998,"15min":0.98899999999999999},"min":{"1min":0.69099999999999995,"5min":0.58799999999999997,"15min":0.501},"max":{"1min":1.131,"5min":1.329,"15min":1.6399999999999999},"last":0.749},{"interface":"front","average":{"1min":1.028,"5min":1.0129999999999999,"15min":0.999},"min":{"1min":0.67300000000000004,"5min":0.54900000000000004,"15min":0.46100000000000002},"max":{"1min":1.3400000000000001,"5min":1.3400000000000001,"15min":1.794},"last":0.80100000000000005}]},{"osd":6,"last update":"Mon Oct 9 08:48:09 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.04,"15min":1.032},"min":{"1min":0.76900000000000002,"5min":0.57499999999999996,"15min":0.44},"max":{"1min":1.409,"5min":1.4350000000000001,"15min":1.7170000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.0920000000000001,"15min":1.0649999999999999},"min":{"1min":0.79100000000000004,"5min":0.63,"15min":0.45000000000000001},"max":{"1min":1.3240000000000001,"5min":1.528,"15min":1.528},"last":0.77400000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626656,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1508,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1544192,"data_stored":1102112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:58 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.93500000000000005,"15min":0.93000000000000005},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2609999999999999,"5min":1.349,"15min":1.349},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0129999999999999,"15min":0.996},"min":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3320000000000001,"5min":1.5,"15min":1.5},"last":1.0760000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:48:23 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.89100000000000001,"15min":0.89500000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":1.2},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.58099999999999996,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.84699999999999998}]},{"osd":2,"last update":"Mon Oct 9 08:48:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.91400000000000003,"15min":0.91600000000000004},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.94899999999999995,"15min":0.93999999999999995},"min":{"1min":0.76500000000000001,"5min":0.625,"15min":0.625},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.1830000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:48:16 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":1.01},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":1.036}]},{"osd":4,"last update":"Mon Oct 9 08:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.873,"15min":0.89200000000000002},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.379},"max":{"1min":1.159,"5min":1.286,"15min":1.629},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.94299999999999995},"min":{"1min":0.70999999999999996,"5min":0.436,"15min":0.436},"max":{"1min":1.2549999999999999,"5min":1.373,"15min":1.671},"last":1.236}]},{"osd":5,"last update":"Mon Oct 9 08:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.96999999999999997,"15min":0.97499999999999998},"min":{"1min":0.80100000000000005,"5min":0.495,"15min":0.495},"max":{"1min":1.2190000000000001,"5min":1.3720000000000001,"15min":1.7410000000000001},"last":1.325},{"interface":"front","average":{"1min":1.036,"5min":1.03,"15min":1.038},"min":{"1min":0.76600000000000001,"5min":0.66700000000000004,"15min":0.51600000000000001},"max":{"1min":1.3420000000000001,"5min":1.46,"15min":4.4420000000000002},"last":1.149}]},{"osd":7,"last update":"Mon Oct 9 08:48:58 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.052,"15min":1.056},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.45900000000000002},"max":{"1min":1.3460000000000001,"5min":1.383,"15min":1.8220000000000001},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.0569999999999999,"15min":1.085},"min":{"1min":0.80000000000000004,"5min":0.53300000000000003,"15min":0.52500000000000002},"max":{"1min":1.4119999999999999,"5min":1.556,"15min":2.262},"last":0.94699999999999995}]}]},{"osd":1,"up_from":83,"seq":356482285604,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32396,"kb_used_data":964,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712756,"statfs":{"total":95995035648,"available":95961862144,"internally_reserved":0,"allocated":987136,"data_stored":516401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.626,"15min":0.625},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83399999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.68000000000000005,"15min":0.67400000000000004},"min":{"1min":0.48499999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.88800000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.72299999999999998,"15min":0.73899999999999999},"min":{"1min":0.41199999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80900000000000005,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.69999999999999996,"15min":0.70399999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.95999999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.1699999999999999,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":1.1890000000000001},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.83499999999999996}]},{"osd":4,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.96699999999999997,"15min":0.96499999999999997},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.4079999999999999,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.97599999999999998,"15min":0.97099999999999997},"min":{"1min":0.72799999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.278,"5min":1.577,"15min":1.577},"last":1.157}]},{"osd":5,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"min":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.2869999999999999,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.218},{"interface":"front","average":{"1min":1.024,"5min":1.0609999999999999,"15min":1.0669999999999999},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.298,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":1.3129999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.093,"15min":1.0860000000000001},"min":{"1min":0.68700000000000006,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.391,"5min":1.522,"15min":1.522},"last":1.034},{"interface":"front","average":{"1min":1.127,"5min":1.1120000000000001,"15min":1.1100000000000001},"min":{"1min":0.81799999999999995,"5min":0.754,"15min":0.754},"max":{"1min":1.3560000000000001,"5min":1.577,"15min":1.577},"last":1.111}]},{"osd":7,"last update":"Mon Oct 9 08:48:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.046,"15min":1.0389999999999999},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.365,"5min":1.462,"15min":1.462},"last":1.5760000000000001},{"interface":"front","average":{"1min":1.119,"5min":1.109,"15min":1.107},"min":{"1min":0.69999999999999996,"5min":0.624,"15min":0.624},"max":{"1min":1.427,"5min":1.625,"15min":1.625},"last":1.246}]}]},{"osd":0,"up_from":78,"seq":335007449130,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33060,"kb_used_data":1564,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712092,"statfs":{"total":95995035648,"available":95961182208,"internally_reserved":0,"allocated":1601536,"data_stored":1109136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.71999999999999997,"15min":0.71899999999999997},"min":{"1min":0.45300000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":1.016},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.77500000000000002,"15min":0.78300000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.01,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":1.0489999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.83199999999999996,"15min":0.84799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.083,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.92900000000000005,"15min":0.95099999999999996},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.92100000000000004}]},{"osd":3,"last update":"Mon Oct 9 08:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":1.1819999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0389999999999999,"15min":1.046},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":1.298},{"interface":"front","average":{"1min":0.873,"5min":0.93700000000000006,"15min":0.96199999999999997},"min":{"1min":0.60799999999999998,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.0469999999999999,"5min":1.528,"15min":1.528},"last":0.99399999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0189999999999999,"15min":1.018},"min":{"1min":0.63500000000000001,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.361,"5min":1.51,"15min":1.51},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0980000000000001,"15min":1.115},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.347}]},{"osd":6,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.1359999999999999,"15min":1.1599999999999999},"min":{"1min":0.77200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3959999999999999,"5min":1.663,"15min":1.663},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.1950000000000001,"15min":1.2150000000000001},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.486,"5min":1.609,"15min":1.609},"last":1.3879999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.214,"15min":1.2330000000000001},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.5880000000000001,"5min":1.679,"15min":1.679},"last":1.234},{"interface":"front","average":{"1min":1.028,"5min":1.117,"15min":1.1479999999999999},"min":{"1min":0.64600000000000002,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.3779999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.075}]}]},{"osd":2,"up_from":88,"seq":377957122078,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36616,"kb_used_data":960,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708536,"statfs":{"total":95995035648,"available":95957540864,"internally_reserved":0,"allocated":983040,"data_stored":514873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.70999999999999996,"15min":0.70199999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.79300000000000004,"15min":0.81000000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2629999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1}]},{"osd":1,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.0009999999999999,"5min":1.127,"15min":1.127},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.755,"5min":0.82899999999999996,"15min":0.84099999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.165,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.68200000000000005}]},{"osd":3,"last update":"Mon Oct 9 08:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.96599999999999997,"15min":0.97199999999999998},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.1919999999999999,"5min":1.24,"15min":1.24},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0469999999999999,"15min":1.0580000000000001},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.95699999999999996}]},{"osd":5,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.131,"15min":1.1579999999999999},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.464,"5min":1.49,"15min":1.49},"last":1.268},{"interface":"front","average":{"1min":1,"5min":1.0649999999999999,"15min":1.0760000000000001},"min":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"max":{"1min":1.2230000000000001,"5min":1.337,"15min":1.337},"last":0.89100000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.1259999999999999,"15min":1.137},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.3100000000000001,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.1870000000000001,"15min":1.204},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.446,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.3420000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.1539999999999999,"15min":1.1599999999999999},"min":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"max":{"1min":1.4099999999999999,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.214,"15min":1.2350000000000001},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.5169999999999999,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.389}]}]},{"osd":3,"up_from":93,"seq":399431958552,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28276,"kb_used_data":940,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716876,"statfs":{"total":95995035648,"available":95966081024,"internally_reserved":0,"allocated":962560,"data_stored":511788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":1.1339999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":1.054},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.80800000000000005}]},{"osd":2,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":1.0029999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":1.2649999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.034,"15min":1.034},"min":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"min":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.2969999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"min":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"max":{"1min":1.369,"5min":1.369,"15min":1.369},"last":1.373},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"min":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"max":{"1min":1.482,"5min":1.482,"15min":1.482},"last":1.21}]},{"osd":7,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.163,"15min":1.163},"min":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"max":{"1min":1.52,"5min":1.52,"15min":1.52},"last":1.351},{"interface":"front","average":{"1min":1.167,"5min":1.167,"15min":1.167},"min":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"max":{"1min":1.581,"5min":1.581,"15min":1.581},"last":1.2330000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019122,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302132,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443020,"statfs":{"total":95995035648,"available":95685652480,"internally_reserved":0,"allocated":962560,"data_stored":512135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:49:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.88100000000000001,"15min":0.879},"min":{"1min":0.61699999999999999,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.238,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.179},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.89900000000000002,"15min":0.89600000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":1.0940000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.85299999999999998,"15min":0.85099999999999998},"min":{"1min":0.64900000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.0349999999999999,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.96399999999999997,"15min":0.95999999999999996},"min":{"1min":0.73999999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":1.135}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93799999999999994,"15min":0.93600000000000005},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.03,"5min":0.96599999999999997,"15min":0.95599999999999996},"min":{"1min":0.80200000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.85199999999999998}]},{"osd":3,"last update":"Mon Oct 9 08:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"min":{"1min":0.75,"5min":0.75,"15min":0.75},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.1519999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:48:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.89500000000000002,"15min":0.91400000000000003},"min":{"1min":0.73799999999999999,"5min":0.51200000000000001,"15min":0.39200000000000002},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.111,"5min":0.99199999999999999,"15min":0.99399999999999999},"min":{"1min":0.96999999999999997,"5min":0.48399999999999999,"15min":0.47399999999999998},"max":{"1min":1.389,"5min":1.389,"15min":1.4730000000000001},"last":1.038}]},{"osd":6,"last update":"Mon Oct 9 08:48:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.95699999999999996,"15min":0.94399999999999995},"min":{"1min":0.71799999999999997,"5min":0.54300000000000004,"15min":0.42899999999999999},"max":{"1min":1.2909999999999999,"5min":1.4059999999999999,"15min":1.4179999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.089,"5min":1.0469999999999999,"15min":1.044},"min":{"1min":0.79600000000000004,"5min":0.57599999999999996,"15min":0.48399999999999999},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":1.069}]},{"osd":7,"last update":"Mon Oct 9 08:49:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0860000000000001,"15min":1.0820000000000001},"min":{"1min":0.81100000000000005,"5min":0.55100000000000005,"15min":0.44800000000000001},"max":{"1min":1.484,"5min":1.484,"15min":1.484},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.121,"15min":1.0860000000000001},"min":{"1min":0.89800000000000002,"5min":0.71899999999999997,"15min":0.38},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":1.0069999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822890,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310308,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434844,"statfs":{"total":95995035648,"available":95677280256,"internally_reserved":0,"allocated":946176,"data_stored":507879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.876,"15min":0.876},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.1419999999999999,"5min":1.262,"15min":1.262},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.91100000000000003,"15min":0.90600000000000003},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.133,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.0980000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:48:20 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.91600000000000004,"15min":0.91300000000000003},"min":{"1min":0.72699999999999998,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":1.115},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"min":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.88100000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.92000000000000004,"15min":0.92700000000000005},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1639999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.98999999999999999,"15min":0.998},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.284,"5min":1.323,"15min":1.323},"last":0.99399999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.006,"15min":1.006},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"max":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":1.0720000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:48:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.878,"15min":0.85999999999999999},"min":{"1min":0.68600000000000005,"5min":0.61299999999999999,"15min":0.46999999999999997},"max":{"1min":1.1890000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.95299999999999996,"15min":0.95099999999999996},"min":{"1min":0.60999999999999999,"5min":0.505,"15min":0.442},"max":{"1min":1.212,"5min":1.405,"15min":1.405},"last":0.74199999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.94999999999999996,"15min":0.96699999999999997},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.45200000000000001},"max":{"1min":1.2370000000000001,"5min":1.333,"15min":1.411},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.083,"5min":1.02,"15min":1.01},"min":{"1min":0.69599999999999995,"5min":0.57899999999999996,"15min":0.40400000000000003},"max":{"1min":1.347,"5min":1.4750000000000001,"15min":3.2919999999999998},"last":0.82499999999999996}]},{"osd":7,"last update":"Mon Oct 9 08:48:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.0640000000000001,"15min":1.042},"min":{"1min":0.78100000000000003,"5min":0.53300000000000003,"15min":0.36899999999999999},"max":{"1min":1.2849999999999999,"5min":1.4119999999999999,"15min":1.4910000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.093,"5min":1.071,"15min":1.0820000000000001},"min":{"1min":0.85399999999999998,"5min":0.59999999999999998,"15min":0.45000000000000001},"max":{"1min":1.3140000000000001,"5min":1.4950000000000001,"15min":1.782},"last":0.96599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:49:10.688 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T08:49:10.688 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T08:49:10.688 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T08:49:10.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T08:49:11.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:10 smithi016 bash[75179]: audit 2023-10-09T08:49:09.787518+0000 mgr.y (mgr.25421) 338 : audit 0 from='client.44597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:11.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:10 smithi016 bash[77892]: audit 2023-10-09T08:49:09.787518+0000 mgr.y (mgr.25421) 338 : audit 0 from='client.44597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:11.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:10 smithi094 bash[59677]: audit 2023-10-09T08:49:09.787518+0000 mgr.y (mgr.25421) 338 : audit 0 from='client.44597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:12.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:11 smithi094 bash[59677]: cluster 2023-10-09T08:49:11.285347+0000 mgr.y (mgr.25421) 339 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:11 smithi016 bash[75179]: cluster 2023-10-09T08:49:11.285347+0000 mgr.y (mgr.25421) 339 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:12.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:11 smithi016 bash[77892]: cluster 2023-10-09T08:49:11.285347+0000 mgr.y (mgr.25421) 339 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:13.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:13 smithi094 bash[59677]: cluster 2023-10-09T08:49:13.286160+0000 mgr.y (mgr.25421) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:13.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:13 smithi016 bash[75179]: cluster 2023-10-09T08:49:13.286160+0000 mgr.y (mgr.25421) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:13.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:13 smithi016 bash[77892]: cluster 2023-10-09T08:49:13.286160+0000 mgr.y (mgr.25421) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:15.512 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:49:15.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:15 smithi016 bash[75179]: cluster 2023-10-09T08:49:15.287091+0000 mgr.y (mgr.25421) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:15 smithi016 bash[77892]: cluster 2023-10-09T08:49:15.287091+0000 mgr.y (mgr.25421) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:15.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:15 smithi094 bash[59677]: cluster 2023-10-09T08:49:15.287091+0000 mgr.y (mgr.25421) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:16.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:49:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:49:15] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.33.4" 2023-10-09T08:49:17.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:17 smithi094 bash[59677]: cluster 2023-10-09T08:49:17.287867+0000 mgr.y (mgr.25421) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:17.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:17 smithi016 bash[75179]: cluster 2023-10-09T08:49:17.287867+0000 mgr.y (mgr.25421) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:17.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:17 smithi016 bash[77892]: cluster 2023-10-09T08:49:17.287867+0000 mgr.y (mgr.25421) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:19.025 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:49:19.025 INFO:teuthology.orchestra.run.smithi016.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T08:49:19.875 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T08:49:19.875 INFO:teuthology.run_tasks:Running task print... 2023-10-09T08:49:19.886 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T08:49:19.886 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T08:49:19.917 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T08:49:19.917 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T08:49:19.918 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T08:49:19.919 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T08:49:19.919 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T08:49:19.919 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T08:49:19.920 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T08:49:19.920 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T08:49:19.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:49:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:19 smithi094 bash[59677]: audit 2023-10-09T08:49:19.021579+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.16:0/3447622696' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:49:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:19 smithi094 bash[59677]: audit 2023-10-09T08:49:19.204662+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:20.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:19 smithi094 bash[59677]: cluster 2023-10-09T08:49:19.288555+0000 mgr.y (mgr.25421) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:20.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:19 smithi016 bash[77892]: audit 2023-10-09T08:49:19.021579+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.16:0/3447622696' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:49:20.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:19 smithi016 bash[77892]: audit 2023-10-09T08:49:19.204662+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:20.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:19 smithi016 bash[77892]: cluster 2023-10-09T08:49:19.288555+0000 mgr.y (mgr.25421) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:20.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:19 smithi016 bash[75179]: audit 2023-10-09T08:49:19.021579+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.16:0/3447622696' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T08:49:20.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:19 smithi016 bash[75179]: audit 2023-10-09T08:49:19.204662+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:20.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:19 smithi016 bash[75179]: cluster 2023-10-09T08:49:19.288555+0000 mgr.y (mgr.25421) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:21 smithi094 bash[59677]: cluster 2023-10-09T08:49:21.289486+0000 mgr.y (mgr.25421) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:21.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:21 smithi016 bash[75179]: cluster 2023-10-09T08:49:21.289486+0000 mgr.y (mgr.25421) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:21.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:21 smithi016 bash[77892]: cluster 2023-10-09T08:49:21.289486+0000 mgr.y (mgr.25421) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:23.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:23 smithi094 bash[59677]: cluster 2023-10-09T08:49:23.290353+0000 mgr.y (mgr.25421) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:23.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:23 smithi016 bash[75179]: cluster 2023-10-09T08:49:23.290353+0000 mgr.y (mgr.25421) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:23 smithi016 bash[77892]: cluster 2023-10-09T08:49:23.290353+0000 mgr.y (mgr.25421) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:24.707 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:49:25.660 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:25 smithi016 bash[75179]: cluster 2023-10-09T08:49:25.291228+0000 mgr.y (mgr.25421) 346 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:25.660 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:25 smithi016 bash[77892]: cluster 2023-10-09T08:49:25.291228+0000 mgr.y (mgr.25421) 346 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:25.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:25 smithi094 bash[59677]: cluster 2023-10-09T08:49:25.291228+0000 mgr.y (mgr.25421) 346 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:26.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:49:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:49:25] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-10-09T08:49:27.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:27 smithi016 bash[75179]: cluster 2023-10-09T08:49:27.292052+0000 mgr.y (mgr.25421) 347 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:27.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:27 smithi016 bash[77892]: cluster 2023-10-09T08:49:27.292052+0000 mgr.y (mgr.25421) 347 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:27.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:27 smithi094 bash[59677]: cluster 2023-10-09T08:49:27.292052+0000 mgr.y (mgr.25421) 347 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:28.048 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:49:28.051 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:49:28.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:28 smithi094 bash[59677]: audit 2023-10-09T08:49:28.026468+0000 mgr.y (mgr.25421) 348 : audit 0 from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:28.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:28 smithi016 bash[75179]: audit 2023-10-09T08:49:28.026468+0000 mgr.y (mgr.25421) 348 : audit 0 from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:28.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:28 smithi016 bash[77892]: audit 2023-10-09T08:49:28.026468+0000 mgr.y (mgr.25421) 348 : audit 0 from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:28.961 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":282,"stamp":"2023-10-09T08:49:27.291515+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1352296,"kb_used_data":9320,"kb_used_omap":126,"kb_used_meta":1342785,"kb_avail":748608920,"statfs":{"total":767960285184,"available":766575534080,"internally_reserved":0,"allocated":9543680,"data_stored":5876632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1375012475},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005066"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950127+0000","last_change":"2023-10-09T08:46:18.676602+0000","last_active":"2023-10-09T08:47:15.950127+0000","last_peered":"2023-10-09T08:47:15.950127+0000","last_clean":"2023-10-09T08:47:15.950127+0000","last_became_active":"2023-10-09T08:46:18.676122+0000","last_became_peered":"2023-10-09T08:46:18.676122+0000","last_unstale":"2023-10-09T08:47:15.950127+0000","last_undegraded":"2023-10-09T08:47:15.950127+0000","last_fullsized":"2023-10-09T08:47:15.950127+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953679+0000","last_change":"2023-10-09T08:46:49.233211+0000","last_active":"2023-10-09T08:47:16.953679+0000","last_peered":"2023-10-09T08:47:16.953679+0000","last_clean":"2023-10-09T08:47:16.953679+0000","last_became_active":"2023-10-09T08:46:49.232775+0000","last_became_peered":"2023-10-09T08:46:49.232775+0000","last_unstale":"2023-10-09T08:47:16.953679+0000","last_undegraded":"2023-10-09T08:47:16.953679+0000","last_fullsized":"2023-10-09T08:47:16.953679+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:59.423857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.966115+0000","last_change":"2023-10-09T08:47:16.966115+0000","last_active":"2023-10-09T08:47:16.966115+0000","last_peered":"2023-10-09T08:47:16.966115+0000","last_clean":"2023-10-09T08:47:16.966115+0000","last_became_active":"2023-10-09T08:47:16.965795+0000","last_became_peered":"2023-10-09T08:47:16.965795+0000","last_unstale":"2023-10-09T08:47:16.966115+0000","last_undegraded":"2023-10-09T08:47:16.966115+0000","last_fullsized":"2023-10-09T08:47:16.966115+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:40.817840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":69,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972982+0000","last_change":"2023-10-09T08:47:16.970132+0000","last_active":"2023-10-09T08:47:16.972982+0000","last_peered":"2023-10-09T08:47:16.972982+0000","last_clean":"2023-10-09T08:47:16.972982+0000","last_became_active":"2023-10-09T08:47:16.969760+0000","last_became_peered":"2023-10-09T08:47:16.969760+0000","last_unstale":"2023-10-09T08:47:16.972982+0000","last_undegraded":"2023-10-09T08:47:16.972982+0000","last_fullsized":"2023-10-09T08:47:16.972982+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.801830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969367+0000","last_change":"2023-10-09T08:47:16.969367+0000","last_active":"2023-10-09T08:47:16.969367+0000","last_peered":"2023-10-09T08:47:16.969367+0000","last_clean":"2023-10-09T08:47:16.969367+0000","last_became_active":"2023-10-09T08:47:16.969000+0000","last_became_peered":"2023-10-09T08:47:16.969000+0000","last_unstale":"2023-10-09T08:47:16.969367+0000","last_undegraded":"2023-10-09T08:47:16.969367+0000","last_fullsized":"2023-10-09T08:47:16.969367+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961355+0000","last_change":"2023-10-09T08:34:27.739113+0000","last_active":"2023-10-09T08:47:15.961355+0000","last_peered":"2023-10-09T08:47:15.961355+0000","last_clean":"2023-10-09T08:47:15.961355+0000","last_became_active":"2023-10-09T08:34:27.738645+0000","last_became_peered":"2023-10-09T08:34:27.738645+0000","last_unstale":"2023-10-09T08:47:15.961355+0000","last_undegraded":"2023-10-09T08:47:15.961355+0000","last_fullsized":"2023-10-09T08:47:15.961355+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968742+0000","last_change":"2023-10-09T08:47:16.968742+0000","last_active":"2023-10-09T08:47:16.968742+0000","last_peered":"2023-10-09T08:47:16.968742+0000","last_clean":"2023-10-09T08:47:16.968742+0000","last_became_active":"2023-10-09T08:47:16.968317+0000","last_became_peered":"2023-10-09T08:47:16.968317+0000","last_unstale":"2023-10-09T08:47:16.968742+0000","last_undegraded":"2023-10-09T08:47:16.968742+0000","last_fullsized":"2023-10-09T08:47:16.968742+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962594+0000","last_change":"2023-10-09T08:46:18.677157+0000","last_active":"2023-10-09T08:47:15.962594+0000","last_peered":"2023-10-09T08:47:15.962594+0000","last_clean":"2023-10-09T08:47:15.962594+0000","last_became_active":"2023-10-09T08:46:18.676750+0000","last_became_peered":"2023-10-09T08:46:18.676750+0000","last_unstale":"2023-10-09T08:47:15.962594+0000","last_undegraded":"2023-10-09T08:47:15.962594+0000","last_fullsized":"2023-10-09T08:47:15.962594+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":83,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965309+0000","last_change":"2023-10-09T08:47:16.964981+0000","last_active":"2023-10-09T08:47:16.965309+0000","last_peered":"2023-10-09T08:47:16.965309+0000","last_clean":"2023-10-09T08:47:16.965309+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.965309+0000","last_undegraded":"2023-10-09T08:47:16.965309+0000","last_fullsized":"2023-10-09T08:47:16.965309+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:04.336414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953808+0000","last_change":"2023-10-09T08:45:49.184830+0000","last_active":"2023-10-09T08:47:16.953808+0000","last_peered":"2023-10-09T08:47:16.953808+0000","last_clean":"2023-10-09T08:47:16.953808+0000","last_became_active":"2023-10-09T08:45:49.184434+0000","last_became_peered":"2023-10-09T08:45:49.184434+0000","last_unstale":"2023-10-09T08:47:16.953808+0000","last_undegraded":"2023-10-09T08:47:16.953808+0000","last_fullsized":"2023-10-09T08:47:16.953808+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:41.880170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950161+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:47:15.950161+0000","last_peered":"2023-10-09T08:47:15.950161+0000","last_clean":"2023-10-09T08:47:15.950161+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:47:15.950161+0000","last_undegraded":"2023-10-09T08:47:15.950161+0000","last_fullsized":"2023-10-09T08:47:15.950161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954285+0000","last_change":"2023-10-09T08:46:18.683082+0000","last_active":"2023-10-09T08:47:16.954285+0000","last_peered":"2023-10-09T08:47:16.954285+0000","last_clean":"2023-10-09T08:47:16.954285+0000","last_became_active":"2023-10-09T08:46:18.682657+0000","last_became_peered":"2023-10-09T08:46:18.682657+0000","last_unstale":"2023-10-09T08:47:16.954285+0000","last_undegraded":"2023-10-09T08:47:16.954285+0000","last_fullsized":"2023-10-09T08:47:16.954285+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:45.531585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956212+0000","last_change":"2023-10-09T08:46:49.232591+0000","last_active":"2023-10-09T08:47:15.956212+0000","last_peered":"2023-10-09T08:47:15.956212+0000","last_clean":"2023-10-09T08:47:15.956212+0000","last_became_active":"2023-10-09T08:46:49.232216+0000","last_became_peered":"2023-10-09T08:46:49.232216+0000","last_unstale":"2023-10-09T08:47:15.956212+0000","last_undegraded":"2023-10-09T08:47:15.956212+0000","last_fullsized":"2023-10-09T08:47:15.956212+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.970417+0000","last_change":"2023-10-09T08:47:16.968372+0000","last_active":"2023-10-09T08:47:16.970417+0000","last_peered":"2023-10-09T08:47:16.970417+0000","last_clean":"2023-10-09T08:47:16.970417+0000","last_became_active":"2023-10-09T08:47:16.968024+0000","last_became_peered":"2023-10-09T08:47:16.968024+0000","last_unstale":"2023-10-09T08:47:16.970417+0000","last_undegraded":"2023-10-09T08:47:16.970417+0000","last_fullsized":"2023-10-09T08:47:16.970417+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:07.642684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962677+0000","last_change":"2023-10-09T08:46:18.678094+0000","last_active":"2023-10-09T08:47:16.962677+0000","last_peered":"2023-10-09T08:47:16.962677+0000","last_clean":"2023-10-09T08:47:16.962677+0000","last_became_active":"2023-10-09T08:46:18.677705+0000","last_became_peered":"2023-10-09T08:46:18.677705+0000","last_unstale":"2023-10-09T08:47:16.962677+0000","last_undegraded":"2023-10-09T08:47:16.962677+0000","last_fullsized":"2023-10-09T08:47:16.962677+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:31.647828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974916+0000","last_change":"2023-10-09T08:47:16.974508+0000","last_active":"2023-10-09T08:47:16.974916+0000","last_peered":"2023-10-09T08:47:16.974916+0000","last_clean":"2023-10-09T08:47:16.974916+0000","last_became_active":"2023-10-09T08:47:16.974255+0000","last_became_peered":"2023-10-09T08:47:16.974255+0000","last_unstale":"2023-10-09T08:47:16.974916+0000","last_undegraded":"2023-10-09T08:47:16.974916+0000","last_fullsized":"2023-10-09T08:47:16.974916+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:40.049955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953435+0000","last_change":"2023-10-09T08:45:49.182610+0000","last_active":"2023-10-09T08:47:16.953435+0000","last_peered":"2023-10-09T08:47:16.953435+0000","last_clean":"2023-10-09T08:47:16.953435+0000","last_became_active":"2023-10-09T08:45:49.182013+0000","last_became_peered":"2023-10-09T08:45:49.182013+0000","last_unstale":"2023-10-09T08:47:16.953435+0000","last_undegraded":"2023-10-09T08:47:16.953435+0000","last_fullsized":"2023-10-09T08:47:16.953435+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:44.316894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962365+0000","last_change":"2023-10-09T08:46:49.233112+0000","last_active":"2023-10-09T08:47:16.962365+0000","last_peered":"2023-10-09T08:47:16.962365+0000","last_clean":"2023-10-09T08:47:16.962365+0000","last_became_active":"2023-10-09T08:46:49.232724+0000","last_became_peered":"2023-10-09T08:46:49.232724+0000","last_unstale":"2023-10-09T08:47:16.962365+0000","last_undegraded":"2023-10-09T08:47:16.962365+0000","last_fullsized":"2023-10-09T08:47:16.962365+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.285159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961023+0000","last_change":"2023-10-09T08:45:49.181115+0000","last_active":"2023-10-09T08:47:15.961023+0000","last_peered":"2023-10-09T08:47:15.961023+0000","last_clean":"2023-10-09T08:47:15.961023+0000","last_became_active":"2023-10-09T08:45:49.180736+0000","last_became_peered":"2023-10-09T08:45:49.180736+0000","last_unstale":"2023-10-09T08:47:15.961023+0000","last_undegraded":"2023-10-09T08:47:15.961023+0000","last_fullsized":"2023-10-09T08:47:15.961023+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954452+0000","last_change":"2023-10-09T08:46:49.232279+0000","last_active":"2023-10-09T08:47:16.954452+0000","last_peered":"2023-10-09T08:47:16.954452+0000","last_clean":"2023-10-09T08:47:16.954452+0000","last_became_active":"2023-10-09T08:46:49.231911+0000","last_became_peered":"2023-10-09T08:46:49.231911+0000","last_unstale":"2023-10-09T08:47:16.954452+0000","last_undegraded":"2023-10-09T08:47:16.954452+0000","last_fullsized":"2023-10-09T08:47:16.954452+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:38.497218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965912+0000","last_change":"2023-10-09T08:47:16.965497+0000","last_active":"2023-10-09T08:47:16.965912+0000","last_peered":"2023-10-09T08:47:16.965912+0000","last_clean":"2023-10-09T08:47:16.965912+0000","last_became_active":"2023-10-09T08:47:16.965181+0000","last_became_peered":"2023-10-09T08:47:16.965181+0000","last_unstale":"2023-10-09T08:47:16.965912+0000","last_undegraded":"2023-10-09T08:47:16.965912+0000","last_fullsized":"2023-10-09T08:47:16.965912+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:52.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956282+0000","last_change":"2023-10-09T08:46:18.680004+0000","last_active":"2023-10-09T08:47:15.956282+0000","last_peered":"2023-10-09T08:47:15.956282+0000","last_clean":"2023-10-09T08:47:15.956282+0000","last_became_active":"2023-10-09T08:46:18.679617+0000","last_became_peered":"2023-10-09T08:46:18.679617+0000","last_unstale":"2023-10-09T08:47:15.956282+0000","last_undegraded":"2023-10-09T08:47:15.956282+0000","last_fullsized":"2023-10-09T08:47:15.956282+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956609+0000","last_change":"2023-10-09T08:34:27.860287+0000","last_active":"2023-10-09T08:47:15.956609+0000","last_peered":"2023-10-09T08:47:15.956609+0000","last_clean":"2023-10-09T08:47:15.956609+0000","last_became_active":"2023-10-09T08:34:27.859851+0000","last_became_peered":"2023-10-09T08:34:27.859851+0000","last_unstale":"2023-10-09T08:47:15.956609+0000","last_undegraded":"2023-10-09T08:47:15.956609+0000","last_fullsized":"2023-10-09T08:47:15.956609+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973899+0000","last_change":"2023-10-09T08:47:16.973286+0000","last_active":"2023-10-09T08:47:16.973899+0000","last_peered":"2023-10-09T08:47:16.973899+0000","last_clean":"2023-10-09T08:47:16.973899+0000","last_became_active":"2023-10-09T08:47:16.972839+0000","last_became_peered":"2023-10-09T08:47:16.972839+0000","last_unstale":"2023-10-09T08:47:16.973899+0000","last_undegraded":"2023-10-09T08:47:16.973899+0000","last_fullsized":"2023-10-09T08:47:16.973899+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:56.524537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":296,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956174+0000","last_change":"2023-10-09T08:45:49.183161+0000","last_active":"2023-10-09T08:47:15.956174+0000","last_peered":"2023-10-09T08:47:15.956174+0000","last_clean":"2023-10-09T08:47:15.956174+0000","last_became_active":"2023-10-09T08:45:49.182560+0000","last_became_peered":"2023-10-09T08:45:49.182560+0000","last_unstale":"2023-10-09T08:47:15.956174+0000","last_undegraded":"2023-10-09T08:47:15.956174+0000","last_fullsized":"2023-10-09T08:47:15.956174+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960367+0000","last_change":"2023-10-09T08:46:49.234616+0000","last_active":"2023-10-09T08:47:15.960367+0000","last_peered":"2023-10-09T08:47:15.960367+0000","last_clean":"2023-10-09T08:47:15.960367+0000","last_became_active":"2023-10-09T08:46:49.234217+0000","last_became_peered":"2023-10-09T08:46:49.234217+0000","last_unstale":"2023-10-09T08:47:15.960367+0000","last_undegraded":"2023-10-09T08:47:15.960367+0000","last_fullsized":"2023-10-09T08:47:15.960367+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962773+0000","last_change":"2023-10-09T08:46:18.681375+0000","last_active":"2023-10-09T08:47:16.962773+0000","last_peered":"2023-10-09T08:47:16.962773+0000","last_clean":"2023-10-09T08:47:16.962773+0000","last_became_active":"2023-10-09T08:46:18.680713+0000","last_became_peered":"2023-10-09T08:46:18.680713+0000","last_unstale":"2023-10-09T08:47:16.962773+0000","last_undegraded":"2023-10-09T08:47:16.962773+0000","last_fullsized":"2023-10-09T08:47:16.962773+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:59.671965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963096+0000","last_change":"2023-10-09T08:46:49.231894+0000","last_active":"2023-10-09T08:47:16.963096+0000","last_peered":"2023-10-09T08:47:16.963096+0000","last_clean":"2023-10-09T08:47:16.963096+0000","last_became_active":"2023-10-09T08:46:49.231531+0000","last_became_peered":"2023-10-09T08:46:49.231531+0000","last_unstale":"2023-10-09T08:47:16.963096+0000","last_undegraded":"2023-10-09T08:47:16.963096+0000","last_fullsized":"2023-10-09T08:47:16.963096+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:50.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968243+0000","last_change":"2023-10-09T08:47:16.966880+0000","last_active":"2023-10-09T08:47:16.968243+0000","last_peered":"2023-10-09T08:47:16.968243+0000","last_clean":"2023-10-09T08:47:16.968243+0000","last_became_active":"2023-10-09T08:47:16.966539+0000","last_became_peered":"2023-10-09T08:47:16.966539+0000","last_unstale":"2023-10-09T08:47:16.968243+0000","last_undegraded":"2023-10-09T08:47:16.968243+0000","last_fullsized":"2023-10-09T08:47:16.968243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:55.047737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975638+0000","last_change":"2023-10-09T08:47:16.975638+0000","last_active":"2023-10-09T08:47:16.975638+0000","last_peered":"2023-10-09T08:47:16.975638+0000","last_clean":"2023-10-09T08:47:16.975638+0000","last_became_active":"2023-10-09T08:47:16.975248+0000","last_became_peered":"2023-10-09T08:47:16.975248+0000","last_unstale":"2023-10-09T08:47:16.975638+0000","last_undegraded":"2023-10-09T08:47:16.975638+0000","last_fullsized":"2023-10-09T08:47:16.975638+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950601+0000","last_change":"2023-10-09T08:45:49.184088+0000","last_active":"2023-10-09T08:47:15.950601+0000","last_peered":"2023-10-09T08:47:15.950601+0000","last_clean":"2023-10-09T08:47:15.950601+0000","last_became_active":"2023-10-09T08:45:49.183674+0000","last_became_peered":"2023-10-09T08:45:49.183674+0000","last_unstale":"2023-10-09T08:47:15.950601+0000","last_undegraded":"2023-10-09T08:47:15.950601+0000","last_fullsized":"2023-10-09T08:47:15.950601+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962059+0000","last_change":"2023-10-09T08:46:18.680020+0000","last_active":"2023-10-09T08:47:15.962059+0000","last_peered":"2023-10-09T08:47:15.962059+0000","last_clean":"2023-10-09T08:47:15.962059+0000","last_became_active":"2023-10-09T08:46:18.679652+0000","last_became_peered":"2023-10-09T08:46:18.679652+0000","last_unstale":"2023-10-09T08:47:15.962059+0000","last_undegraded":"2023-10-09T08:47:15.962059+0000","last_fullsized":"2023-10-09T08:47:15.962059+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963009+0000","last_change":"2023-10-09T08:46:18.684501+0000","last_active":"2023-10-09T08:47:16.963009+0000","last_peered":"2023-10-09T08:47:16.963009+0000","last_clean":"2023-10-09T08:47:16.963009+0000","last_became_active":"2023-10-09T08:46:18.684159+0000","last_became_peered":"2023-10-09T08:46:18.684159+0000","last_unstale":"2023-10-09T08:47:16.963009+0000","last_undegraded":"2023-10-09T08:47:16.963009+0000","last_fullsized":"2023-10-09T08:47:16.963009+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:21.270259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.976238+0000","last_change":"2023-10-09T08:47:16.976238+0000","last_active":"2023-10-09T08:47:16.976238+0000","last_peered":"2023-10-09T08:47:16.976238+0000","last_clean":"2023-10-09T08:47:16.976238+0000","last_became_active":"2023-10-09T08:47:16.975840+0000","last_became_peered":"2023-10-09T08:47:16.975840+0000","last_unstale":"2023-10-09T08:47:16.976238+0000","last_undegraded":"2023-10-09T08:47:16.976238+0000","last_fullsized":"2023-10-09T08:47:16.976238+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962626+0000","last_change":"2023-10-09T08:46:18.680373+0000","last_active":"2023-10-09T08:47:16.962626+0000","last_peered":"2023-10-09T08:47:16.962626+0000","last_clean":"2023-10-09T08:47:16.962626+0000","last_became_active":"2023-10-09T08:46:18.679999+0000","last_became_peered":"2023-10-09T08:46:18.679999+0000","last_unstale":"2023-10-09T08:47:16.962626+0000","last_undegraded":"2023-10-09T08:47:16.962626+0000","last_fullsized":"2023-10-09T08:47:16.962626+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:32.374511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965963+0000","last_change":"2023-10-09T08:47:16.965479+0000","last_active":"2023-10-09T08:47:16.965963+0000","last_peered":"2023-10-09T08:47:16.965963+0000","last_clean":"2023-10-09T08:47:16.965963+0000","last_became_active":"2023-10-09T08:47:16.965141+0000","last_became_peered":"2023-10-09T08:47:16.965141+0000","last_unstale":"2023-10-09T08:47:16.965963+0000","last_undegraded":"2023-10-09T08:47:16.965963+0000","last_fullsized":"2023-10-09T08:47:16.965963+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:50.145649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.970436+0000","last_change":"2023-10-09T08:47:16.968443+0000","last_active":"2023-10-09T08:47:16.970436+0000","last_peered":"2023-10-09T08:47:16.970436+0000","last_clean":"2023-10-09T08:47:16.970436+0000","last_became_active":"2023-10-09T08:47:16.967631+0000","last_became_peered":"2023-10-09T08:47:16.967631+0000","last_unstale":"2023-10-09T08:47:16.970436+0000","last_undegraded":"2023-10-09T08:47:16.970436+0000","last_fullsized":"2023-10-09T08:47:16.970436+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:19.412588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951852+0000","last_change":"2023-10-09T08:46:18.676843+0000","last_active":"2023-10-09T08:47:15.951852+0000","last_peered":"2023-10-09T08:47:15.951852+0000","last_clean":"2023-10-09T08:47:15.951852+0000","last_became_active":"2023-10-09T08:46:18.676468+0000","last_became_peered":"2023-10-09T08:46:18.676468+0000","last_unstale":"2023-10-09T08:47:15.951852+0000","last_undegraded":"2023-10-09T08:47:15.951852+0000","last_fullsized":"2023-10-09T08:47:15.951852+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950381+0000","last_change":"2023-10-09T08:46:49.235099+0000","last_active":"2023-10-09T08:47:15.950381+0000","last_peered":"2023-10-09T08:47:15.950381+0000","last_clean":"2023-10-09T08:47:15.950381+0000","last_became_active":"2023-10-09T08:46:49.234136+0000","last_became_peered":"2023-10-09T08:46:49.234136+0000","last_unstale":"2023-10-09T08:47:15.950381+0000","last_undegraded":"2023-10-09T08:47:15.950381+0000","last_fullsized":"2023-10-09T08:47:15.950381+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.952239+0000","last_change":"2023-10-09T08:46:49.234714+0000","last_active":"2023-10-09T08:47:15.952239+0000","last_peered":"2023-10-09T08:47:15.952239+0000","last_clean":"2023-10-09T08:47:15.952239+0000","last_became_active":"2023-10-09T08:46:49.234307+0000","last_became_peered":"2023-10-09T08:46:49.234307+0000","last_unstale":"2023-10-09T08:47:15.952239+0000","last_undegraded":"2023-10-09T08:47:15.952239+0000","last_fullsized":"2023-10-09T08:47:15.952239+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957582+0000","last_change":"2023-10-09T08:46:18.681222+0000","last_active":"2023-10-09T08:47:15.957582+0000","last_peered":"2023-10-09T08:47:15.957582+0000","last_clean":"2023-10-09T08:47:15.957582+0000","last_became_active":"2023-10-09T08:46:18.680710+0000","last_became_peered":"2023-10-09T08:46:18.680710+0000","last_unstale":"2023-10-09T08:47:15.957582+0000","last_undegraded":"2023-10-09T08:47:15.957582+0000","last_fullsized":"2023-10-09T08:47:15.957582+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972109+0000","last_change":"2023-10-09T08:47:16.972109+0000","last_active":"2023-10-09T08:47:16.972109+0000","last_peered":"2023-10-09T08:47:16.972109+0000","last_clean":"2023-10-09T08:47:16.972109+0000","last_became_active":"2023-10-09T08:47:16.971716+0000","last_became_peered":"2023-10-09T08:47:16.971716+0000","last_unstale":"2023-10-09T08:47:16.972109+0000","last_undegraded":"2023-10-09T08:47:16.972109+0000","last_fullsized":"2023-10-09T08:47:16.972109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965045+0000","last_change":"2023-10-09T08:47:16.965045+0000","last_active":"2023-10-09T08:47:16.965045+0000","last_peered":"2023-10-09T08:47:16.965045+0000","last_clean":"2023-10-09T08:47:16.965045+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.965045+0000","last_undegraded":"2023-10-09T08:47:16.965045+0000","last_fullsized":"2023-10-09T08:47:16.965045+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:49.126847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953696+0000","last_change":"2023-10-09T08:46:18.683057+0000","last_active":"2023-10-09T08:47:16.953696+0000","last_peered":"2023-10-09T08:47:16.953696+0000","last_clean":"2023-10-09T08:47:16.953696+0000","last_became_active":"2023-10-09T08:46:18.682672+0000","last_became_peered":"2023-10-09T08:46:18.682672+0000","last_unstale":"2023-10-09T08:47:16.953696+0000","last_undegraded":"2023-10-09T08:47:16.953696+0000","last_fullsized":"2023-10-09T08:47:16.953696+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:42.975534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975514+0000","last_change":"2023-10-09T08:47:16.975514+0000","last_active":"2023-10-09T08:47:16.975514+0000","last_peered":"2023-10-09T08:47:16.975514+0000","last_clean":"2023-10-09T08:47:16.975514+0000","last_became_active":"2023-10-09T08:47:16.975184+0000","last_became_peered":"2023-10-09T08:47:16.975184+0000","last_unstale":"2023-10-09T08:47:16.975514+0000","last_undegraded":"2023-10-09T08:47:16.975514+0000","last_fullsized":"2023-10-09T08:47:16.975514+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957336+0000","last_change":"2023-10-09T08:46:49.231597+0000","last_active":"2023-10-09T08:47:15.957336+0000","last_peered":"2023-10-09T08:47:15.957336+0000","last_clean":"2023-10-09T08:47:15.957336+0000","last_became_active":"2023-10-09T08:46:49.231198+0000","last_became_peered":"2023-10-09T08:46:49.231198+0000","last_unstale":"2023-10-09T08:47:15.957336+0000","last_undegraded":"2023-10-09T08:47:15.957336+0000","last_fullsized":"2023-10-09T08:47:15.957336+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961042+0000","last_change":"2023-10-09T08:34:27.733891+0000","last_active":"2023-10-09T08:47:15.961042+0000","last_peered":"2023-10-09T08:47:15.961042+0000","last_clean":"2023-10-09T08:47:15.961042+0000","last_became_active":"2023-10-09T08:34:27.733379+0000","last_became_peered":"2023-10-09T08:34:27.733379+0000","last_unstale":"2023-10-09T08:47:15.961042+0000","last_undegraded":"2023-10-09T08:47:15.961042+0000","last_fullsized":"2023-10-09T08:47:15.961042+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972571+0000","last_change":"2023-10-09T08:47:16.972571+0000","last_active":"2023-10-09T08:47:16.972571+0000","last_peered":"2023-10-09T08:47:16.972571+0000","last_clean":"2023-10-09T08:47:16.972571+0000","last_became_active":"2023-10-09T08:47:16.972035+0000","last_became_peered":"2023-10-09T08:47:16.972035+0000","last_unstale":"2023-10-09T08:47:16.972571+0000","last_undegraded":"2023-10-09T08:47:16.972571+0000","last_fullsized":"2023-10-09T08:47:16.972571+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957950+0000","last_change":"2023-10-09T08:46:18.674774+0000","last_active":"2023-10-09T08:47:15.957950+0000","last_peered":"2023-10-09T08:47:15.957950+0000","last_clean":"2023-10-09T08:47:15.957950+0000","last_became_active":"2023-10-09T08:46:18.674334+0000","last_became_peered":"2023-10-09T08:46:18.674334+0000","last_unstale":"2023-10-09T08:47:15.957950+0000","last_undegraded":"2023-10-09T08:47:15.957950+0000","last_fullsized":"2023-10-09T08:47:15.957950+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960548+0000","last_change":"2023-10-09T08:34:29.490932+0000","last_active":"2023-10-09T08:47:15.960548+0000","last_peered":"2023-10-09T08:47:15.960548+0000","last_clean":"2023-10-09T08:47:15.960548+0000","last_became_active":"2023-10-09T08:34:29.489841+0000","last_became_peered":"2023-10-09T08:34:29.489841+0000","last_unstale":"2023-10-09T08:47:15.960548+0000","last_undegraded":"2023-10-09T08:47:15.960548+0000","last_fullsized":"2023-10-09T08:47:15.960548+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958369+0000","last_change":"2023-10-09T08:45:49.182558+0000","last_active":"2023-10-09T08:47:15.958369+0000","last_peered":"2023-10-09T08:47:15.958369+0000","last_clean":"2023-10-09T08:47:15.958369+0000","last_became_active":"2023-10-09T08:45:49.182145+0000","last_became_peered":"2023-10-09T08:45:49.182145+0000","last_unstale":"2023-10-09T08:47:15.958369+0000","last_undegraded":"2023-10-09T08:47:15.958369+0000","last_fullsized":"2023-10-09T08:47:15.958369+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974455+0000","last_change":"2023-10-09T08:47:16.974455+0000","last_active":"2023-10-09T08:47:16.974455+0000","last_peered":"2023-10-09T08:47:16.974455+0000","last_clean":"2023-10-09T08:47:16.974455+0000","last_became_active":"2023-10-09T08:47:16.973803+0000","last_became_peered":"2023-10-09T08:47:16.973803+0000","last_unstale":"2023-10-09T08:47:16.974455+0000","last_undegraded":"2023-10-09T08:47:16.974455+0000","last_fullsized":"2023-10-09T08:47:16.974455+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:02.753047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954654+0000","last_change":"2023-10-09T08:45:49.188401+0000","last_active":"2023-10-09T08:47:15.954654+0000","last_peered":"2023-10-09T08:47:15.954654+0000","last_clean":"2023-10-09T08:47:15.954654+0000","last_became_active":"2023-10-09T08:45:49.188014+0000","last_became_peered":"2023-10-09T08:45:49.188014+0000","last_unstale":"2023-10-09T08:47:15.954654+0000","last_undegraded":"2023-10-09T08:47:15.954654+0000","last_fullsized":"2023-10-09T08:47:15.954654+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972320+0000","last_change":"2023-10-09T08:47:16.971847+0000","last_active":"2023-10-09T08:47:16.972320+0000","last_peered":"2023-10-09T08:47:16.972320+0000","last_clean":"2023-10-09T08:47:16.972320+0000","last_became_active":"2023-10-09T08:47:16.970986+0000","last_became_peered":"2023-10-09T08:47:16.970986+0000","last_unstale":"2023-10-09T08:47:16.972320+0000","last_undegraded":"2023-10-09T08:47:16.972320+0000","last_fullsized":"2023-10-09T08:47:16.972320+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:06.084956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954882+0000","last_change":"2023-10-09T08:34:27.726666+0000","last_active":"2023-10-09T08:47:15.954882+0000","last_peered":"2023-10-09T08:47:15.954882+0000","last_clean":"2023-10-09T08:47:15.954882+0000","last_became_active":"2023-10-09T08:34:27.726189+0000","last_became_peered":"2023-10-09T08:34:27.726189+0000","last_unstale":"2023-10-09T08:47:15.954882+0000","last_undegraded":"2023-10-09T08:47:15.954882+0000","last_fullsized":"2023-10-09T08:47:15.954882+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975645+0000","last_change":"2023-10-09T08:47:16.975645+0000","last_active":"2023-10-09T08:47:16.975645+0000","last_peered":"2023-10-09T08:47:16.975645+0000","last_clean":"2023-10-09T08:47:16.975645+0000","last_became_active":"2023-10-09T08:47:16.975277+0000","last_became_peered":"2023-10-09T08:47:16.975277+0000","last_unstale":"2023-10-09T08:47:16.975645+0000","last_undegraded":"2023-10-09T08:47:16.975645+0000","last_fullsized":"2023-10-09T08:47:16.975645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.971694+0000","last_change":"2023-10-09T08:47:16.971357+0000","last_active":"2023-10-09T08:47:16.971694+0000","last_peered":"2023-10-09T08:47:16.971694+0000","last_clean":"2023-10-09T08:47:16.971694+0000","last_became_active":"2023-10-09T08:47:16.971034+0000","last_became_peered":"2023-10-09T08:47:16.971034+0000","last_unstale":"2023-10-09T08:47:16.971694+0000","last_undegraded":"2023-10-09T08:47:16.971694+0000","last_fullsized":"2023-10-09T08:47:16.971694+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:05.892369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974125+0000","last_change":"2023-10-09T08:47:16.974125+0000","last_active":"2023-10-09T08:47:16.974125+0000","last_peered":"2023-10-09T08:47:16.974125+0000","last_clean":"2023-10-09T08:47:16.974125+0000","last_became_active":"2023-10-09T08:47:16.973763+0000","last_became_peered":"2023-10-09T08:47:16.973763+0000","last_unstale":"2023-10-09T08:47:16.974125+0000","last_undegraded":"2023-10-09T08:47:16.974125+0000","last_fullsized":"2023-10-09T08:47:16.974125+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:31.548547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958645+0000","last_change":"2023-10-09T08:46:49.233182+0000","last_active":"2023-10-09T08:47:15.958645+0000","last_peered":"2023-10-09T08:47:15.958645+0000","last_clean":"2023-10-09T08:47:15.958645+0000","last_became_active":"2023-10-09T08:46:49.232746+0000","last_became_peered":"2023-10-09T08:46:49.232746+0000","last_unstale":"2023-10-09T08:47:15.958645+0000","last_undegraded":"2023-10-09T08:47:15.958645+0000","last_fullsized":"2023-10-09T08:47:15.958645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951634+0000","last_change":"2023-10-09T08:46:18.676902+0000","last_active":"2023-10-09T08:47:15.951634+0000","last_peered":"2023-10-09T08:47:15.951634+0000","last_clean":"2023-10-09T08:47:15.951634+0000","last_became_active":"2023-10-09T08:46:18.676560+0000","last_became_peered":"2023-10-09T08:46:18.676560+0000","last_unstale":"2023-10-09T08:47:15.951634+0000","last_undegraded":"2023-10-09T08:47:15.951634+0000","last_fullsized":"2023-10-09T08:47:15.951634+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":68,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951207+0000","last_change":"2023-10-09T08:34:27.730961+0000","last_active":"2023-10-09T08:47:15.951207+0000","last_peered":"2023-10-09T08:47:15.951207+0000","last_clean":"2023-10-09T08:47:15.951207+0000","last_became_active":"2023-10-09T08:34:27.730198+0000","last_became_peered":"2023-10-09T08:34:27.730198+0000","last_unstale":"2023-10-09T08:47:15.951207+0000","last_undegraded":"2023-10-09T08:47:15.951207+0000","last_fullsized":"2023-10-09T08:47:15.951207+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963049+0000","last_change":"2023-10-09T08:46:49.231880+0000","last_active":"2023-10-09T08:47:16.963049+0000","last_peered":"2023-10-09T08:47:16.963049+0000","last_clean":"2023-10-09T08:47:16.963049+0000","last_became_active":"2023-10-09T08:46:49.231471+0000","last_became_peered":"2023-10-09T08:46:49.231471+0000","last_unstale":"2023-10-09T08:47:16.963049+0000","last_undegraded":"2023-10-09T08:47:16.963049+0000","last_fullsized":"2023-10-09T08:47:16.963049+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:31.742745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973891+0000","last_change":"2023-10-09T08:47:16.973891+0000","last_active":"2023-10-09T08:47:16.973891+0000","last_peered":"2023-10-09T08:47:16.973891+0000","last_clean":"2023-10-09T08:47:16.973891+0000","last_became_active":"2023-10-09T08:47:16.973405+0000","last_became_peered":"2023-10-09T08:47:16.973405+0000","last_unstale":"2023-10-09T08:47:16.973891+0000","last_undegraded":"2023-10-09T08:47:16.973891+0000","last_fullsized":"2023-10-09T08:47:16.973891+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972498+0000","last_change":"2023-10-09T08:47:16.972498+0000","last_active":"2023-10-09T08:47:16.972498+0000","last_peered":"2023-10-09T08:47:16.972498+0000","last_clean":"2023-10-09T08:47:16.972498+0000","last_became_active":"2023-10-09T08:47:16.972184+0000","last_became_peered":"2023-10-09T08:47:16.972184+0000","last_unstale":"2023-10-09T08:47:16.972498+0000","last_undegraded":"2023-10-09T08:47:16.972498+0000","last_fullsized":"2023-10-09T08:47:16.972498+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:03.559858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960484+0000","last_change":"2023-10-09T08:46:18.675538+0000","last_active":"2023-10-09T08:47:15.960484+0000","last_peered":"2023-10-09T08:47:15.960484+0000","last_clean":"2023-10-09T08:47:15.960484+0000","last_became_active":"2023-10-09T08:46:18.675149+0000","last_became_peered":"2023-10-09T08:46:18.675149+0000","last_unstale":"2023-10-09T08:47:15.960484+0000","last_undegraded":"2023-10-09T08:47:15.960484+0000","last_fullsized":"2023-10-09T08:47:15.960484+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969948+0000","last_change":"2023-10-09T08:47:16.969948+0000","last_active":"2023-10-09T08:47:16.969948+0000","last_peered":"2023-10-09T08:47:16.969948+0000","last_clean":"2023-10-09T08:47:16.969948+0000","last_became_active":"2023-10-09T08:47:16.969666+0000","last_became_peered":"2023-10-09T08:47:16.969666+0000","last_unstale":"2023-10-09T08:47:16.969948+0000","last_undegraded":"2023-10-09T08:47:16.969948+0000","last_fullsized":"2023-10-09T08:47:16.969948+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:25.089633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972317+0000","last_change":"2023-10-09T08:47:16.971965+0000","last_active":"2023-10-09T08:47:16.972317+0000","last_peered":"2023-10-09T08:47:16.972317+0000","last_clean":"2023-10-09T08:47:16.972317+0000","last_became_active":"2023-10-09T08:47:16.971651+0000","last_became_peered":"2023-10-09T08:47:16.971651+0000","last_unstale":"2023-10-09T08:47:16.972317+0000","last_undegraded":"2023-10-09T08:47:16.972317+0000","last_fullsized":"2023-10-09T08:47:16.972317+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:04.259603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956192+0000","last_change":"2023-10-09T08:46:18.680365+0000","last_active":"2023-10-09T08:47:15.956192+0000","last_peered":"2023-10-09T08:47:15.956192+0000","last_clean":"2023-10-09T08:47:15.956192+0000","last_became_active":"2023-10-09T08:46:18.679875+0000","last_became_peered":"2023-10-09T08:46:18.679875+0000","last_unstale":"2023-10-09T08:47:15.956192+0000","last_undegraded":"2023-10-09T08:47:15.956192+0000","last_fullsized":"2023-10-09T08:47:15.956192+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965460+0000","last_change":"2023-10-09T08:47:16.965460+0000","last_active":"2023-10-09T08:47:16.965460+0000","last_peered":"2023-10-09T08:47:16.965460+0000","last_clean":"2023-10-09T08:47:16.965460+0000","last_became_active":"2023-10-09T08:47:16.965040+0000","last_became_peered":"2023-10-09T08:47:16.965040+0000","last_unstale":"2023-10-09T08:47:16.965460+0000","last_undegraded":"2023-10-09T08:47:16.965460+0000","last_fullsized":"2023-10-09T08:47:16.965460+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.025735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972434+0000","last_change":"2023-10-09T08:47:16.972106+0000","last_active":"2023-10-09T08:47:16.972434+0000","last_peered":"2023-10-09T08:47:16.972434+0000","last_clean":"2023-10-09T08:47:16.972434+0000","last_became_active":"2023-10-09T08:47:16.971511+0000","last_became_peered":"2023-10-09T08:47:16.971511+0000","last_unstale":"2023-10-09T08:47:16.972434+0000","last_undegraded":"2023-10-09T08:47:16.972434+0000","last_fullsized":"2023-10-09T08:47:16.972434+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:12.674597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969696+0000","last_change":"2023-10-09T08:47:16.968800+0000","last_active":"2023-10-09T08:47:16.969696+0000","last_peered":"2023-10-09T08:47:16.969696+0000","last_clean":"2023-10-09T08:47:16.969696+0000","last_became_active":"2023-10-09T08:47:16.968430+0000","last_became_peered":"2023-10-09T08:47:16.968430+0000","last_unstale":"2023-10-09T08:47:16.969696+0000","last_undegraded":"2023-10-09T08:47:16.969696+0000","last_fullsized":"2023-10-09T08:47:16.969696+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:08.011498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965051+0000","last_change":"2023-10-09T08:47:16.965051+0000","last_active":"2023-10-09T08:47:16.965051+0000","last_peered":"2023-10-09T08:47:16.965051+0000","last_clean":"2023-10-09T08:47:16.965051+0000","last_became_active":"2023-10-09T08:47:16.964664+0000","last_became_peered":"2023-10-09T08:47:16.964664+0000","last_unstale":"2023-10-09T08:47:16.965051+0000","last_undegraded":"2023-10-09T08:47:16.965051+0000","last_fullsized":"2023-10-09T08:47:16.965051+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.377202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950228+0000","last_change":"2023-10-09T08:46:49.233483+0000","last_active":"2023-10-09T08:47:15.950228+0000","last_peered":"2023-10-09T08:47:15.950228+0000","last_clean":"2023-10-09T08:47:15.950228+0000","last_became_active":"2023-10-09T08:46:49.233033+0000","last_became_peered":"2023-10-09T08:46:49.233033+0000","last_unstale":"2023-10-09T08:47:15.950228+0000","last_undegraded":"2023-10-09T08:47:15.950228+0000","last_fullsized":"2023-10-09T08:47:15.950228+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430427,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310904,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434248,"statfs":{"total":95995035648,"available":95676669952,"internally_reserved":0,"allocated":1556480,"data_stored":1102308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.92900000000000005,"15min":0.92200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.2569999999999999,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.046},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.94199999999999995,"15min":0.94499999999999995},"min":{"1min":0.64300000000000002,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1679999999999999,"5min":1.262,"15min":1.262},"last":0.94499999999999995}]},{"osd":1,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.3,"5min":1.446,"15min":1.446},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97999999999999998,"15min":0.97499999999999998},"min":{"1min":0.64200000000000002,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.2589999999999999,"5min":1.319,"15min":1.319},"last":0.86799999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.91400000000000003,"15min":0.92400000000000004},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.125,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.92700000000000005,"15min":0.93000000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.84699999999999998}]},{"osd":3,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91800000000000004,"15min":0.92000000000000004},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.131,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0089999999999999,"15min":1.012},"min":{"1min":0.76100000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.28,"5min":1.335,"15min":1.335},"last":1.1399999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.88500000000000001,"15min":0.88900000000000001},"min":{"1min":0.59099999999999997,"5min":0.53200000000000003,"15min":0.40300000000000002},"max":{"1min":1.202,"5min":1.2809999999999999,"15min":1.4630000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.03,"5min":0.98399999999999999,"15min":0.94799999999999995},"min":{"1min":0.67800000000000005,"5min":0.54800000000000004,"15min":0.40000000000000002},"max":{"1min":1.351,"5min":1.488,"15min":1.488},"last":0.91600000000000004}]},{"osd":5,"last update":"Mon Oct 9 08:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.97399999999999998,"15min":0.99299999999999999},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.501},"max":{"1min":1.23,"5min":1.3129999999999999,"15min":1.385},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.68200000000000005,"5min":0.60899999999999999,"15min":0.46100000000000002},"max":{"1min":1.2310000000000001,"5min":1.3400000000000001,"15min":1.794},"last":0.81200000000000006}]},{"osd":6,"last update":"Mon Oct 9 08:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.038,"15min":1.038},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.44},"max":{"1min":1.3180000000000001,"5min":1.4350000000000001,"15min":1.7170000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.117,"5min":1.101,"15min":1.073},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.45000000000000001},"max":{"1min":1.4330000000000001,"5min":1.528,"15min":1.528},"last":1.097}]}]},{"osd":6,"up_from":42,"seq":180388626660,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1508,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1544192,"data_stored":1102112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:58 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.93500000000000005,"15min":0.93000000000000005},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2609999999999999,"5min":1.349,"15min":1.349},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0129999999999999,"15min":0.996},"min":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3320000000000001,"5min":1.5,"15min":1.5},"last":0.93799999999999994}]},{"osd":1,"last update":"Mon Oct 9 08:49:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.90000000000000002,"15min":0.89800000000000002},"min":{"1min":0.65100000000000002,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.8720000000000001,"5min":1.8720000000000001,"15min":1.8720000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.95699999999999996,"15min":0.95999999999999996},"min":{"1min":0.67100000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2649999999999999,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.093}]},{"osd":2,"last update":"Mon Oct 9 08:48:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.91400000000000003,"15min":0.91600000000000004},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.94899999999999995,"15min":0.93999999999999995},"min":{"1min":0.76500000000000001,"5min":0.625,"15min":0.625},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.69299999999999995}]},{"osd":3,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.91500000000000004,"15min":0.91100000000000003},"min":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.94499999999999995,"15min":0.94699999999999995},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.099,"5min":1.167,"15min":1.167},"last":1.0169999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:49:12 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.89900000000000002,"15min":0.90800000000000003},"min":{"1min":0.55300000000000005,"5min":0.50700000000000001,"15min":0.379},"max":{"1min":1.381,"5min":1.381,"15min":1.629},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.022,"5min":0.93300000000000005,"15min":0.95499999999999996},"min":{"1min":0.66400000000000003,"5min":0.436,"15min":0.436},"max":{"1min":1.7250000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.88200000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:49:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.004,"15min":0.98999999999999999},"min":{"1min":0.65800000000000003,"5min":0.495,"15min":0.495},"max":{"1min":1.464,"5min":1.464,"15min":1.7410000000000001},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0309999999999999,"15min":1.048},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.51600000000000001},"max":{"1min":1.825,"5min":1.825,"15min":4.4420000000000002},"last":1.163}]},{"osd":7,"last update":"Mon Oct 9 08:48:58 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.052,"15min":1.056},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.45900000000000002},"max":{"1min":1.3460000000000001,"5min":1.383,"15min":1.8220000000000001},"last":1.036},{"interface":"front","average":{"1min":1.073,"5min":1.0569999999999999,"15min":1.085},"min":{"1min":0.80000000000000004,"5min":0.53300000000000003,"15min":0.52500000000000002},"max":{"1min":1.4119999999999999,"5min":1.556,"15min":2.262},"last":0.94999999999999996}]}]},{"osd":1,"up_from":83,"seq":356482285608,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32396,"kb_used_data":964,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712756,"statfs":{"total":95995035648,"available":95961862144,"internally_reserved":0,"allocated":987136,"data_stored":516401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.63400000000000001,"15min":0.628},"min":{"1min":0.39300000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68600000000000005,"15min":0.67600000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.47799999999999998}]},{"osd":2,"last update":"Mon Oct 9 08:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.72299999999999998,"15min":0.73899999999999999},"min":{"1min":0.41199999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80900000000000005,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.69999999999999996,"15min":0.70399999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.628}]},{"osd":3,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.84799999999999998,"15min":0.85499999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.498}]},{"osd":4,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.97099999999999997,"15min":0.96599999999999997},"min":{"1min":0.70499999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.98499999999999999,"15min":0.97399999999999998},"min":{"1min":0.68000000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3,"5min":1.577,"15min":1.577},"last":0.94799999999999995}]},{"osd":5,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.0569999999999999,"15min":1.0640000000000001},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.04,"5min":1.0549999999999999,"15min":1.0649999999999999},"min":{"1min":0.745,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":0.79200000000000004}]},{"osd":6,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.095,"15min":1.087},"min":{"1min":0.72599999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.4319999999999999,"5min":1.522,"15min":1.522},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.121,"5min":1.115,"15min":1.111},"min":{"1min":0.755,"5min":0.754,"15min":0.754},"max":{"1min":1.383,"5min":1.577,"15min":1.577},"last":1.036}]},{"osd":7,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0549999999999999,"15min":1.042},"min":{"1min":0.73299999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.097,"5min":1.107,"15min":1.107},"min":{"1min":0.79600000000000004,"5min":0.624,"15min":0.624},"max":{"1min":1.4039999999999999,"5min":1.625,"15min":1.625},"last":0.81499999999999995}]}]},{"osd":0,"up_from":78,"seq":335007449134,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33060,"kb_used_data":1564,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712092,"statfs":{"total":95995035648,"available":95961182208,"internally_reserved":0,"allocated":1601536,"data_stored":1109136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.71099999999999997,"15min":0.71599999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.0780000000000001,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.76300000000000001,"15min":0.77900000000000003},"min":{"1min":0.44500000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.0489999999999999,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.78600000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.83199999999999996,"15min":0.84799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.083,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.92900000000000005,"15min":0.95099999999999996},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.90000000000000002}]},{"osd":3,"last update":"Mon Oct 9 08:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.79200000000000004,"15min":0.79800000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2130000000000001,"5min":1.306,"15min":1.306},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.82799999999999996,"15min":0.82999999999999996},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2250000000000001,"5min":1.321,"15min":1.321},"last":1.0389999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0389999999999999,"15min":1.046},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":1.1659999999999999},{"interface":"front","average":{"1min":0.873,"5min":0.93700000000000006,"15min":0.96199999999999997},"min":{"1min":0.60799999999999998,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.0469999999999999,"5min":1.528,"15min":1.528},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0189999999999999,"15min":1.018},"min":{"1min":0.63500000000000001,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.361,"5min":1.51,"15min":1.51},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0980000000000001,"15min":1.115},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.1080000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.1359999999999999,"15min":1.1599999999999999},"min":{"1min":0.77200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3959999999999999,"5min":1.663,"15min":1.663},"last":1.069},{"interface":"front","average":{"1min":1.071,"5min":1.1950000000000001,"15min":1.2150000000000001},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.486,"5min":1.609,"15min":1.609},"last":1.319}]},{"osd":7,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.214,"15min":1.2330000000000001},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.5880000000000001,"5min":1.679,"15min":1.679},"last":1.2689999999999999},{"interface":"front","average":{"1min":1.028,"5min":1.117,"15min":1.1479999999999999},"min":{"1min":0.64600000000000002,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.3779999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.01}]}]},{"osd":2,"up_from":88,"seq":377957122082,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36616,"kb_used_data":960,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708536,"statfs":{"total":95995035648,"available":95957540864,"internally_reserved":0,"allocated":983040,"data_stored":514873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.70999999999999996,"15min":0.70199999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.79300000000000004,"15min":0.81000000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2629999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.002}]},{"osd":1,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.0009999999999999,"5min":1.127,"15min":1.127},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.755,"5min":0.82899999999999996,"15min":0.84099999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.165,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.72799999999999998}]},{"osd":3,"last update":"Mon Oct 9 08:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.83399999999999996,"15min":0.82699999999999996},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.80700000000000005,"15min":0.80900000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.67500000000000004}]},{"osd":4,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.96599999999999997,"15min":0.97199999999999998},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.1919999999999999,"5min":1.24,"15min":1.24},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0469999999999999,"15min":1.0580000000000001},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.78500000000000003}]},{"osd":5,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.131,"15min":1.1579999999999999},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.464,"5min":1.49,"15min":1.49},"last":1.0660000000000001},{"interface":"front","average":{"1min":1,"5min":1.0649999999999999,"15min":1.0760000000000001},"min":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"max":{"1min":1.2230000000000001,"5min":1.337,"15min":1.337},"last":0.86899999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.1259999999999999,"15min":1.137},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.3100000000000001,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.1870000000000001,"15min":1.204},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.446,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.1779999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.1539999999999999,"15min":1.1599999999999999},"min":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"max":{"1min":1.4099999999999999,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.087,"5min":1.214,"15min":1.2350000000000001},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.5169999999999999,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.238}]}]},{"osd":3,"up_from":93,"seq":399431958556,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28276,"kb_used_data":940,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716876,"statfs":{"total":95995035648,"available":95966081024,"internally_reserved":0,"allocated":962560,"data_stored":511788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.71499999999999997,"15min":0.72699999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.93400000000000005,"5min":1.198,"15min":1.198},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.752,"5min":0.78600000000000003,"15min":0.79200000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1339999999999999,"5min":1.21,"15min":1.21},"last":0.96699999999999997}]},{"osd":1,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.77800000000000002,"15min":0.78600000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.80800000000000005,"15min":0.81000000000000005},"min":{"1min":0.51300000000000001,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.0649999999999999,"5min":1.111,"15min":1.111},"last":1.1200000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.76200000000000001,"15min":0.76900000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.80500000000000005,"15min":0.81100000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.0029999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":1.284}]},{"osd":4,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.012,"15min":1.016},"min":{"1min":0.79500000000000004,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.2150000000000001,"5min":1.321,"15min":1.321},"last":1.4179999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.0169999999999999,"15min":1.0309999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.2649999999999999,"5min":1.411,"15min":1.411},"last":1.3839999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.016,"15min":1.028},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.2689999999999999,"5min":1.306,"15min":1.306},"last":1.242},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0640000000000001,"15min":1.073},"min":{"1min":0.80200000000000005,"5min":0.79900000000000004,"15min":0.79900000000000004},"max":{"1min":1.429,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.3400000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.1080000000000001,"15min":1.1220000000000001},"min":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"max":{"1min":1.373,"5min":1.373,"15min":1.373},"last":1.601},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.1819999999999999,"15min":1.1970000000000001},"min":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"max":{"1min":1.3160000000000001,"5min":1.482,"15min":1.482},"last":1.51}]},{"osd":7,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.1439999999999999,"15min":1.1559999999999999},"min":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.351,"5min":1.52,"15min":1.52},"last":1.4490000000000001},{"interface":"front","average":{"1min":1.123,"5min":1.1579999999999999,"15min":1.1639999999999999},"min":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"max":{"1min":1.399,"5min":1.581,"15min":1.581},"last":1.4770000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019126,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302132,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443020,"statfs":{"total":95995035648,"available":95685652480,"internally_reserved":0,"allocated":962560,"data_stored":512135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:49:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.88100000000000001,"15min":0.879},"min":{"1min":0.61699999999999999,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.238,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.89900000000000002,"15min":0.89600000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":1.0069999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:49:22 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.85599999999999998,"15min":0.85199999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.256,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.97399999999999998,"15min":0.96299999999999997},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.234,"5min":1.321,"15min":1.321},"last":1.034}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93799999999999994,"15min":0.93600000000000005},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.03,"5min":0.96599999999999997,"15min":0.95599999999999996},"min":{"1min":0.80200000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.284}]},{"osd":3,"last update":"Mon Oct 9 08:49:17 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.032,"15min":1.032},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":1.2050000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:48:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.89500000000000002,"15min":0.91400000000000003},"min":{"1min":0.73799999999999999,"5min":0.51200000000000001,"15min":0.39200000000000002},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.111,"5min":0.99199999999999999,"15min":0.99399999999999999},"min":{"1min":0.96999999999999997,"5min":0.48399999999999999,"15min":0.47399999999999998},"max":{"1min":1.389,"5min":1.389,"15min":1.4730000000000001},"last":0.84199999999999997}]},{"osd":6,"last update":"Mon Oct 9 08:49:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.99399999999999999,"15min":0.96199999999999997},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.42899999999999999},"max":{"1min":1.948,"5min":1.948,"15min":1.948},"last":0.92600000000000005},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.0669999999999999,"15min":1.056},"min":{"1min":0.66400000000000003,"5min":0.57599999999999996,"15min":0.48399999999999999},"max":{"1min":1.478,"5min":1.478,"15min":1.478},"last":0.94999999999999996}]},{"osd":7,"last update":"Mon Oct 9 08:49:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0860000000000001,"15min":1.0820000000000001},"min":{"1min":0.81100000000000005,"5min":0.55100000000000005,"15min":0.44800000000000001},"max":{"1min":1.484,"5min":1.484,"15min":1.484},"last":1.087},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.121,"15min":1.0860000000000001},"min":{"1min":0.89800000000000002,"5min":0.71899999999999997,"15min":0.38},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":1.0589999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822894,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310308,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434844,"statfs":{"total":95995035648,"available":95677280256,"internally_reserved":0,"allocated":946176,"data_stored":507879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.876,"15min":0.876},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.1419999999999999,"5min":1.262,"15min":1.262},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.91100000000000003,"15min":0.90600000000000003},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.133,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.0800000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.89800000000000002,"15min":0.90700000000000003},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.1379999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.90500000000000003,"15min":0.91200000000000003},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.113,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.70799999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.92000000000000004,"15min":0.92700000000000005},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1639999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":1.034},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.98999999999999999,"15min":0.998},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.284,"5min":1.323,"15min":1.323},"last":0.82599999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.998,"15min":1.0029999999999999},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.177,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.995},{"interface":"front","average":{"1min":0.878,"5min":0.93799999999999994,"15min":0.94799999999999995},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.0720000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.96299999999999997}]},{"osd":4,"last update":"Mon Oct 9 08:48:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.878,"15min":0.85999999999999999},"min":{"1min":0.68600000000000005,"5min":0.61299999999999999,"15min":0.46999999999999997},"max":{"1min":1.1890000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.751},{"interface":"front","average":{"1min":1.01,"5min":0.95299999999999996,"15min":0.95099999999999996},"min":{"1min":0.60999999999999999,"5min":0.505,"15min":0.442},"max":{"1min":1.212,"5min":1.405,"15min":1.405},"last":0.91900000000000004}]},{"osd":6,"last update":"Mon Oct 9 08:49:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.93799999999999994,"15min":0.96399999999999997},"min":{"1min":0.68000000000000005,"5min":0.55900000000000005,"15min":0.45200000000000001},"max":{"1min":1.1619999999999999,"5min":1.333,"15min":1.411},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.038,"15min":1.0249999999999999},"min":{"1min":0.70899999999999996,"5min":0.69099999999999995,"15min":0.40400000000000003},"max":{"1min":1.397,"5min":1.4750000000000001,"15min":3.2919999999999998},"last":1.123}]},{"osd":7,"last update":"Mon Oct 9 08:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.0820000000000001,"15min":1.05},"min":{"1min":0.67300000000000004,"5min":0.53300000000000003,"15min":0.36899999999999999},"max":{"1min":1.3600000000000001,"5min":1.4119999999999999,"15min":1.4119999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.071,"15min":1.0780000000000001},"min":{"1min":0.77100000000000002,"5min":0.66300000000000003,"15min":0.45000000000000001},"max":{"1min":1.274,"5min":1.4950000000000001,"15min":1.782},"last":0.86799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:49:28.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:49:30.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:29 smithi094 bash[59677]: cluster 2023-10-09T08:49:29.292659+0000 mgr.y (mgr.25421) 349 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:30.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:29 smithi016 bash[75179]: cluster 2023-10-09T08:49:29.292659+0000 mgr.y (mgr.25421) 349 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:29 smithi016 bash[77892]: cluster 2023-10-09T08:49:29.292659+0000 mgr.y (mgr.25421) 349 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:31 smithi094 bash[59677]: cluster 2023-10-09T08:49:31.293638+0000 mgr.y (mgr.25421) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:31.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:31 smithi016 bash[75179]: cluster 2023-10-09T08:49:31.293638+0000 mgr.y (mgr.25421) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:31.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:31 smithi016 bash[77892]: cluster 2023-10-09T08:49:31.293638+0000 mgr.y (mgr.25421) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:33.745 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:33 smithi094 bash[59677]: cluster 2023-10-09T08:49:33.294484+0000 mgr.y (mgr.25421) 351 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:33.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:33 smithi016 bash[75179]: cluster 2023-10-09T08:49:33.294484+0000 mgr.y (mgr.25421) 351 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:33 smithi016 bash[77892]: cluster 2023-10-09T08:49:33.294484+0000 mgr.y (mgr.25421) 351 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:33.784 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:49:35.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:34 smithi016 bash[75179]: audit 2023-10-09T08:49:34.008796+0000 mon.a (mon.0) 481 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:49:35.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:34 smithi016 bash[75179]: audit 2023-10-09T08:49:34.205018+0000 mon.a (mon.0) 482 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:35.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:34 smithi016 bash[77892]: audit 2023-10-09T08:49:34.008796+0000 mon.a (mon.0) 481 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:49:35.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:34 smithi016 bash[77892]: audit 2023-10-09T08:49:34.205018+0000 mon.a (mon.0) 482 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:34 smithi094 bash[59677]: audit 2023-10-09T08:49:34.008796+0000 mon.a (mon.0) 481 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:49:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:34 smithi094 bash[59677]: audit 2023-10-09T08:49:34.205018+0000 mon.a (mon.0) 482 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:36.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:49:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:49:35] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-10-09T08:49:36.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:35 smithi016 bash[77892]: cluster 2023-10-09T08:49:35.295330+0000 mgr.y (mgr.25421) 352 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:36.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:35 smithi016 bash[75179]: cluster 2023-10-09T08:49:35.295330+0000 mgr.y (mgr.25421) 352 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:36.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:35 smithi094 bash[59677]: cluster 2023-10-09T08:49:35.295330+0000 mgr.y (mgr.25421) 352 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:37.341 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:49:37.344 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:49:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:37 smithi094 bash[59677]: cluster 2023-10-09T08:49:37.296094+0000 mgr.y (mgr.25421) 353 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:37.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:37 smithi094 bash[59677]: audit 2023-10-09T08:49:37.319814+0000 mgr.y (mgr.25421) 354 : audit 0 from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:37.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:37 smithi016 bash[77892]: cluster 2023-10-09T08:49:37.296094+0000 mgr.y (mgr.25421) 353 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:37.781 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:37 smithi016 bash[77892]: audit 2023-10-09T08:49:37.319814+0000 mgr.y (mgr.25421) 354 : audit 0 from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:37.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:37 smithi016 bash[75179]: cluster 2023-10-09T08:49:37.296094+0000 mgr.y (mgr.25421) 353 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:37.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:37 smithi016 bash[75179]: audit 2023-10-09T08:49:37.319814+0000 mgr.y (mgr.25421) 354 : audit 0 from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:49:38.332 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":287,"stamp":"2023-10-09T08:49:37.295573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1352296,"kb_used_data":9320,"kb_used_omap":126,"kb_used_meta":1342785,"kb_avail":748608920,"statfs":{"total":767960285184,"available":766575534080,"internally_reserved":0,"allocated":9543680,"data_stored":5876632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1375012475},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004936"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950127+0000","last_change":"2023-10-09T08:46:18.676602+0000","last_active":"2023-10-09T08:47:15.950127+0000","last_peered":"2023-10-09T08:47:15.950127+0000","last_clean":"2023-10-09T08:47:15.950127+0000","last_became_active":"2023-10-09T08:46:18.676122+0000","last_became_peered":"2023-10-09T08:46:18.676122+0000","last_unstale":"2023-10-09T08:47:15.950127+0000","last_undegraded":"2023-10-09T08:47:15.950127+0000","last_fullsized":"2023-10-09T08:47:15.950127+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953679+0000","last_change":"2023-10-09T08:46:49.233211+0000","last_active":"2023-10-09T08:47:16.953679+0000","last_peered":"2023-10-09T08:47:16.953679+0000","last_clean":"2023-10-09T08:47:16.953679+0000","last_became_active":"2023-10-09T08:46:49.232775+0000","last_became_peered":"2023-10-09T08:46:49.232775+0000","last_unstale":"2023-10-09T08:47:16.953679+0000","last_undegraded":"2023-10-09T08:47:16.953679+0000","last_fullsized":"2023-10-09T08:47:16.953679+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:59.423857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.966115+0000","last_change":"2023-10-09T08:47:16.966115+0000","last_active":"2023-10-09T08:47:16.966115+0000","last_peered":"2023-10-09T08:47:16.966115+0000","last_clean":"2023-10-09T08:47:16.966115+0000","last_became_active":"2023-10-09T08:47:16.965795+0000","last_became_peered":"2023-10-09T08:47:16.965795+0000","last_unstale":"2023-10-09T08:47:16.966115+0000","last_undegraded":"2023-10-09T08:47:16.966115+0000","last_fullsized":"2023-10-09T08:47:16.966115+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:40.817840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":69,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972982+0000","last_change":"2023-10-09T08:47:16.970132+0000","last_active":"2023-10-09T08:47:16.972982+0000","last_peered":"2023-10-09T08:47:16.972982+0000","last_clean":"2023-10-09T08:47:16.972982+0000","last_became_active":"2023-10-09T08:47:16.969760+0000","last_became_peered":"2023-10-09T08:47:16.969760+0000","last_unstale":"2023-10-09T08:47:16.972982+0000","last_undegraded":"2023-10-09T08:47:16.972982+0000","last_fullsized":"2023-10-09T08:47:16.972982+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.801830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969367+0000","last_change":"2023-10-09T08:47:16.969367+0000","last_active":"2023-10-09T08:47:16.969367+0000","last_peered":"2023-10-09T08:47:16.969367+0000","last_clean":"2023-10-09T08:47:16.969367+0000","last_became_active":"2023-10-09T08:47:16.969000+0000","last_became_peered":"2023-10-09T08:47:16.969000+0000","last_unstale":"2023-10-09T08:47:16.969367+0000","last_undegraded":"2023-10-09T08:47:16.969367+0000","last_fullsized":"2023-10-09T08:47:16.969367+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961355+0000","last_change":"2023-10-09T08:34:27.739113+0000","last_active":"2023-10-09T08:47:15.961355+0000","last_peered":"2023-10-09T08:47:15.961355+0000","last_clean":"2023-10-09T08:47:15.961355+0000","last_became_active":"2023-10-09T08:34:27.738645+0000","last_became_peered":"2023-10-09T08:34:27.738645+0000","last_unstale":"2023-10-09T08:47:15.961355+0000","last_undegraded":"2023-10-09T08:47:15.961355+0000","last_fullsized":"2023-10-09T08:47:15.961355+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968742+0000","last_change":"2023-10-09T08:47:16.968742+0000","last_active":"2023-10-09T08:47:16.968742+0000","last_peered":"2023-10-09T08:47:16.968742+0000","last_clean":"2023-10-09T08:47:16.968742+0000","last_became_active":"2023-10-09T08:47:16.968317+0000","last_became_peered":"2023-10-09T08:47:16.968317+0000","last_unstale":"2023-10-09T08:47:16.968742+0000","last_undegraded":"2023-10-09T08:47:16.968742+0000","last_fullsized":"2023-10-09T08:47:16.968742+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962594+0000","last_change":"2023-10-09T08:46:18.677157+0000","last_active":"2023-10-09T08:47:15.962594+0000","last_peered":"2023-10-09T08:47:15.962594+0000","last_clean":"2023-10-09T08:47:15.962594+0000","last_became_active":"2023-10-09T08:46:18.676750+0000","last_became_peered":"2023-10-09T08:46:18.676750+0000","last_unstale":"2023-10-09T08:47:15.962594+0000","last_undegraded":"2023-10-09T08:47:15.962594+0000","last_fullsized":"2023-10-09T08:47:15.962594+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":83,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965309+0000","last_change":"2023-10-09T08:47:16.964981+0000","last_active":"2023-10-09T08:47:16.965309+0000","last_peered":"2023-10-09T08:47:16.965309+0000","last_clean":"2023-10-09T08:47:16.965309+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.965309+0000","last_undegraded":"2023-10-09T08:47:16.965309+0000","last_fullsized":"2023-10-09T08:47:16.965309+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:04.336414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953808+0000","last_change":"2023-10-09T08:45:49.184830+0000","last_active":"2023-10-09T08:47:16.953808+0000","last_peered":"2023-10-09T08:47:16.953808+0000","last_clean":"2023-10-09T08:47:16.953808+0000","last_became_active":"2023-10-09T08:45:49.184434+0000","last_became_peered":"2023-10-09T08:45:49.184434+0000","last_unstale":"2023-10-09T08:47:16.953808+0000","last_undegraded":"2023-10-09T08:47:16.953808+0000","last_fullsized":"2023-10-09T08:47:16.953808+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:41.880170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950161+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:47:15.950161+0000","last_peered":"2023-10-09T08:47:15.950161+0000","last_clean":"2023-10-09T08:47:15.950161+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:47:15.950161+0000","last_undegraded":"2023-10-09T08:47:15.950161+0000","last_fullsized":"2023-10-09T08:47:15.950161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954285+0000","last_change":"2023-10-09T08:46:18.683082+0000","last_active":"2023-10-09T08:47:16.954285+0000","last_peered":"2023-10-09T08:47:16.954285+0000","last_clean":"2023-10-09T08:47:16.954285+0000","last_became_active":"2023-10-09T08:46:18.682657+0000","last_became_peered":"2023-10-09T08:46:18.682657+0000","last_unstale":"2023-10-09T08:47:16.954285+0000","last_undegraded":"2023-10-09T08:47:16.954285+0000","last_fullsized":"2023-10-09T08:47:16.954285+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:45.531585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956212+0000","last_change":"2023-10-09T08:46:49.232591+0000","last_active":"2023-10-09T08:47:15.956212+0000","last_peered":"2023-10-09T08:47:15.956212+0000","last_clean":"2023-10-09T08:47:15.956212+0000","last_became_active":"2023-10-09T08:46:49.232216+0000","last_became_peered":"2023-10-09T08:46:49.232216+0000","last_unstale":"2023-10-09T08:47:15.956212+0000","last_undegraded":"2023-10-09T08:47:15.956212+0000","last_fullsized":"2023-10-09T08:47:15.956212+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.970417+0000","last_change":"2023-10-09T08:47:16.968372+0000","last_active":"2023-10-09T08:47:16.970417+0000","last_peered":"2023-10-09T08:47:16.970417+0000","last_clean":"2023-10-09T08:47:16.970417+0000","last_became_active":"2023-10-09T08:47:16.968024+0000","last_became_peered":"2023-10-09T08:47:16.968024+0000","last_unstale":"2023-10-09T08:47:16.970417+0000","last_undegraded":"2023-10-09T08:47:16.970417+0000","last_fullsized":"2023-10-09T08:47:16.970417+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:07.642684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962677+0000","last_change":"2023-10-09T08:46:18.678094+0000","last_active":"2023-10-09T08:47:16.962677+0000","last_peered":"2023-10-09T08:47:16.962677+0000","last_clean":"2023-10-09T08:47:16.962677+0000","last_became_active":"2023-10-09T08:46:18.677705+0000","last_became_peered":"2023-10-09T08:46:18.677705+0000","last_unstale":"2023-10-09T08:47:16.962677+0000","last_undegraded":"2023-10-09T08:47:16.962677+0000","last_fullsized":"2023-10-09T08:47:16.962677+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:31.647828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974916+0000","last_change":"2023-10-09T08:47:16.974508+0000","last_active":"2023-10-09T08:47:16.974916+0000","last_peered":"2023-10-09T08:47:16.974916+0000","last_clean":"2023-10-09T08:47:16.974916+0000","last_became_active":"2023-10-09T08:47:16.974255+0000","last_became_peered":"2023-10-09T08:47:16.974255+0000","last_unstale":"2023-10-09T08:47:16.974916+0000","last_undegraded":"2023-10-09T08:47:16.974916+0000","last_fullsized":"2023-10-09T08:47:16.974916+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:40.049955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953435+0000","last_change":"2023-10-09T08:45:49.182610+0000","last_active":"2023-10-09T08:47:16.953435+0000","last_peered":"2023-10-09T08:47:16.953435+0000","last_clean":"2023-10-09T08:47:16.953435+0000","last_became_active":"2023-10-09T08:45:49.182013+0000","last_became_peered":"2023-10-09T08:45:49.182013+0000","last_unstale":"2023-10-09T08:47:16.953435+0000","last_undegraded":"2023-10-09T08:47:16.953435+0000","last_fullsized":"2023-10-09T08:47:16.953435+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:44.316894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962365+0000","last_change":"2023-10-09T08:46:49.233112+0000","last_active":"2023-10-09T08:47:16.962365+0000","last_peered":"2023-10-09T08:47:16.962365+0000","last_clean":"2023-10-09T08:47:16.962365+0000","last_became_active":"2023-10-09T08:46:49.232724+0000","last_became_peered":"2023-10-09T08:46:49.232724+0000","last_unstale":"2023-10-09T08:47:16.962365+0000","last_undegraded":"2023-10-09T08:47:16.962365+0000","last_fullsized":"2023-10-09T08:47:16.962365+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.285159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961023+0000","last_change":"2023-10-09T08:45:49.181115+0000","last_active":"2023-10-09T08:47:15.961023+0000","last_peered":"2023-10-09T08:47:15.961023+0000","last_clean":"2023-10-09T08:47:15.961023+0000","last_became_active":"2023-10-09T08:45:49.180736+0000","last_became_peered":"2023-10-09T08:45:49.180736+0000","last_unstale":"2023-10-09T08:47:15.961023+0000","last_undegraded":"2023-10-09T08:47:15.961023+0000","last_fullsized":"2023-10-09T08:47:15.961023+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.954452+0000","last_change":"2023-10-09T08:46:49.232279+0000","last_active":"2023-10-09T08:47:16.954452+0000","last_peered":"2023-10-09T08:47:16.954452+0000","last_clean":"2023-10-09T08:47:16.954452+0000","last_became_active":"2023-10-09T08:46:49.231911+0000","last_became_peered":"2023-10-09T08:46:49.231911+0000","last_unstale":"2023-10-09T08:47:16.954452+0000","last_undegraded":"2023-10-09T08:47:16.954452+0000","last_fullsized":"2023-10-09T08:47:16.954452+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:38.497218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965912+0000","last_change":"2023-10-09T08:47:16.965497+0000","last_active":"2023-10-09T08:47:16.965912+0000","last_peered":"2023-10-09T08:47:16.965912+0000","last_clean":"2023-10-09T08:47:16.965912+0000","last_became_active":"2023-10-09T08:47:16.965181+0000","last_became_peered":"2023-10-09T08:47:16.965181+0000","last_unstale":"2023-10-09T08:47:16.965912+0000","last_undegraded":"2023-10-09T08:47:16.965912+0000","last_fullsized":"2023-10-09T08:47:16.965912+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:52.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956282+0000","last_change":"2023-10-09T08:46:18.680004+0000","last_active":"2023-10-09T08:47:15.956282+0000","last_peered":"2023-10-09T08:47:15.956282+0000","last_clean":"2023-10-09T08:47:15.956282+0000","last_became_active":"2023-10-09T08:46:18.679617+0000","last_became_peered":"2023-10-09T08:46:18.679617+0000","last_unstale":"2023-10-09T08:47:15.956282+0000","last_undegraded":"2023-10-09T08:47:15.956282+0000","last_fullsized":"2023-10-09T08:47:15.956282+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956609+0000","last_change":"2023-10-09T08:34:27.860287+0000","last_active":"2023-10-09T08:47:15.956609+0000","last_peered":"2023-10-09T08:47:15.956609+0000","last_clean":"2023-10-09T08:47:15.956609+0000","last_became_active":"2023-10-09T08:34:27.859851+0000","last_became_peered":"2023-10-09T08:34:27.859851+0000","last_unstale":"2023-10-09T08:47:15.956609+0000","last_undegraded":"2023-10-09T08:47:15.956609+0000","last_fullsized":"2023-10-09T08:47:15.956609+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973899+0000","last_change":"2023-10-09T08:47:16.973286+0000","last_active":"2023-10-09T08:47:16.973899+0000","last_peered":"2023-10-09T08:47:16.973899+0000","last_clean":"2023-10-09T08:47:16.973899+0000","last_became_active":"2023-10-09T08:47:16.972839+0000","last_became_peered":"2023-10-09T08:47:16.972839+0000","last_unstale":"2023-10-09T08:47:16.973899+0000","last_undegraded":"2023-10-09T08:47:16.973899+0000","last_fullsized":"2023-10-09T08:47:16.973899+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:56.524537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":296,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956174+0000","last_change":"2023-10-09T08:45:49.183161+0000","last_active":"2023-10-09T08:47:15.956174+0000","last_peered":"2023-10-09T08:47:15.956174+0000","last_clean":"2023-10-09T08:47:15.956174+0000","last_became_active":"2023-10-09T08:45:49.182560+0000","last_became_peered":"2023-10-09T08:45:49.182560+0000","last_unstale":"2023-10-09T08:47:15.956174+0000","last_undegraded":"2023-10-09T08:47:15.956174+0000","last_fullsized":"2023-10-09T08:47:15.956174+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960367+0000","last_change":"2023-10-09T08:46:49.234616+0000","last_active":"2023-10-09T08:47:15.960367+0000","last_peered":"2023-10-09T08:47:15.960367+0000","last_clean":"2023-10-09T08:47:15.960367+0000","last_became_active":"2023-10-09T08:46:49.234217+0000","last_became_peered":"2023-10-09T08:46:49.234217+0000","last_unstale":"2023-10-09T08:47:15.960367+0000","last_undegraded":"2023-10-09T08:47:15.960367+0000","last_fullsized":"2023-10-09T08:47:15.960367+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962773+0000","last_change":"2023-10-09T08:46:18.681375+0000","last_active":"2023-10-09T08:47:16.962773+0000","last_peered":"2023-10-09T08:47:16.962773+0000","last_clean":"2023-10-09T08:47:16.962773+0000","last_became_active":"2023-10-09T08:46:18.680713+0000","last_became_peered":"2023-10-09T08:46:18.680713+0000","last_unstale":"2023-10-09T08:47:16.962773+0000","last_undegraded":"2023-10-09T08:47:16.962773+0000","last_fullsized":"2023-10-09T08:47:16.962773+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:59.671965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963096+0000","last_change":"2023-10-09T08:46:49.231894+0000","last_active":"2023-10-09T08:47:16.963096+0000","last_peered":"2023-10-09T08:47:16.963096+0000","last_clean":"2023-10-09T08:47:16.963096+0000","last_became_active":"2023-10-09T08:46:49.231531+0000","last_became_peered":"2023-10-09T08:46:49.231531+0000","last_unstale":"2023-10-09T08:47:16.963096+0000","last_undegraded":"2023-10-09T08:47:16.963096+0000","last_fullsized":"2023-10-09T08:47:16.963096+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:50.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.968243+0000","last_change":"2023-10-09T08:47:16.966880+0000","last_active":"2023-10-09T08:47:16.968243+0000","last_peered":"2023-10-09T08:47:16.968243+0000","last_clean":"2023-10-09T08:47:16.968243+0000","last_became_active":"2023-10-09T08:47:16.966539+0000","last_became_peered":"2023-10-09T08:47:16.966539+0000","last_unstale":"2023-10-09T08:47:16.968243+0000","last_undegraded":"2023-10-09T08:47:16.968243+0000","last_fullsized":"2023-10-09T08:47:16.968243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:55.047737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975638+0000","last_change":"2023-10-09T08:47:16.975638+0000","last_active":"2023-10-09T08:47:16.975638+0000","last_peered":"2023-10-09T08:47:16.975638+0000","last_clean":"2023-10-09T08:47:16.975638+0000","last_became_active":"2023-10-09T08:47:16.975248+0000","last_became_peered":"2023-10-09T08:47:16.975248+0000","last_unstale":"2023-10-09T08:47:16.975638+0000","last_undegraded":"2023-10-09T08:47:16.975638+0000","last_fullsized":"2023-10-09T08:47:16.975638+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950601+0000","last_change":"2023-10-09T08:45:49.184088+0000","last_active":"2023-10-09T08:47:15.950601+0000","last_peered":"2023-10-09T08:47:15.950601+0000","last_clean":"2023-10-09T08:47:15.950601+0000","last_became_active":"2023-10-09T08:45:49.183674+0000","last_became_peered":"2023-10-09T08:45:49.183674+0000","last_unstale":"2023-10-09T08:47:15.950601+0000","last_undegraded":"2023-10-09T08:47:15.950601+0000","last_fullsized":"2023-10-09T08:47:15.950601+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.962059+0000","last_change":"2023-10-09T08:46:18.680020+0000","last_active":"2023-10-09T08:47:15.962059+0000","last_peered":"2023-10-09T08:47:15.962059+0000","last_clean":"2023-10-09T08:47:15.962059+0000","last_became_active":"2023-10-09T08:46:18.679652+0000","last_became_peered":"2023-10-09T08:46:18.679652+0000","last_unstale":"2023-10-09T08:47:15.962059+0000","last_undegraded":"2023-10-09T08:47:15.962059+0000","last_fullsized":"2023-10-09T08:47:15.962059+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963009+0000","last_change":"2023-10-09T08:46:18.684501+0000","last_active":"2023-10-09T08:47:16.963009+0000","last_peered":"2023-10-09T08:47:16.963009+0000","last_clean":"2023-10-09T08:47:16.963009+0000","last_became_active":"2023-10-09T08:46:18.684159+0000","last_became_peered":"2023-10-09T08:46:18.684159+0000","last_unstale":"2023-10-09T08:47:16.963009+0000","last_undegraded":"2023-10-09T08:47:16.963009+0000","last_fullsized":"2023-10-09T08:47:16.963009+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:21.270259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.976238+0000","last_change":"2023-10-09T08:47:16.976238+0000","last_active":"2023-10-09T08:47:16.976238+0000","last_peered":"2023-10-09T08:47:16.976238+0000","last_clean":"2023-10-09T08:47:16.976238+0000","last_became_active":"2023-10-09T08:47:16.975840+0000","last_became_peered":"2023-10-09T08:47:16.975840+0000","last_unstale":"2023-10-09T08:47:16.976238+0000","last_undegraded":"2023-10-09T08:47:16.976238+0000","last_fullsized":"2023-10-09T08:47:16.976238+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.962626+0000","last_change":"2023-10-09T08:46:18.680373+0000","last_active":"2023-10-09T08:47:16.962626+0000","last_peered":"2023-10-09T08:47:16.962626+0000","last_clean":"2023-10-09T08:47:16.962626+0000","last_became_active":"2023-10-09T08:46:18.679999+0000","last_became_peered":"2023-10-09T08:46:18.679999+0000","last_unstale":"2023-10-09T08:47:16.962626+0000","last_undegraded":"2023-10-09T08:47:16.962626+0000","last_fullsized":"2023-10-09T08:47:16.962626+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:32.374511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965963+0000","last_change":"2023-10-09T08:47:16.965479+0000","last_active":"2023-10-09T08:47:16.965963+0000","last_peered":"2023-10-09T08:47:16.965963+0000","last_clean":"2023-10-09T08:47:16.965963+0000","last_became_active":"2023-10-09T08:47:16.965141+0000","last_became_peered":"2023-10-09T08:47:16.965141+0000","last_unstale":"2023-10-09T08:47:16.965963+0000","last_undegraded":"2023-10-09T08:47:16.965963+0000","last_fullsized":"2023-10-09T08:47:16.965963+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:50.145649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.970436+0000","last_change":"2023-10-09T08:47:16.968443+0000","last_active":"2023-10-09T08:47:16.970436+0000","last_peered":"2023-10-09T08:47:16.970436+0000","last_clean":"2023-10-09T08:47:16.970436+0000","last_became_active":"2023-10-09T08:47:16.967631+0000","last_became_peered":"2023-10-09T08:47:16.967631+0000","last_unstale":"2023-10-09T08:47:16.970436+0000","last_undegraded":"2023-10-09T08:47:16.970436+0000","last_fullsized":"2023-10-09T08:47:16.970436+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:19.412588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951852+0000","last_change":"2023-10-09T08:46:18.676843+0000","last_active":"2023-10-09T08:47:15.951852+0000","last_peered":"2023-10-09T08:47:15.951852+0000","last_clean":"2023-10-09T08:47:15.951852+0000","last_became_active":"2023-10-09T08:46:18.676468+0000","last_became_peered":"2023-10-09T08:46:18.676468+0000","last_unstale":"2023-10-09T08:47:15.951852+0000","last_undegraded":"2023-10-09T08:47:15.951852+0000","last_fullsized":"2023-10-09T08:47:15.951852+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950381+0000","last_change":"2023-10-09T08:46:49.235099+0000","last_active":"2023-10-09T08:47:15.950381+0000","last_peered":"2023-10-09T08:47:15.950381+0000","last_clean":"2023-10-09T08:47:15.950381+0000","last_became_active":"2023-10-09T08:46:49.234136+0000","last_became_peered":"2023-10-09T08:46:49.234136+0000","last_unstale":"2023-10-09T08:47:15.950381+0000","last_undegraded":"2023-10-09T08:47:15.950381+0000","last_fullsized":"2023-10-09T08:47:15.950381+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.952239+0000","last_change":"2023-10-09T08:46:49.234714+0000","last_active":"2023-10-09T08:47:15.952239+0000","last_peered":"2023-10-09T08:47:15.952239+0000","last_clean":"2023-10-09T08:47:15.952239+0000","last_became_active":"2023-10-09T08:46:49.234307+0000","last_became_peered":"2023-10-09T08:46:49.234307+0000","last_unstale":"2023-10-09T08:47:15.952239+0000","last_undegraded":"2023-10-09T08:47:15.952239+0000","last_fullsized":"2023-10-09T08:47:15.952239+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957582+0000","last_change":"2023-10-09T08:46:18.681222+0000","last_active":"2023-10-09T08:47:15.957582+0000","last_peered":"2023-10-09T08:47:15.957582+0000","last_clean":"2023-10-09T08:47:15.957582+0000","last_became_active":"2023-10-09T08:46:18.680710+0000","last_became_peered":"2023-10-09T08:46:18.680710+0000","last_unstale":"2023-10-09T08:47:15.957582+0000","last_undegraded":"2023-10-09T08:47:15.957582+0000","last_fullsized":"2023-10-09T08:47:15.957582+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972109+0000","last_change":"2023-10-09T08:47:16.972109+0000","last_active":"2023-10-09T08:47:16.972109+0000","last_peered":"2023-10-09T08:47:16.972109+0000","last_clean":"2023-10-09T08:47:16.972109+0000","last_became_active":"2023-10-09T08:47:16.971716+0000","last_became_peered":"2023-10-09T08:47:16.971716+0000","last_unstale":"2023-10-09T08:47:16.972109+0000","last_undegraded":"2023-10-09T08:47:16.972109+0000","last_fullsized":"2023-10-09T08:47:16.972109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965801+0000","last_change":"2023-10-09T08:47:16.965045+0000","last_active":"2023-10-09T08:47:16.965801+0000","last_peered":"2023-10-09T08:47:16.965801+0000","last_clean":"2023-10-09T08:47:16.965801+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:47:16.965801+0000","last_undegraded":"2023-10-09T08:47:16.965801+0000","last_fullsized":"2023-10-09T08:47:16.965801+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:49.126847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.953696+0000","last_change":"2023-10-09T08:46:18.683057+0000","last_active":"2023-10-09T08:47:16.953696+0000","last_peered":"2023-10-09T08:47:16.953696+0000","last_clean":"2023-10-09T08:47:16.953696+0000","last_became_active":"2023-10-09T08:46:18.682672+0000","last_became_peered":"2023-10-09T08:46:18.682672+0000","last_unstale":"2023-10-09T08:47:16.953696+0000","last_undegraded":"2023-10-09T08:47:16.953696+0000","last_fullsized":"2023-10-09T08:47:16.953696+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:42.975534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975514+0000","last_change":"2023-10-09T08:47:16.975514+0000","last_active":"2023-10-09T08:47:16.975514+0000","last_peered":"2023-10-09T08:47:16.975514+0000","last_clean":"2023-10-09T08:47:16.975514+0000","last_became_active":"2023-10-09T08:47:16.975184+0000","last_became_peered":"2023-10-09T08:47:16.975184+0000","last_unstale":"2023-10-09T08:47:16.975514+0000","last_undegraded":"2023-10-09T08:47:16.975514+0000","last_fullsized":"2023-10-09T08:47:16.975514+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957336+0000","last_change":"2023-10-09T08:46:49.231597+0000","last_active":"2023-10-09T08:47:15.957336+0000","last_peered":"2023-10-09T08:47:15.957336+0000","last_clean":"2023-10-09T08:47:15.957336+0000","last_became_active":"2023-10-09T08:46:49.231198+0000","last_became_peered":"2023-10-09T08:46:49.231198+0000","last_unstale":"2023-10-09T08:47:15.957336+0000","last_undegraded":"2023-10-09T08:47:15.957336+0000","last_fullsized":"2023-10-09T08:47:15.957336+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.961042+0000","last_change":"2023-10-09T08:34:27.733891+0000","last_active":"2023-10-09T08:47:15.961042+0000","last_peered":"2023-10-09T08:47:15.961042+0000","last_clean":"2023-10-09T08:47:15.961042+0000","last_became_active":"2023-10-09T08:34:27.733379+0000","last_became_peered":"2023-10-09T08:34:27.733379+0000","last_unstale":"2023-10-09T08:47:15.961042+0000","last_undegraded":"2023-10-09T08:47:15.961042+0000","last_fullsized":"2023-10-09T08:47:15.961042+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972571+0000","last_change":"2023-10-09T08:47:16.972571+0000","last_active":"2023-10-09T08:47:16.972571+0000","last_peered":"2023-10-09T08:47:16.972571+0000","last_clean":"2023-10-09T08:47:16.972571+0000","last_became_active":"2023-10-09T08:47:16.972035+0000","last_became_peered":"2023-10-09T08:47:16.972035+0000","last_unstale":"2023-10-09T08:47:16.972571+0000","last_undegraded":"2023-10-09T08:47:16.972571+0000","last_fullsized":"2023-10-09T08:47:16.972571+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.957950+0000","last_change":"2023-10-09T08:46:18.674774+0000","last_active":"2023-10-09T08:47:15.957950+0000","last_peered":"2023-10-09T08:47:15.957950+0000","last_clean":"2023-10-09T08:47:15.957950+0000","last_became_active":"2023-10-09T08:46:18.674334+0000","last_became_peered":"2023-10-09T08:46:18.674334+0000","last_unstale":"2023-10-09T08:47:15.957950+0000","last_undegraded":"2023-10-09T08:47:15.957950+0000","last_fullsized":"2023-10-09T08:47:15.957950+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960548+0000","last_change":"2023-10-09T08:34:29.490932+0000","last_active":"2023-10-09T08:47:15.960548+0000","last_peered":"2023-10-09T08:47:15.960548+0000","last_clean":"2023-10-09T08:47:15.960548+0000","last_became_active":"2023-10-09T08:34:29.489841+0000","last_became_peered":"2023-10-09T08:34:29.489841+0000","last_unstale":"2023-10-09T08:47:15.960548+0000","last_undegraded":"2023-10-09T08:47:15.960548+0000","last_fullsized":"2023-10-09T08:47:15.960548+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958369+0000","last_change":"2023-10-09T08:45:49.182558+0000","last_active":"2023-10-09T08:47:15.958369+0000","last_peered":"2023-10-09T08:47:15.958369+0000","last_clean":"2023-10-09T08:47:15.958369+0000","last_became_active":"2023-10-09T08:45:49.182145+0000","last_became_peered":"2023-10-09T08:45:49.182145+0000","last_unstale":"2023-10-09T08:47:15.958369+0000","last_undegraded":"2023-10-09T08:47:15.958369+0000","last_fullsized":"2023-10-09T08:47:15.958369+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974455+0000","last_change":"2023-10-09T08:47:16.974455+0000","last_active":"2023-10-09T08:47:16.974455+0000","last_peered":"2023-10-09T08:47:16.974455+0000","last_clean":"2023-10-09T08:47:16.974455+0000","last_became_active":"2023-10-09T08:47:16.973803+0000","last_became_peered":"2023-10-09T08:47:16.973803+0000","last_unstale":"2023-10-09T08:47:16.974455+0000","last_undegraded":"2023-10-09T08:47:16.974455+0000","last_fullsized":"2023-10-09T08:47:16.974455+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:02.753047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":69,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954654+0000","last_change":"2023-10-09T08:45:49.188401+0000","last_active":"2023-10-09T08:47:15.954654+0000","last_peered":"2023-10-09T08:47:15.954654+0000","last_clean":"2023-10-09T08:47:15.954654+0000","last_became_active":"2023-10-09T08:45:49.188014+0000","last_became_peered":"2023-10-09T08:45:49.188014+0000","last_unstale":"2023-10-09T08:47:15.954654+0000","last_undegraded":"2023-10-09T08:47:15.954654+0000","last_fullsized":"2023-10-09T08:47:15.954654+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972320+0000","last_change":"2023-10-09T08:47:16.971847+0000","last_active":"2023-10-09T08:47:16.972320+0000","last_peered":"2023-10-09T08:47:16.972320+0000","last_clean":"2023-10-09T08:47:16.972320+0000","last_became_active":"2023-10-09T08:47:16.970986+0000","last_became_peered":"2023-10-09T08:47:16.970986+0000","last_unstale":"2023-10-09T08:47:16.972320+0000","last_undegraded":"2023-10-09T08:47:16.972320+0000","last_fullsized":"2023-10-09T08:47:16.972320+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:06.084956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.954882+0000","last_change":"2023-10-09T08:34:27.726666+0000","last_active":"2023-10-09T08:47:15.954882+0000","last_peered":"2023-10-09T08:47:15.954882+0000","last_clean":"2023-10-09T08:47:15.954882+0000","last_became_active":"2023-10-09T08:34:27.726189+0000","last_became_peered":"2023-10-09T08:34:27.726189+0000","last_unstale":"2023-10-09T08:47:15.954882+0000","last_undegraded":"2023-10-09T08:47:15.954882+0000","last_fullsized":"2023-10-09T08:47:15.954882+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.975645+0000","last_change":"2023-10-09T08:47:16.975645+0000","last_active":"2023-10-09T08:47:16.975645+0000","last_peered":"2023-10-09T08:47:16.975645+0000","last_clean":"2023-10-09T08:47:16.975645+0000","last_became_active":"2023-10-09T08:47:16.975277+0000","last_became_peered":"2023-10-09T08:47:16.975277+0000","last_unstale":"2023-10-09T08:47:16.975645+0000","last_undegraded":"2023-10-09T08:47:16.975645+0000","last_fullsized":"2023-10-09T08:47:16.975645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.971694+0000","last_change":"2023-10-09T08:47:16.971357+0000","last_active":"2023-10-09T08:47:16.971694+0000","last_peered":"2023-10-09T08:47:16.971694+0000","last_clean":"2023-10-09T08:47:16.971694+0000","last_became_active":"2023-10-09T08:47:16.971034+0000","last_became_peered":"2023-10-09T08:47:16.971034+0000","last_unstale":"2023-10-09T08:47:16.971694+0000","last_undegraded":"2023-10-09T08:47:16.971694+0000","last_fullsized":"2023-10-09T08:47:16.971694+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:05.892369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.974842+0000","last_change":"2023-10-09T08:47:16.974125+0000","last_active":"2023-10-09T08:47:16.974842+0000","last_peered":"2023-10-09T08:47:16.974842+0000","last_clean":"2023-10-09T08:47:16.974842+0000","last_became_active":"2023-10-09T08:47:16.973763+0000","last_became_peered":"2023-10-09T08:47:16.973763+0000","last_unstale":"2023-10-09T08:47:16.974842+0000","last_undegraded":"2023-10-09T08:47:16.974842+0000","last_fullsized":"2023-10-09T08:47:16.974842+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:31.548547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.958645+0000","last_change":"2023-10-09T08:46:49.233182+0000","last_active":"2023-10-09T08:47:15.958645+0000","last_peered":"2023-10-09T08:47:15.958645+0000","last_clean":"2023-10-09T08:47:15.958645+0000","last_became_active":"2023-10-09T08:46:49.232746+0000","last_became_peered":"2023-10-09T08:46:49.232746+0000","last_unstale":"2023-10-09T08:47:15.958645+0000","last_undegraded":"2023-10-09T08:47:15.958645+0000","last_fullsized":"2023-10-09T08:47:15.958645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951634+0000","last_change":"2023-10-09T08:46:18.676902+0000","last_active":"2023-10-09T08:47:15.951634+0000","last_peered":"2023-10-09T08:47:15.951634+0000","last_clean":"2023-10-09T08:47:15.951634+0000","last_became_active":"2023-10-09T08:46:18.676560+0000","last_became_peered":"2023-10-09T08:46:18.676560+0000","last_unstale":"2023-10-09T08:47:15.951634+0000","last_undegraded":"2023-10-09T08:47:15.951634+0000","last_fullsized":"2023-10-09T08:47:15.951634+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":68,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.951207+0000","last_change":"2023-10-09T08:34:27.730961+0000","last_active":"2023-10-09T08:47:15.951207+0000","last_peered":"2023-10-09T08:47:15.951207+0000","last_clean":"2023-10-09T08:47:15.951207+0000","last_became_active":"2023-10-09T08:34:27.730198+0000","last_became_peered":"2023-10-09T08:34:27.730198+0000","last_unstale":"2023-10-09T08:47:15.951207+0000","last_undegraded":"2023-10-09T08:47:15.951207+0000","last_fullsized":"2023-10-09T08:47:15.951207+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.963049+0000","last_change":"2023-10-09T08:46:49.231880+0000","last_active":"2023-10-09T08:47:16.963049+0000","last_peered":"2023-10-09T08:47:16.963049+0000","last_clean":"2023-10-09T08:47:16.963049+0000","last_became_active":"2023-10-09T08:46:49.231471+0000","last_became_peered":"2023-10-09T08:46:49.231471+0000","last_unstale":"2023-10-09T08:47:16.963049+0000","last_undegraded":"2023-10-09T08:47:16.963049+0000","last_fullsized":"2023-10-09T08:47:16.963049+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:31.742745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973891+0000","last_change":"2023-10-09T08:47:16.973891+0000","last_active":"2023-10-09T08:47:16.973891+0000","last_peered":"2023-10-09T08:47:16.973891+0000","last_clean":"2023-10-09T08:47:16.973891+0000","last_became_active":"2023-10-09T08:47:16.973405+0000","last_became_peered":"2023-10-09T08:47:16.973405+0000","last_unstale":"2023-10-09T08:47:16.973891+0000","last_undegraded":"2023-10-09T08:47:16.973891+0000","last_fullsized":"2023-10-09T08:47:16.973891+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.973134+0000","last_change":"2023-10-09T08:47:16.972498+0000","last_active":"2023-10-09T08:47:16.973134+0000","last_peered":"2023-10-09T08:47:16.973134+0000","last_clean":"2023-10-09T08:47:16.973134+0000","last_became_active":"2023-10-09T08:47:16.972184+0000","last_became_peered":"2023-10-09T08:47:16.972184+0000","last_unstale":"2023-10-09T08:47:16.973134+0000","last_undegraded":"2023-10-09T08:47:16.973134+0000","last_fullsized":"2023-10-09T08:47:16.973134+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:03.559858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.960484+0000","last_change":"2023-10-09T08:46:18.675538+0000","last_active":"2023-10-09T08:47:15.960484+0000","last_peered":"2023-10-09T08:47:15.960484+0000","last_clean":"2023-10-09T08:47:15.960484+0000","last_became_active":"2023-10-09T08:46:18.675149+0000","last_became_peered":"2023-10-09T08:46:18.675149+0000","last_unstale":"2023-10-09T08:47:15.960484+0000","last_undegraded":"2023-10-09T08:47:15.960484+0000","last_fullsized":"2023-10-09T08:47:15.960484+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972970+0000","last_change":"2023-10-09T08:47:16.969948+0000","last_active":"2023-10-09T08:47:16.972970+0000","last_peered":"2023-10-09T08:47:16.972970+0000","last_clean":"2023-10-09T08:47:16.972970+0000","last_became_active":"2023-10-09T08:47:16.969666+0000","last_became_peered":"2023-10-09T08:47:16.969666+0000","last_unstale":"2023-10-09T08:47:16.972970+0000","last_undegraded":"2023-10-09T08:47:16.972970+0000","last_fullsized":"2023-10-09T08:47:16.972970+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:25.089633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972317+0000","last_change":"2023-10-09T08:47:16.971965+0000","last_active":"2023-10-09T08:47:16.972317+0000","last_peered":"2023-10-09T08:47:16.972317+0000","last_clean":"2023-10-09T08:47:16.972317+0000","last_became_active":"2023-10-09T08:47:16.971651+0000","last_became_peered":"2023-10-09T08:47:16.971651+0000","last_unstale":"2023-10-09T08:47:16.972317+0000","last_undegraded":"2023-10-09T08:47:16.972317+0000","last_fullsized":"2023-10-09T08:47:16.972317+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:04.259603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.956192+0000","last_change":"2023-10-09T08:46:18.680365+0000","last_active":"2023-10-09T08:47:15.956192+0000","last_peered":"2023-10-09T08:47:15.956192+0000","last_clean":"2023-10-09T08:47:15.956192+0000","last_became_active":"2023-10-09T08:46:18.679875+0000","last_became_peered":"2023-10-09T08:46:18.679875+0000","last_unstale":"2023-10-09T08:47:15.956192+0000","last_undegraded":"2023-10-09T08:47:15.956192+0000","last_fullsized":"2023-10-09T08:47:15.956192+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965460+0000","last_change":"2023-10-09T08:47:16.965460+0000","last_active":"2023-10-09T08:47:16.965460+0000","last_peered":"2023-10-09T08:47:16.965460+0000","last_clean":"2023-10-09T08:47:16.965460+0000","last_became_active":"2023-10-09T08:47:16.965040+0000","last_became_peered":"2023-10-09T08:47:16.965040+0000","last_unstale":"2023-10-09T08:47:16.965460+0000","last_undegraded":"2023-10-09T08:47:16.965460+0000","last_fullsized":"2023-10-09T08:47:16.965460+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.025735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.972434+0000","last_change":"2023-10-09T08:47:16.972106+0000","last_active":"2023-10-09T08:47:16.972434+0000","last_peered":"2023-10-09T08:47:16.972434+0000","last_clean":"2023-10-09T08:47:16.972434+0000","last_became_active":"2023-10-09T08:47:16.971511+0000","last_became_peered":"2023-10-09T08:47:16.971511+0000","last_unstale":"2023-10-09T08:47:16.972434+0000","last_undegraded":"2023-10-09T08:47:16.972434+0000","last_fullsized":"2023-10-09T08:47:16.972434+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:12.674597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.969696+0000","last_change":"2023-10-09T08:47:16.968800+0000","last_active":"2023-10-09T08:47:16.969696+0000","last_peered":"2023-10-09T08:47:16.969696+0000","last_clean":"2023-10-09T08:47:16.969696+0000","last_became_active":"2023-10-09T08:47:16.968430+0000","last_became_peered":"2023-10-09T08:47:16.968430+0000","last_unstale":"2023-10-09T08:47:16.969696+0000","last_undegraded":"2023-10-09T08:47:16.969696+0000","last_fullsized":"2023-10-09T08:47:16.969696+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:08.011498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T08:47:16.965051+0000","last_change":"2023-10-09T08:47:16.965051+0000","last_active":"2023-10-09T08:47:16.965051+0000","last_peered":"2023-10-09T08:47:16.965051+0000","last_clean":"2023-10-09T08:47:16.965051+0000","last_became_active":"2023-10-09T08:47:16.964664+0000","last_became_peered":"2023-10-09T08:47:16.964664+0000","last_unstale":"2023-10-09T08:47:16.965051+0000","last_undegraded":"2023-10-09T08:47:16.965051+0000","last_fullsized":"2023-10-09T08:47:16.965051+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.377202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":93,"state":"active+clean","last_fresh":"2023-10-09T08:47:15.950228+0000","last_change":"2023-10-09T08:46:49.233483+0000","last_active":"2023-10-09T08:47:15.950228+0000","last_peered":"2023-10-09T08:47:15.950228+0000","last_clean":"2023-10-09T08:47:15.950228+0000","last_became_active":"2023-10-09T08:46:49.233033+0000","last_became_peered":"2023-10-09T08:46:49.233033+0000","last_unstale":"2023-10-09T08:47:15.950228+0000","last_undegraded":"2023-10-09T08:47:15.950228+0000","last_fullsized":"2023-10-09T08:47:15.950228+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430429,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310904,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434248,"statfs":{"total":95995035648,"available":95676669952,"internally_reserved":0,"allocated":1556480,"data_stored":1102308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.92900000000000005,"15min":0.92200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.2569999999999999,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.747},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.94199999999999995,"15min":0.94499999999999995},"min":{"1min":0.64300000000000002,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1679999999999999,"5min":1.262,"15min":1.262},"last":0.89100000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.3,"5min":1.446,"15min":1.446},"last":1.113},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97999999999999998,"15min":0.97499999999999998},"min":{"1min":0.64200000000000002,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.2589999999999999,"5min":1.319,"15min":1.319},"last":0.99199999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:48:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.91400000000000003,"15min":0.92400000000000004},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.125,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.92700000000000005,"15min":0.93000000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":1.1699999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91800000000000004,"15min":0.92000000000000004},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.131,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0089999999999999,"15min":1.012},"min":{"1min":0.76100000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.28,"5min":1.335,"15min":1.335},"last":1.0329999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.88500000000000001,"15min":0.88900000000000001},"min":{"1min":0.59099999999999997,"5min":0.53200000000000003,"15min":0.40300000000000002},"max":{"1min":1.202,"5min":1.2809999999999999,"15min":1.4630000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.03,"5min":0.98399999999999999,"15min":0.94799999999999995},"min":{"1min":0.67800000000000005,"5min":0.54800000000000004,"15min":0.40000000000000002},"max":{"1min":1.351,"5min":1.488,"15min":1.488},"last":0.84899999999999998}]},{"osd":5,"last update":"Mon Oct 9 08:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.97399999999999998,"15min":0.99299999999999999},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.501},"max":{"1min":1.23,"5min":1.3129999999999999,"15min":1.385},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.68200000000000005,"5min":0.60899999999999999,"15min":0.46100000000000002},"max":{"1min":1.2310000000000001,"5min":1.3400000000000001,"15min":1.794},"last":1.0840000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.038,"15min":1.038},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.44},"max":{"1min":1.3180000000000001,"5min":1.4350000000000001,"15min":1.7170000000000001},"last":0.93000000000000005},{"interface":"front","average":{"1min":1.117,"5min":1.101,"15min":1.073},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.45000000000000001},"max":{"1min":1.4330000000000001,"5min":1.528,"15min":1.528},"last":0.95999999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626662,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1508,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1544192,"data_stored":1102112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:58 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.93500000000000005,"15min":0.93000000000000005},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2609999999999999,"5min":1.349,"15min":1.349},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0129999999999999,"15min":0.996},"min":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3320000000000001,"5min":1.5,"15min":1.5},"last":1.169}]},{"osd":1,"last update":"Mon Oct 9 08:49:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.90000000000000002,"15min":0.89800000000000002},"min":{"1min":0.65100000000000002,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.8720000000000001,"5min":1.8720000000000001,"15min":1.8720000000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.95699999999999996,"15min":0.95999999999999996},"min":{"1min":0.67100000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2649999999999999,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.1020000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:48:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.91400000000000003,"15min":0.91600000000000004},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.94899999999999995,"15min":0.93999999999999995},"min":{"1min":0.76500000000000001,"5min":0.625,"15min":0.625},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.034}]},{"osd":3,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.91500000000000004,"15min":0.91100000000000003},"min":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.94499999999999995,"15min":0.94699999999999995},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.099,"5min":1.167,"15min":1.167},"last":0.91700000000000004}]},{"osd":4,"last update":"Mon Oct 9 08:49:12 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.89900000000000002,"15min":0.90800000000000003},"min":{"1min":0.55300000000000005,"5min":0.50700000000000001,"15min":0.379},"max":{"1min":1.381,"5min":1.381,"15min":1.629},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.022,"5min":0.93300000000000005,"15min":0.95499999999999996},"min":{"1min":0.66400000000000003,"5min":0.436,"15min":0.436},"max":{"1min":1.7250000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.077}]},{"osd":5,"last update":"Mon Oct 9 08:49:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.004,"15min":0.98999999999999999},"min":{"1min":0.65800000000000003,"5min":0.495,"15min":0.495},"max":{"1min":1.464,"5min":1.464,"15min":1.7410000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0309999999999999,"15min":1.048},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.51600000000000001},"max":{"1min":1.825,"5min":1.825,"15min":4.4420000000000002},"last":1.0029999999999999}]},{"osd":7,"last update":"Mon Oct 9 08:48:58 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.052,"15min":1.056},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.45900000000000002},"max":{"1min":1.3460000000000001,"5min":1.383,"15min":1.8220000000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.0569999999999999,"15min":1.085},"min":{"1min":0.80000000000000004,"5min":0.53300000000000003,"15min":0.52500000000000002},"max":{"1min":1.4119999999999999,"5min":1.556,"15min":2.262},"last":1.141}]}]},{"osd":1,"up_from":83,"seq":356482285610,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32396,"kb_used_data":964,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712756,"statfs":{"total":95995035648,"available":95961862144,"internally_reserved":0,"allocated":987136,"data_stored":516401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.63400000000000001,"15min":0.628},"min":{"1min":0.39300000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68600000000000005,"15min":0.67600000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.94899999999999995}]},{"osd":2,"last update":"Mon Oct 9 08:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.72299999999999998,"15min":0.73899999999999999},"min":{"1min":0.41199999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80900000000000005,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":1.2689999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.69999999999999996,"15min":0.70399999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":1.0580000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.84799999999999998,"15min":0.85499999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.01}]},{"osd":4,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.97099999999999997,"15min":0.96599999999999997},"min":{"1min":0.70499999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.238},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.98499999999999999,"15min":0.97399999999999998},"min":{"1min":0.68000000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3,"5min":1.577,"15min":1.577},"last":1.417}]},{"osd":5,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.0569999999999999,"15min":1.0640000000000001},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.3839999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.0549999999999999,"15min":1.0649999999999999},"min":{"1min":0.745,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":1.363}]},{"osd":6,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.095,"15min":1.087},"min":{"1min":0.72599999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.4319999999999999,"5min":1.522,"15min":1.522},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.115,"15min":1.111},"min":{"1min":0.755,"5min":0.754,"15min":0.754},"max":{"1min":1.383,"5min":1.577,"15min":1.577},"last":1.085}]},{"osd":7,"last update":"Mon Oct 9 08:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0549999999999999,"15min":1.042},"min":{"1min":0.73299999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.159},{"interface":"front","average":{"1min":1.097,"5min":1.107,"15min":1.107},"min":{"1min":0.79600000000000004,"5min":0.624,"15min":0.624},"max":{"1min":1.4039999999999999,"5min":1.625,"15min":1.625},"last":1.3180000000000001}]}]},{"osd":0,"up_from":78,"seq":335007449136,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33060,"kb_used_data":1564,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712092,"statfs":{"total":95995035648,"available":95961182208,"internally_reserved":0,"allocated":1601536,"data_stored":1109136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.71099999999999997,"15min":0.71599999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.0780000000000001,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":1.0609999999999999},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.76300000000000001,"15min":0.77900000000000003},"min":{"1min":0.44500000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.0489999999999999,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.82099999999999995}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.83199999999999996,"15min":0.84799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.083,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.92900000000000005,"15min":0.95099999999999996},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.94499999999999995}]},{"osd":3,"last update":"Mon Oct 9 08:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.79200000000000004,"15min":0.79800000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2130000000000001,"5min":1.306,"15min":1.306},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.82799999999999996,"15min":0.82999999999999996},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2250000000000001,"5min":1.321,"15min":1.321},"last":1.032}]},{"osd":4,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0389999999999999,"15min":1.046},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":1.1459999999999999},{"interface":"front","average":{"1min":0.873,"5min":0.93700000000000006,"15min":0.96199999999999997},"min":{"1min":0.60799999999999998,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.0469999999999999,"5min":1.528,"15min":1.528},"last":0.80200000000000005}]},{"osd":5,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0189999999999999,"15min":1.018},"min":{"1min":0.63500000000000001,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.361,"5min":1.51,"15min":1.51},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0980000000000001,"15min":1.115},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.1000000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.1359999999999999,"15min":1.1599999999999999},"min":{"1min":0.77200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3959999999999999,"5min":1.663,"15min":1.663},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.1950000000000001,"15min":1.2150000000000001},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.486,"5min":1.609,"15min":1.609},"last":1.24}]},{"osd":7,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.214,"15min":1.2330000000000001},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.5880000000000001,"5min":1.679,"15min":1.679},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.028,"5min":1.117,"15min":1.1479999999999999},"min":{"1min":0.64600000000000002,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.3779999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":0.98599999999999999}]}]},{"osd":2,"up_from":88,"seq":377957122084,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36616,"kb_used_data":960,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708536,"statfs":{"total":95995035648,"available":95957540864,"internally_reserved":0,"allocated":983040,"data_stored":514873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.70999999999999996,"15min":0.70199999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.79300000000000004,"15min":0.81000000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2629999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.1020000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.0009999999999999,"5min":1.127,"15min":1.127},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.755,"5min":0.82899999999999996,"15min":0.84099999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.165,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.95599999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.83399999999999996,"15min":0.82699999999999996},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.84099999999999997},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.80700000000000005,"15min":0.80900000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.70399999999999996}]},{"osd":4,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.96599999999999997,"15min":0.97199999999999998},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.1919999999999999,"5min":1.24,"15min":1.24},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0469999999999999,"15min":1.0580000000000001},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.0580000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.131,"15min":1.1579999999999999},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.464,"5min":1.49,"15min":1.49},"last":1.204},{"interface":"front","average":{"1min":1,"5min":1.0649999999999999,"15min":1.0760000000000001},"min":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"max":{"1min":1.2230000000000001,"5min":1.337,"15min":1.337},"last":1.0489999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.1259999999999999,"15min":1.137},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.3100000000000001,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.1870000000000001,"15min":1.204},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.446,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.294}]},{"osd":7,"last update":"Mon Oct 9 08:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.1539999999999999,"15min":1.1599999999999999},"min":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"max":{"1min":1.4099999999999999,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":1.236},{"interface":"front","average":{"1min":1.087,"5min":1.214,"15min":1.2350000000000001},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.5169999999999999,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.3740000000000001}]}]},{"osd":3,"up_from":93,"seq":399431958558,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28276,"kb_used_data":940,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716876,"statfs":{"total":95995035648,"available":95966081024,"internally_reserved":0,"allocated":962560,"data_stored":511788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.71499999999999997,"15min":0.72699999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.93400000000000005,"5min":1.198,"15min":1.198},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.752,"5min":0.78600000000000003,"15min":0.79200000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1339999999999999,"5min":1.21,"15min":1.21},"last":0.95699999999999996}]},{"osd":1,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.77800000000000002,"15min":0.78600000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.80800000000000005,"15min":0.81000000000000005},"min":{"1min":0.51300000000000001,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.0649999999999999,"5min":1.111,"15min":1.111},"last":1.2290000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.76200000000000001,"15min":0.76900000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.80500000000000005,"15min":0.81100000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.0029999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":1.298}]},{"osd":4,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.012,"15min":1.016},"min":{"1min":0.79500000000000004,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.2150000000000001,"5min":1.321,"15min":1.321},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.0169999999999999,"15min":1.0309999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.2649999999999999,"5min":1.411,"15min":1.411},"last":0.85599999999999998}]},{"osd":5,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.016,"15min":1.028},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.2689999999999999,"5min":1.306,"15min":1.306},"last":1.196},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0640000000000001,"15min":1.073},"min":{"1min":0.80200000000000005,"5min":0.79900000000000004,"15min":0.79900000000000004},"max":{"1min":1.429,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.0660000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.1080000000000001,"15min":1.1220000000000001},"min":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"max":{"1min":1.373,"5min":1.373,"15min":1.373},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.1819999999999999,"15min":1.1970000000000001},"min":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"max":{"1min":1.3160000000000001,"5min":1.482,"15min":1.482},"last":1.159}]},{"osd":7,"last update":"Mon Oct 9 08:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.1439999999999999,"15min":1.1559999999999999},"min":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.351,"5min":1.52,"15min":1.52},"last":1.26},{"interface":"front","average":{"1min":1.123,"5min":1.1579999999999999,"15min":1.1639999999999999},"min":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"max":{"1min":1.399,"5min":1.581,"15min":1.581},"last":1.385}]}]},{"osd":4,"up_from":30,"seq":128849019128,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302132,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443020,"statfs":{"total":95995035648,"available":95685652480,"internally_reserved":0,"allocated":962560,"data_stored":512135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:49:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.88100000000000001,"15min":0.879},"min":{"1min":0.61699999999999999,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.238,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.89900000000000002,"15min":0.89600000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.66300000000000003}]},{"osd":1,"last update":"Mon Oct 9 08:49:22 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.85599999999999998,"15min":0.85199999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.256,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.97399999999999998,"15min":0.96299999999999997},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.234,"5min":1.321,"15min":1.321},"last":0.80000000000000004}]},{"osd":2,"last update":"Mon Oct 9 08:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93799999999999994,"15min":0.93600000000000005},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.998},{"interface":"front","average":{"1min":1.03,"5min":0.96599999999999997,"15min":0.95599999999999996},"min":{"1min":0.80200000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.1499999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:49:17 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":0.97899999999999998},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.032,"15min":1.032},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":1.2130000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:49:32 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.90900000000000003,"15min":0.91600000000000004},"min":{"1min":0.63500000000000001,"5min":0.51200000000000001,"15min":0.39200000000000002},"max":{"1min":1.29,"5min":1.431,"15min":1.431},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.014,"15min":1},"min":{"1min":0.64200000000000002,"5min":0.60099999999999998,"15min":0.47399999999999998},"max":{"1min":1.3129999999999999,"5min":1.389,"15min":1.4730000000000001},"last":0.94999999999999996}]},{"osd":6,"last update":"Mon Oct 9 08:49:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.99399999999999999,"15min":0.96199999999999997},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.42899999999999999},"max":{"1min":1.948,"5min":1.948,"15min":1.948},"last":0.86299999999999999},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.0669999999999999,"15min":1.056},"min":{"1min":0.66400000000000003,"5min":0.57599999999999996,"15min":0.48399999999999999},"max":{"1min":1.478,"5min":1.478,"15min":1.478},"last":1.0960000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:49:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0860000000000001,"15min":1.0820000000000001},"min":{"1min":0.81100000000000005,"5min":0.55100000000000005,"15min":0.44800000000000001},"max":{"1min":1.484,"5min":1.484,"15min":1.484},"last":1.075},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.121,"15min":1.0860000000000001},"min":{"1min":0.89800000000000002,"5min":0.71899999999999997,"15min":0.38},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":1.119}]}]},{"osd":5,"up_from":36,"seq":154618822896,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310308,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434844,"statfs":{"total":95995035648,"available":95677280256,"internally_reserved":0,"allocated":946176,"data_stored":507879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.876,"15min":0.876},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.1419999999999999,"5min":1.262,"15min":1.262},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.91100000000000003,"15min":0.90600000000000003},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.133,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.93000000000000005}]},{"osd":1,"last update":"Mon Oct 9 08:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.89800000000000002,"15min":0.90700000000000003},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.1379999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.90500000000000003,"15min":0.91200000000000003},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.113,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":1.125}]},{"osd":2,"last update":"Mon Oct 9 08:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.92000000000000004,"15min":0.92700000000000005},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1639999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.98999999999999999,"15min":0.998},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.284,"5min":1.323,"15min":1.323},"last":1.147}]},{"osd":3,"last update":"Mon Oct 9 08:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.998,"15min":1.0029999999999999},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.177,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.878,"5min":0.93799999999999994,"15min":0.94799999999999995},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.0720000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":1.044}]},{"osd":4,"last update":"Mon Oct 9 08:49:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.88500000000000001,"15min":0.86399999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.46999999999999997},"max":{"1min":1.258,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.048,"5min":0.97599999999999998,"15min":0.96199999999999997},"min":{"1min":0.68999999999999995,"5min":0.505,"15min":0.442},"max":{"1min":1.244,"5min":1.405,"15min":1.405},"last":0.85599999999999998}]},{"osd":6,"last update":"Mon Oct 9 08:49:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.93799999999999994,"15min":0.96399999999999997},"min":{"1min":0.68000000000000005,"5min":0.55900000000000005,"15min":0.45200000000000001},"max":{"1min":1.1619999999999999,"5min":1.333,"15min":1.411},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.038,"15min":1.0249999999999999},"min":{"1min":0.70899999999999996,"5min":0.69099999999999995,"15min":0.40400000000000003},"max":{"1min":1.397,"5min":1.4750000000000001,"15min":3.2919999999999998},"last":0.999}]},{"osd":7,"last update":"Mon Oct 9 08:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.0820000000000001,"15min":1.05},"min":{"1min":0.67300000000000004,"5min":0.53300000000000003,"15min":0.36899999999999999},"max":{"1min":1.3600000000000001,"5min":1.4119999999999999,"15min":1.4119999999999999},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.071,"15min":1.0780000000000001},"min":{"1min":0.77100000000000002,"5min":0.66300000000000003,"15min":0.45000000000000001},"max":{"1min":1.274,"5min":1.4950000000000001,"15min":1.782},"last":1.1819999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:49:38.338 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T08:49:38.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T08:49:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:39 smithi094 bash[59677]: cluster 2023-10-09T08:49:39.296664+0000 mgr.y (mgr.25421) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:40.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:39 smithi016 bash[75179]: cluster 2023-10-09T08:49:39.296664+0000 mgr.y (mgr.25421) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:40.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:39 smithi016 bash[77892]: cluster 2023-10-09T08:49:39.296664+0000 mgr.y (mgr.25421) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:41 smithi094 bash[59677]: audit 2023-10-09T08:49:40.101228+0000 mon.a (mon.0) 483 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:41 smithi094 bash[59677]: audit 2023-10-09T08:49:40.108980+0000 mon.a (mon.0) 484 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:41 smithi094 bash[59677]: audit 2023-10-09T08:49:40.307356+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:41 smithi094 bash[59677]: audit 2023-10-09T08:49:40.315756+0000 mon.a (mon.0) 486 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:41 smithi094 bash[59677]: audit 2023-10-09T08:49:40.798641+0000 mon.a (mon.0) 487 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:49:41.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:41 smithi094 bash[59677]: audit 2023-10-09T08:49:40.800514+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:49:41.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:41 smithi094 bash[59677]: audit 2023-10-09T08:49:40.810403+0000 mon.a (mon.0) 489 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[75179]: audit 2023-10-09T08:49:40.101228+0000 mon.a (mon.0) 483 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[75179]: audit 2023-10-09T08:49:40.108980+0000 mon.a (mon.0) 484 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[75179]: audit 2023-10-09T08:49:40.307356+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[75179]: audit 2023-10-09T08:49:40.315756+0000 mon.a (mon.0) 486 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[75179]: audit 2023-10-09T08:49:40.798641+0000 mon.a (mon.0) 487 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[75179]: audit 2023-10-09T08:49:40.800514+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[75179]: audit 2023-10-09T08:49:40.810403+0000 mon.a (mon.0) 489 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[77892]: audit 2023-10-09T08:49:40.101228+0000 mon.a (mon.0) 483 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[77892]: audit 2023-10-09T08:49:40.108980+0000 mon.a (mon.0) 484 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[77892]: audit 2023-10-09T08:49:40.307356+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[77892]: audit 2023-10-09T08:49:40.315756+0000 mon.a (mon.0) 486 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:41.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[77892]: audit 2023-10-09T08:49:40.798641+0000 mon.a (mon.0) 487 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:49:41.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[77892]: audit 2023-10-09T08:49:40.800514+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:49:41.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:41 smithi016 bash[77892]: audit 2023-10-09T08:49:40.810403+0000 mon.a (mon.0) 489 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:49:42.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:42 smithi094 bash[59677]: cluster 2023-10-09T08:49:41.297647+0000 mgr.y (mgr.25421) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:42.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:42 smithi016 bash[75179]: cluster 2023-10-09T08:49:41.297647+0000 mgr.y (mgr.25421) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:42.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:42 smithi016 bash[77892]: cluster 2023-10-09T08:49:41.297647+0000 mgr.y (mgr.25421) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:42.960 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:49:43.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:43 smithi094 bash[59677]: cluster 2023-10-09T08:49:43.298473+0000 mgr.y (mgr.25421) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:43.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:43 smithi016 bash[75179]: cluster 2023-10-09T08:49:43.298473+0000 mgr.y (mgr.25421) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:43.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:43 smithi016 bash[77892]: cluster 2023-10-09T08:49:43.298473+0000 mgr.y (mgr.25421) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:45.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:45 smithi016 bash[75179]: cluster 2023-10-09T08:49:45.299436+0000 mgr.y (mgr.25421) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:45.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:45 smithi016 bash[77892]: cluster 2023-10-09T08:49:45.299436+0000 mgr.y (mgr.25421) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:45 smithi094 bash[59677]: cluster 2023-10-09T08:49:45.299436+0000 mgr.y (mgr.25421) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:45.992 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:49:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:49:45] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:epoch 94 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:fsid f674ade6-667c-11ee-8db5-212e2dc638e7 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:created 2023-10-09T08:23:18.622123+0000 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:modified 2023-10-09T08:47:16.948445+0000 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:crush_version 22 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:full_ratio 0.95 2023-10-09T08:49:46.470 INFO:teuthology.orchestra.run.smithi016.stdout:backfillfull_ratio 0.9 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:nearfull_ratio 0.85 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:require_min_compat_client luminous 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:min_compat_client luminous 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:require_osd_release pacific 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:stretch_mode_enabled false 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 72 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 55 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T08:49:46.471 INFO:teuthology.orchestra.run.smithi016.stdout:max_osd 8 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 up in weight 1 up_from 78 up_thru 93 down_at 75 last_clean_interval [8,74) [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916] [v2:172.21.15.16:6804/2159748916,v1:172.21.15.16:6805/2159748916] exists,up 0334a36b-f8d4-486c-a3f7-6b5278148938 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 up in weight 1 up_from 83 up_thru 93 down_at 80 last_clean_interval [13,79) [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354] [v2:172.21.15.16:6812/3704674354,v1:172.21.15.16:6813/3704674354] exists,up fc3d1f7f-18df-4428-90b3-68c1d8296b11 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 up in weight 1 up_from 88 up_thru 93 down_at 85 last_clean_interval [18,84) [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489] [v2:172.21.15.16:6820/1227549489,v1:172.21.15.16:6821/1227549489] exists,up 19af46cb-b0d8-410d-bf62-d4352bafb831 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 up in weight 1 up_from 93 up_thru 93 down_at 90 last_clean_interval [25,89) [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985] [v2:172.21.15.16:6828/1617049985,v1:172.21.15.16:6829/1617049985] exists,up 9ee2e425-a2d1-432d-838c-e246649de096 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 up in weight 1 up_from 30 up_thru 93 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189] [v2:172.21.15.94:6802/2620675189,v1:172.21.15.94:6803/2620675189] exists,up 44c51083-8ea0-4d32-8849-c5524f94afa4 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 up in weight 1 up_from 36 up_thru 93 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646] [v2:172.21.15.94:6810/1438595646,v1:172.21.15.94:6811/1438595646] exists,up fc3fc2f1-b5f5-4986-9260-98c23cd9c49e 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 up in weight 1 up_from 42 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278] [v2:172.21.15.94:6818/4105872278,v1:172.21.15.94:6819/4105872278] exists,up deb8f0b9-0b80-4dea-b180-353b84fb5daf 2023-10-09T08:49:46.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 up in weight 1 up_from 48 up_thru 93 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234] [v2:172.21.15.94:6826/3615932234,v1:172.21.15.94:6827/3615932234] exists,up 84cd66c8-c28e-4d58-ac4d-3836315c04f2 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:pg_upmap_items 4.4 [1,5] 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:pg_upmap_items 4.9 [1,2] 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:pg_upmap_items 4.a [1,2] 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/147602172 expires 2023-10-10T08:23:46.198487+0000 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/2419669477 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/1478513102 expires 2023-10-10T08:38:06.212642+0000 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3538653132 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.94:0/3558864062 expires 2023-10-10T08:39:43.331826+0000 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/1403279027 expires 2023-10-10T08:32:04.579457+0000 2023-10-09T08:49:46.473 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/1291849630 expires 2023-10-10T08:32:04.579457+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.94:0/1375017130 expires 2023-10-10T08:39:43.331826+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/1383471245 expires 2023-10-10T08:24:16.162112+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/1247249547 expires 2023-10-10T08:24:16.162112+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3686891819 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/2495526270 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.94:0/2012987098 expires 2023-10-10T08:39:43.331826+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/4047990666 expires 2023-10-10T08:24:16.162112+0000 2023-10-09T08:49:46.474 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/243905061 expires 2023-10-10T08:23:46.198487+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/1710200957 expires 2023-10-10T08:38:06.212642+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.94:0/727051339 expires 2023-10-10T08:39:43.331826+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/2576600022 expires 2023-10-10T08:23:46.198487+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3915272728 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/4047990666 expires 2023-10-10T08:24:16.162112+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/243905061 expires 2023-10-10T08:23:46.198487+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/1021312597 expires 2023-10-10T08:38:06.212642+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3387393375 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.475 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/1710200957 expires 2023-10-10T08:38:06.212642+0000 2023-10-09T08:49:46.476 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/2495526270 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.476 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.94:6834/2010618240 expires 2023-10-10T08:39:43.331826+0000 2023-10-09T08:49:46.476 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.94:6835/2010618240 expires 2023-10-10T08:39:43.331826+0000 2023-10-09T08:49:46.476 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/3038487937 expires 2023-10-10T08:32:04.579457+0000 2023-10-09T08:49:46.476 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/2254945868 expires 2023-10-10T08:40:49.059520+0000 2023-10-09T08:49:46.476 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/3038487937 expires 2023-10-10T08:32:04.579457+0000 2023-10-09T08:49:46.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:46 smithi094 bash[59677]: audit 2023-10-09T08:49:46.464965+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.16:0/774756356' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T08:49:47.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:46 smithi016 bash[75179]: audit 2023-10-09T08:49:46.464965+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.16:0/774756356' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T08:49:47.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:46 smithi016 bash[77892]: audit 2023-10-09T08:49:46.464965+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.16:0/774756356' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T08:49:47.336 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 78 up_thru 93 down_at 75 last_clean_interval [8,74) [v2:172.21.15.16:6802/2159748916,v1:172.21.15.16:6803/2159748916] [v2:172.21.15.16:6804/2159748916,v1:172.21.15.16:6805/2159748916] exists,up 0334a36b-f8d4-486c-a3f7-6b5278148938', 'osd.1 up in weight 1 up_from 83 up_thru 93 down_at 80 last_clean_interval [13,79) [v2:172.21.15.16:6810/3704674354,v1:172.21.15.16:6811/3704674354] [v2:172.21.15.16:6812/3704674354,v1:172.21.15.16:6813/3704674354] exists,up fc3d1f7f-18df-4428-90b3-68c1d8296b11', 'osd.2 up in weight 1 up_from 88 up_thru 93 down_at 85 last_clean_interval [18,84) [v2:172.21.15.16:6818/1227549489,v1:172.21.15.16:6819/1227549489] [v2:172.21.15.16:6820/1227549489,v1:172.21.15.16:6821/1227549489] exists,up 19af46cb-b0d8-410d-bf62-d4352bafb831', 'osd.3 up in weight 1 up_from 93 up_thru 93 down_at 90 last_clean_interval [25,89) [v2:172.21.15.16:6826/1617049985,v1:172.21.15.16:6827/1617049985] [v2:172.21.15.16:6828/1617049985,v1:172.21.15.16:6829/1617049985] exists,up 9ee2e425-a2d1-432d-838c-e246649de096', 'osd.4 up in weight 1 up_from 30 up_thru 93 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6800/2620675189,v1:172.21.15.94:6801/2620675189] [v2:172.21.15.94:6802/2620675189,v1:172.21.15.94:6803/2620675189] exists,up 44c51083-8ea0-4d32-8849-c5524f94afa4', 'osd.5 up in weight 1 up_from 36 up_thru 93 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6808/1438595646,v1:172.21.15.94:6809/1438595646] [v2:172.21.15.94:6810/1438595646,v1:172.21.15.94:6811/1438595646] exists,up fc3fc2f1-b5f5-4986-9260-98c23cd9c49e', 'osd.6 up in weight 1 up_from 42 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6816/4105872278,v1:172.21.15.94:6817/4105872278] [v2:172.21.15.94:6818/4105872278,v1:172.21.15.94:6819/4105872278] exists,up deb8f0b9-0b80-4dea-b180-353b84fb5daf', 'osd.7 up in weight 1 up_from 48 up_thru 93 down_at 0 last_clean_interval [0,0) [v2:172.21.15.94:6824/3615932234,v1:172.21.15.94:6825/3615932234] [v2:172.21.15.94:6826/3615932234,v1:172.21.15.94:6827/3615932234] exists,up 84cd66c8-c28e-4d58-ac4d-3836315c04f2'] 2023-10-09T08:49:47.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T08:49:47.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:47 smithi016 bash[75179]: cluster 2023-10-09T08:49:47.300271+0000 mgr.y (mgr.25421) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:47.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:47 smithi016 bash[77892]: cluster 2023-10-09T08:49:47.300271+0000 mgr.y (mgr.25421) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:47.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:47 smithi094 bash[59677]: cluster 2023-10-09T08:49:47.300271+0000 mgr.y (mgr.25421) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:50.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:49 smithi094 bash[59677]: audit 2023-10-09T08:49:49.205262+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:50.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:49 smithi094 bash[59677]: cluster 2023-10-09T08:49:49.300867+0000 mgr.y (mgr.25421) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:50.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:49 smithi016 bash[75179]: audit 2023-10-09T08:49:49.205262+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:50.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:49 smithi016 bash[75179]: cluster 2023-10-09T08:49:49.300867+0000 mgr.y (mgr.25421) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:50.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:49 smithi016 bash[77892]: audit 2023-10-09T08:49:49.205262+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:49:50.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:49 smithi016 bash[77892]: cluster 2023-10-09T08:49:49.300867+0000 mgr.y (mgr.25421) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:51.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:51 smithi094 bash[59677]: cluster 2023-10-09T08:49:51.301883+0000 mgr.y (mgr.25421) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:51.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:51 smithi016 bash[75179]: cluster 2023-10-09T08:49:51.301883+0000 mgr.y (mgr.25421) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:51.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:51 smithi016 bash[77892]: cluster 2023-10-09T08:49:51.301883+0000 mgr.y (mgr.25421) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:52.116 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:49:53.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:53 smithi016 bash[75179]: cluster 2023-10-09T08:49:53.302633+0000 mgr.y (mgr.25421) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:53.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:53 smithi016 bash[77892]: cluster 2023-10-09T08:49:53.302633+0000 mgr.y (mgr.25421) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:53 smithi094 bash[59677]: cluster 2023-10-09T08:49:53.302633+0000 mgr.y (mgr.25421) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:55.258 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:49:55.258 INFO:teuthology.orchestra.run.smithi016.stdout: "name": "mon.a", 2023-10-09T08:49:55.258 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster": "ceph", 2023-10-09T08:49:55.258 INFO:teuthology.orchestra.run.smithi016.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T08:49:55.258 INFO:teuthology.orchestra.run.smithi016.stdout: "admin_socket_mode": "", 2023-10-09T08:49:55.258 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T08:49:55.258 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_client_required": "cephx, none", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_cluster_required": "cephx", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_debug": "true", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_service_required": "cephx", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_supported": "", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio": "true", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T08:49:55.259 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio_reap_max": "16", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_async_discard": "false", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_block_size": "4096", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_aio": "false", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_enable_discard": "false", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_flock_retry": "3", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T08:49:55.260 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_inject_crash": "0", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_ioring": "false", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_ioring_hipri": "false", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_type": "", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_allocator": "hybrid", 2023-10-09T08:49:55.261 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_buffered_io": "true", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T08:49:55.262 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_replay_recovery": "false", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_sync_write": "false", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_allocator": "hybrid", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T08:49:55.263 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_create": "true", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_db_create": "false", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_db_path": "", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_db_size": "0", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_path": "", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T08:49:55.264 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_size": "107374182400", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_wal_create": "false", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_wal_path": "", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs": "true", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T08:49:55.265 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_autotune": "true", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_size": "0", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T08:49:55.266 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_type": "2q", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_clone_cow": "true", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_mode": "none", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T08:49:55.267 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_freelist": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T08:49:55.268 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T08:49:55.269 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fail_eio": "false", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T08:49:55.270 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T08:49:55.271 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T08:49:55.272 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_blob_size": "0", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T08:49:55.273 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T08:49:55.274 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_mem": "512", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_tracing": "false", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T08:49:55.275 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T08:49:55.276 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_cluster_require_version": "2", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_require_signatures": "false", 2023-10-09T08:49:55.277 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_require_version": "2", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_service_require_signatures": "false", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_service_require_version": "2", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_sign_messages": "true", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "chdir": "", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "client_acl_type": "", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "client_asio_thread_count": "2", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "client_cache_mid": "0.750000", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "client_cache_size": "16384", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "client_caps_release_delay": "5", 2023-10-09T08:49:55.278 INFO:teuthology.orchestra.run.smithi016.stdout: "client_check_pool_perm": "true", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_debug_force_sync_read": "false", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_debug_getattr_caps": "false", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_die_on_failed_remount": "false", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_dirsize_rbytes": "true", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_force_lazyio": "false", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_fs": "", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_inject_release_failure": "false", 2023-10-09T08:49:55.279 INFO:teuthology.orchestra.run.smithi016.stdout: "client_max_inline_size": "4096", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mds_namespace": "", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_metadata": "", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mount_gid": "-1", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mount_timeout": "300", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mount_uid": "-1", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mountpoint": "/", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_notify_timeout": "10", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc": "true", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T08:49:55.280 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_max_objects": "1000", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_size": "209715200", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_permissions": "true", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_quota": "true", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_quota_df": "true", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_readahead_max_bytes": "0", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_readahead_max_periods": "4", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_readahead_min": "131072", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_reconnect_stale": "false", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_shutdown_timeout": "30", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_snapdir": ".snap", 2023-10-09T08:49:55.281 INFO:teuthology.orchestra.run.smithi016.stdout: "client_tick_interval": "1", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_trace": "", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_use_faked_inos": "false", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_use_random_mds": "false", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_graylog": "false", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_graylog_port": "12201", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_monitors": "default=true", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_syslog": "false", 2023-10-09T08:49:55.282 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_syslog_level": "info", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster_addr": "-", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster_network": "", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster_network_interface": "", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zlib_isal": "false", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zlib_level": "5", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zstd_level": "1", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T08:49:55.283 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_seastar_smp": "1", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "crush_location": "", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "crush_location_hook": "", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "crush_location_hook_timeout": "10", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "daemonize": "false", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "daos_pool": "tank", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T08:49:55.284 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_alienstore": "0/5", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_asok": "1/5", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_asok_assert_abort": "false", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_auth": "1/5", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_bdev": "1/3", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_bluefs": "1/5", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_bluestore": "1/5", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_buffer": "0/1", 2023-10-09T08:49:55.285 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_cephsqlite": "0/5", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_client": "0/5", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_compressor": "1/5", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_context": "0/1", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_crush": "1/1", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_crypto": "1/5", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_cyanstore": "0/5", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T08:49:55.286 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_dpdk": "1/5", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_eventtrace": "1/5", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_filer": "0/1", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_filestore": "1/3", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_finisher": "1/1", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_fuse": "1/5", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_javaclient": "1/5", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_journal": "1/3", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_journaler": "0/5", 2023-10-09T08:49:55.287 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_kstore": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_leveldb": "4/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_lockdep": "0/1", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mclock": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_balancer": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_locker": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_log": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_migrator": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_memstore": "1/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mgr": "2/5", 2023-10-09T08:49:55.288 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mgrc": "1/5", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mon": "20/20", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_monc": "0/10", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_ms": "1/1", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_none": "0/5", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_objclass": "0/5", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_objectcacher": "0/5", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_objecter": "0/1", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_optracker": "0/5", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_osd": "1/5", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_paxos": "20/20", 2023-10-09T08:49:55.289 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_perfcounter": "1/5", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_prioritycache": "1/5", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rados": "0/5", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd": "0/5", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd_replay": "0/5", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_refs": "0/0", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_reserver": "1/1", 2023-10-09T08:49:55.290 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw": "1/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_access": "1/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_flight": "1/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_sync": "1/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rocksdb": "4/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore": "0/5", 2023-10-09T08:49:55.291 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_backref": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_cache": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_device": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_epm": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_journal": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_lba": "0/5", 2023-10-09T08:49:55.292 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_odata": "0/5", 2023-10-09T08:49:55.293 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_omap": "0/5", 2023-10-09T08:49:55.293 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_onode": "0/5", 2023-10-09T08:49:55.293 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_t": "0/5", 2023-10-09T08:49:55.293 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_tm": "0/5", 2023-10-09T08:49:55.293 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_striper": "0/1", 2023-10-09T08:49:55.293 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_test": "0/5", 2023-10-09T08:49:55.293 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_throttle": "1/1", 2023-10-09T08:49:55.294 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_timer": "0/1", 2023-10-09T08:49:55.294 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_tp": "0/5", 2023-10-09T08:49:55.294 INFO:teuthology.orchestra.run.smithi016.stdout: "device_failure_prediction_mode": "none", 2023-10-09T08:49:55.294 INFO:teuthology.orchestra.run.smithi016.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T08:49:55.294 INFO:teuthology.orchestra.run.smithi016.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T08:49:55.294 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_graylog": "false", 2023-10-09T08:49:55.294 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_journald": "false", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_stderr": "true", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_syslog": "false", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event_tracing": "false", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_http_port": "9926", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_prio_limit": "5", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_sort_metrics": "true", 2023-10-09T08:49:55.295 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_stats_period": "5", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "fake_statfs_for_testing": "0", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "fatal_signal_handlers": "true", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "filer_max_purge_ops": "10", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "filer_max_truncate_ops": "128", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_blackhole": "false", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T08:49:55.296 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_btrfs_snap": "true", 2023-10-09T08:49:55.297 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_caller_concurrency": "10", 2023-10-09T08:49:55.297 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T08:49:55.297 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T08:49:55.297 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T08:49:55.297 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_omap_check": "false", 2023-10-09T08:49:55.297 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T08:49:55.297 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_verify_split": "false", 2023-10-09T08:49:55.298 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_dump_file": "", 2023-10-09T08:49:55.298 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T08:49:55.298 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T08:49:55.298 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fadvise": "true", 2023-10-09T08:49:55.298 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fail_eio": "true", 2023-10-09T08:49:55.298 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T08:49:55.298 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fd_cache_size": "128", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fiemap": "false", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_inject_stall": "0", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_journal_parallel": "false", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_journal_trailing": "false", 2023-10-09T08:49:55.299 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_journal_writeahead": "false", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_kill_at": "0", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T08:49:55.300 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T08:49:55.301 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_merge_threshold": "-10", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_odsync_write": "false", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_omap_backend_path": "", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T08:49:55.302 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T08:49:55.303 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_op_threads": "2", 2023-10-09T08:49:55.303 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_punch_hole": "false", 2023-10-09T08:49:55.303 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T08:49:55.303 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T08:49:55.303 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T08:49:55.303 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T08:49:55.303 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_ops": "50", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_seek_data_hole": "false", 2023-10-09T08:49:55.304 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_sloppy_crc": "false", 2023-10-09T08:49:55.305 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T08:49:55.305 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_splice": "false", 2023-10-09T08:49:55.305 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_split_multiple": "2", 2023-10-09T08:49:55.305 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_split_rand_factor": "20", 2023-10-09T08:49:55.305 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_update_to": "1000", 2023-10-09T08:49:55.305 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T08:49:55.305 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T08:49:55.306 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T08:49:55.306 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T08:49:55.306 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T08:49:55.306 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T08:49:55.306 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T08:49:55.306 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_xfs_extsize": "false", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_zfs_snap": "false", 2023-10-09T08:49:55.307 INFO:teuthology.orchestra.run.smithi016.stdout: "fio_dir": "/tmp/fio", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fsid": "f674ade6-667c-11ee-8db5-212e2dc638e7", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_allow_other": "true", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_big_writes": "true", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_debug": "false", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_default_permissions": "false", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_disable_pagecache": "false", 2023-10-09T08:49:55.308 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_max_write": "0", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_multithreaded": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_require_active_mds": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_set_user_groups": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_splice_move": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_splice_read": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_splice_write": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "gss_target_name": "ceph", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "heartbeat_file": "", 2023-10-09T08:49:55.309 INFO:teuthology.orchestra.run.smithi016.stdout: "heartbeat_inject_failure": "0", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "heartbeat_interval": "5", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "host": "smithi016", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T08:49:55.310 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "inject_early_sigterm": "false", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "jaeger_agent_port": "6799", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "jaeger_tracing_enable": "false", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_aio": "true", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_align_min_size": "65536", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_block_align": "true", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_block_size": "4096", 2023-10-09T08:49:55.311 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_dio": "true", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_discard": "false", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_force_aio": "false", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_ignore_corruption": "false", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_max_write_entries": "100", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_replay_from": "0", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_write_header_frequency": "0", 2023-10-09T08:49:55.312 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_zero_on_create": "false", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "journaler_prefetch_periods": "10", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "journaler_prezero_periods": "5", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "journaler_write_head_interval": "15", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "key": "", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "keyfile": "", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_backend": "rocksdb", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T08:49:55.313 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_max_bytes": "67108864", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_max_ops": "512", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_onode_map_size": "1024", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_sync_transaction": "false", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_block_size": "65536", 2023-10-09T08:49:55.314 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_bloom_size": "0", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_cache_size": "536870912", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_compression": "false", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_log": "", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_max_open_files": "0", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_paranoid": "false", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "librados_thread_count": "2", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "lockdep": "false", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "lockdep_force_backtrace": "false", 2023-10-09T08:49:55.315 INFO:teuthology.orchestra.run.smithi016.stdout: "log_coarse_timestamps": "true", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_flush_on_exit": "false", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_graylog_port": "12201", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_max_new": "1000", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_max_recent": "10000", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_stderr_prefix": "debug ", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_file": "true", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_graylog": "false", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_journald": "false", 2023-10-09T08:49:55.316 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_stderr": "false", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_syslog": "false", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_action_on_write_error": "1", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_alternate_name_max": "8192", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_asio_thread_count": "2", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_export_pin": "true", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T08:49:55.317 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_interval": "10", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_max": "-1", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_max_until": "-1", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_merge_size": "50", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T08:49:55.318 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_mode": "0", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_bits": "3", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_size": "10000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T08:49:55.319 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_mid": "0.700000", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_trim_interval": "1", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T08:49:55.320 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_auth_pins": "false", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_frag": "false", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_scatterstat": "false", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_subtrees": "false", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T08:49:55.321 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_default_dir_hash": "2", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_defer_session_stale": "true", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_max_entries": "0", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_prefetch": "true", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T08:49:55.322 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T08:49:55.323 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_early_reply": "true", 2023-10-09T08:49:55.323 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_enable_op_tracker": "true", 2023-10-09T08:49:55.323 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_enforce_unique_name": "true", 2023-10-09T08:49:55.323 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T08:49:55.323 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T08:49:55.323 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T08:49:55.323 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T08:49:55.381 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T08:49:55.381 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T08:49:55.381 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T08:49:55.381 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T08:49:55.381 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T08:49:55.381 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T08:49:55.381 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_health_dummy": "false", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_join_fs": "", 2023-10-09T08:49:55.382 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_journal_format": "1", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_create_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_export_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_import_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_journal_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_link_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_openc_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_rename_at": "0", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T08:49:55.383 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_max_events": "-1", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_max_segments": "128", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_pause": "false", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_segment_size": "0", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_completed_requests": "100000", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_export_size": "20971520", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_file_recover": "32", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_purge_files": "64", 2023-10-09T08:49:55.384 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_purge_ops": "8192", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_metrics_update_interval": "2", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_min_caps_per_client": "100", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_numa_node": "-1", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T08:49:55.385 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_history_duration": "600", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_history_size": "20", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_log_threshold": "5", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_ping_grace": "15", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_ping_interval": "5", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_max_caps": "30000", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T08:49:55.386 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_replay_interval": "1.000000", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_root_ino_gid": "0", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_root_ino_uid": "0", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T08:49:55.387 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_shutdown_check": "0", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_skip_ino": "0", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_snap_min_uid": "0", 2023-10-09T08:49:55.388 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_snap_rstat": "false", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_symlink_recovery": "true", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_thrash_exports": "0", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_thrash_fragments": "0", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_tick_interval": "5.000000", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_valgrind_exit": "false", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_verify_backtrace": "1", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_verify_scatter": "false", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_wipe_sessions": "false", 2023-10-09T08:49:55.389 INFO:teuthology.orchestra.run.smithi016.stdout: "mempool_debug": "false", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_page_set": "false", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_page_size": "65536", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_client_bytes": "134217728", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_client_messages": "512", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T08:49:55.390 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_disabled_modules": "", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mds_messages": "128", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mon_messages": "128", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_osd_messages": "8192", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_pool": "true", 2023-10-09T08:49:55.391 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_standby_modules": "true", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_stats_period": "5", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_stats_threshold": "5", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_tick_period": "2", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_allow_pool_delete": "true", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T08:49:55.392 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_bytes": "104857600", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T08:49:55.393 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T08:49:55.394 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_journald": "false", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_stderr": "true", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_compact_on_start": "false", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_compact_on_trim": "true", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T08:49:55.395 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cpu_threads": "4", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data_avail_crit": "5", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data_avail_warn": "5", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_dump_json": "false", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T08:49:55.396 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_extra_checks": "false", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_election_default_strategy": "3", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_election_timeout": "5.000000", 2023-10-09T08:49:55.397 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_enable_op_tracker": "true", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_fake_pool_delete": "false", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_force_quorum_join": "false", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_log_update_period": "5", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_max_detail": "50", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_to_clog": "true", 2023-10-09T08:49:55.398 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_host": "[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_host_override": "", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_initial_members": "", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T08:49:55.399 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_lease": "5.000000", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_log_full_interval": "50", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_log_max": "10000", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_log_max_summary": "50", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_log_epochs": "500", 2023-10-09T08:49:55.400 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T08:49:55.401 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T08:49:55.401 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_osd": "10000", 2023-10-09T08:49:55.401 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T08:49:55.401 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T08:49:55.401 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T08:49:55.401 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T08:49:55.402 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T08:49:55.402 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T08:49:55.402 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_memory_autotune": "true", 2023-10-09T08:49:55.402 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_memory_target": "2147483648", 2023-10-09T08:49:55.402 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_digest_period": "5", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_complaint_time": "30", 2023-10-09T08:49:55.403 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_duration": "600", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_size": "20", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_log_threshold": "5", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T08:49:55.404 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_cache_size": "500", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T08:49:55.405 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T08:49:55.406 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_report_timeout": "900", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T08:49:55.407 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_max_osds": "4", 2023-10-09T08:49:55.408 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_interval": "86400", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_max_keys": "100", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_timeout": "300", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_session_timeout": "300", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_smart_report_timeout": "5", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T08:49:55.409 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_pool_size": "4", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_debug": "false", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T08:49:55.410 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_tick_interval": "5", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T08:49:55.411 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_older_version": "true", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T08:49:55.412 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "monmap": "", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_tracing_enabled": "false", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_op_threads": "3", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T08:49:55.413 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_cm": "false", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_device_name": "", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T08:49:55.414 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_sl": "3", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_type": "ib", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_reap_threshold": "5", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_before_connect": "false", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_ipv4": "true", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_ipv6": "false", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_msgr1": "true", 2023-10-09T08:49:55.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_msgr2": "true", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_port_max": "7568", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_port_min": "6800", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_retry_count": "3", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_retry_delay": "5", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_client": "false", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_mds": "false", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_mgr": "false", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_mon": "false", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_osd": "false", 2023-10-09T08:49:55.416 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_client_mode": "crc secure", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_cluster_type": "", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_compress_secure": "false", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_crc_data": "true", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_crc_header": "true", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_bug": "true", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_old_message": "true", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T08:49:55.417 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_hugepages": "", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_lro": "true", 2023-10-09T08:49:55.418 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_pmd": "", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_port_id": "0", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dump_on_send": "false", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T08:49:55.419 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_delay_type": "", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_network_congestion": "0", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_socket_failures": "0", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_max_accept_failures": "4", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_max_backoff": "15.000000", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_osd_compress_mode": "none", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T08:49:55.420 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_pq_min_cost": "65536", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_public_type": "", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_service_mode": "crc secure", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_nodelay": "true", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_type": "async+posix", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "no_config_file": "false", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T08:49:55.421 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_inflight_ops": "1024", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_timeout": "10.000000", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "objectstore_blackhole": "false", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "openssl_engine_opts": "", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T08:49:55.422 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_max_ops": "4", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_slop": "0.020000", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_asio_thread_count": "2", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backfill_scan_max": "512", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backfill_scan_min": "64", 2023-10-09T08:49:55.423 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backoff_on_peering": "false", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_beacon_report_interval": "300", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_duration": "30", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_blkin_trace_all": "false", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T08:49:55.424 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_update_on_start": "true", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_message_cap": "256", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_op_priority": "63", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_watch_timeout": "30", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_command_max_records": "256", 2023-10-09T08:49:55.425 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_compact_on_start": "false", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_update_on_start": "true", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T08:49:55.426 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T08:49:55.427 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T08:49:55.427 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T08:49:55.427 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T08:49:55.427 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T08:49:55.427 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T08:49:55.427 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T08:49:55.427 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T08:49:55.428 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T08:49:55.428 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T08:49:55.428 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_op_order": "false", 2023-10-09T08:49:55.428 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T08:49:55.428 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T08:49:55.428 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T08:49:55.429 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T08:49:55.429 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_shutdown": "false", 2023-10-09T08:49:55.429 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T08:49:55.429 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T08:49:55.429 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T08:49:55.429 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T08:49:55.430 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T08:49:55.430 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T08:49:55.430 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T08:49:55.430 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T08:49:55.430 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T08:49:55.431 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T08:49:55.431 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T08:49:55.431 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T08:49:55.431 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T08:49:55.431 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T08:49:55.431 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_default_notify_timeout": "30", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_enable_op_tracker": "true", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T08:49:55.432 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T08:49:55.433 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T08:49:55.433 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T08:49:55.433 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_info": "true", 2023-10-09T08:49:55.433 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_shutdown": "true", 2023-10-09T08:49:55.433 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T08:49:55.433 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T08:49:55.433 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T08:49:55.434 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T08:49:55.434 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T08:49:55.434 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_function_tracing": "false", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_grace": "20", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_interval": "6", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_stale": "600", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T08:49:55.435 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_journal_size": "5120", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_kill_backfill_at": "0", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_cache_size": "50", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_dedup": "true", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_message_max": "40", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T08:49:55.436 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_attr_name_len": "100", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_attr_size": "0", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_backfills": "1", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_markdown_count": "5", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_markdown_period": "600", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_object_name_len": "2048", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_object_size": "134217728", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T08:49:55.437 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pgls": "1024", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_push_cost": "8388608", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_push_objects": "10", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_scrubs": "1", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T08:49:55.438 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_write_size": "90", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_profile": "balanced", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T08:49:55.439 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_base": "805306368", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T08:49:55.440 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_target": "4294967296", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_target_autotune": "false", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_min_recovery_priority": "0", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_report_interval": "5", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T08:49:55.441 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_num_cache_shards": "32", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_numa_node": "-1", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objecter_finishers": "1", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objectstore": "bluestore", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objectstore_fuse": "false", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objectstore_tracing": "false", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T08:49:55.442 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_duration": "600", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_size": "20", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_log_threshold": "5", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_shards": "0", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T08:49:55.443 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_thread_timeout": "15", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_open_classes_on_start": "true", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_os_flags": "0", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_peering_op_priority": "255", 2023-10-09T08:49:55.444 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_delete_priority": "5", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T08:49:55.445 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T08:49:55.446 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T08:49:55.446 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T08:49:55.446 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T08:49:55.446 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_crimson": "false", 2023-10-09T08:49:55.446 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T08:49:55.446 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T08:49:55.446 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flags": "0", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_min_size": "0", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T08:49:55.447 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_size": "3", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_type": "replicated", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T08:49:55.448 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_cost": "20971520", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_active": "0", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T08:49:55.449 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_op_priority": "3", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_priority": "5", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_repair_during_recovery": "false", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T08:49:55.450 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_cost": "52428800", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_end_hour": "0", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T08:49:55.451 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_priority": "5", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T08:49:55.452 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_skip_data_digest": "false", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_smart_report_timeout": "5", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_priority": "5", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T08:49:55.453 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_target_transaction_size": "30", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T08:49:55.454 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tracing": "false", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_use_stale_snap": "false", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T08:49:55.455 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_kill_at": "0", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_max_join_drift": "10", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_min": "500", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_min_wait": "0.050000", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_service_trim_max": "500", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_service_trim_min": "250", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_stash_full_interval": "25", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_trim_max": "500", 2023-10-09T08:49:55.456 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_trim_min": "250", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "perf": "true", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "pid_file": "", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "public_addr": "-", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "public_addrv": "", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "public_bind_addr": "-", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "public_network": "172.21.0.0/20", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "public_network_interface": "", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "qat_compressor_enabled": "false", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T08:49:55.457 INFO:teuthology.orchestra.run.smithi016.stdout: "rados_mon_op_timeout": "0", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rados_osd_op_timeout": "0", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rados_tracing": "false", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_atime_update_interval": "60", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache": "true", 2023-10-09T08:49:55.458 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_policy": "writearound", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_size": "33554432", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T08:49:55.459 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T08:49:55.461 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_compression_hint": "none", 2023-10-09T08:49:55.461 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T08:49:55.461 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T08:49:55.461 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_clone_format": "auto", 2023-10-09T08:49:55.461 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_data_pool": "", 2023-10-09T08:49:55.461 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_features": "61", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_format": "2", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_map_options": "", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_order": "22", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_pool": "rbd", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_stripe_count": "0", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T08:49:55.462 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_io_scheduler": "simple", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T08:49:55.463 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_order": "24", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_pool": "", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_splay_width": "4", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T08:49:55.464 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T08:49:55.465 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T08:49:55.466 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_op_threads": "1", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_plugins": "", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T08:49:55.467 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T08:49:55.468 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T08:49:55.469 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_tracing": "false", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_validate_names": "true", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_validate_pool": "true", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "restapi_base_url": "", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "restapi_log_level": "", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_admin_entry": "admin", 2023-10-09T08:49:55.470 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_backend_store": "rados", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_barbican_url": "", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_beast_enable_async": "true", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T08:49:55.471 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T08:49:55.472 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T08:49:55.472 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T08:49:55.472 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cache_enabled": "true", 2023-10-09T08:49:55.472 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T08:49:55.472 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T08:49:55.472 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_config_store": "rados", 2023-10-09T08:49:55.473 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_content_length_compat": "false", 2023-10-09T08:49:55.473 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T08:49:55.473 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T08:49:55.473 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T08:49:55.473 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cross_domain_policy": "", 2023-10-09T08:49:55.474 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T08:49:55.474 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T08:49:55.474 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T08:49:55.474 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T08:49:55.474 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T08:49:55.474 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T08:49:55.474 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T08:49:55.475 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T08:49:55.476 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T08:49:55.476 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T08:49:55.476 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T08:49:55.476 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T08:49:55.476 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T08:49:55.476 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T08:49:55.476 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T08:49:55.477 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T08:49:55.477 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T08:49:55.477 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T08:49:55.477 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T08:49:55.477 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T08:49:55.477 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T08:49:55.477 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T08:49:55.478 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T08:49:55.479 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T08:49:55.479 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T08:49:55.479 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T08:49:55.479 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T08:49:55.479 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T08:49:55.479 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T08:49:55.480 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T08:49:55.480 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T08:49:55.480 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T08:49:55.480 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T08:49:55.480 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T08:49:55.481 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T08:49:55.481 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T08:49:55.481 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T08:49:55.481 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_window": "30", 2023-10-09T08:49:55.481 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T08:49:55.481 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T08:49:55.481 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T08:49:55.482 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T08:49:55.482 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T08:49:55.482 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T08:49:55.482 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T08:49:55.482 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T08:49:55.482 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T08:49:55.483 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T08:49:55.483 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T08:49:55.483 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T08:49:55.483 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T08:49:55.483 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T08:49:55.483 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T08:49:55.483 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T08:49:55.484 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dns_name": "", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dns_s3website_name": "", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_ops_log": "false", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_static_website": "false", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_usage_log": "false", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T08:49:55.485 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_expose_bucket": "false", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_extended_http_attrs": "", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_filter": "none", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_objs": "32", 2023-10-09T08:49:55.486 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_init_timeout": "300", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T08:49:55.487 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_password": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_project": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_token": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_user": "", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_api_version": "2", 2023-10-09T08:49:55.488 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_url": "", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T08:49:55.489 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_objs": "32", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_worker": "3", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T08:49:55.490 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_http_headers": "", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T08:49:55.491 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_attr_size": "0", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_listing_results": "1000", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_notify_retries": "10", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T08:49:55.492 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T08:49:55.493 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T08:49:55.494 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_num_control_oids": "8", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_numa_node": "-1", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_op_tracing": "false", 2023-10-09T08:49:55.495 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_opa_token": "", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_opa_url": "", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_rados": "false", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T08:49:55.496 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_print_continue": "true", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rados_tracing": "false", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_realm": "", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_realm_id": "", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T08:49:55.497 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_region": "", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_request_uri": "", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T08:49:55.498 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_resolve_cname": "false", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_run_sync_thread": "true", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_script_uri": "", 2023-10-09T08:49:55.499 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_service_provider_name": "", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_client_id": "", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_client_secret": "", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_entry": "sts", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_key": "sts", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T08:49:55.500 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_auth_url": "", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_custom_header": "", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_need_stats": "true", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_tenant_name": "", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_url": "", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T08:49:55.501 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_lease_period": "120", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T08:49:55.502 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_thread_pool_size": "512", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_comment": "", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_createby": "", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_encoding": "", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_flag": "false", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_origin": "", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_tracker": "", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T08:49:55.503 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_max_shards": "32", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_use_opa_authz": "false", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T08:49:55.504 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_unique_email": "true", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_verify_ssl": "true", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zone": "", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zone_id": "", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zonegroup": "", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zonegroup_id": "", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T08:49:55.505 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_block_size": "4096", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T08:49:55.506 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_partition_filters": "false", 2023-10-09T08:49:55.507 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_perf": "false", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "run_dir": "/var/run/ceph", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_block_create": "true", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_device_size": "53687091200", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T08:49:55.508 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_main_device_type": "SSD", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_segment_size": "67108864", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "set_keepcaps": "false", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "setgroup": "ceph", 2023-10-09T08:49:55.509 INFO:teuthology.orchestra.run.smithi016.stdout: "setuser": "ceph", 2023-10-09T08:49:55.510 INFO:teuthology.orchestra.run.smithi016.stdout: "setuser_match_path": "", 2023-10-09T08:49:55.510 INFO:teuthology.orchestra.run.smithi016.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T08:49:55.510 INFO:teuthology.orchestra.run.smithi016.stdout: "thp": "false", 2023-10-09T08:49:55.510 INFO:teuthology.orchestra.run.smithi016.stdout: "threadpool_default_timeout": "60", 2023-10-09T08:49:55.510 INFO:teuthology.orchestra.run.smithi016.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T08:49:55.510 INFO:teuthology.orchestra.run.smithi016.stdout: "throttler_perf_counter": "true" 2023-10-09T08:49:55.510 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:49:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:55 smithi016 bash[75179]: cluster 2023-10-09T08:49:55.303530+0000 mgr.y (mgr.25421) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:55 smithi016 bash[77892]: cluster 2023-10-09T08:49:55.303530+0000 mgr.y (mgr.25421) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:55.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:55 smithi094 bash[59677]: cluster 2023-10-09T08:49:55.303530+0000 mgr.y (mgr.25421) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:56.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:49:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:49:55] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.33.4" 2023-10-09T08:49:56.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T08:49:57.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:57 smithi094 bash[59677]: cluster 2023-10-09T08:49:57.304309+0000 mgr.y (mgr.25421) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:57.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:57 smithi016 bash[75179]: cluster 2023-10-09T08:49:57.304309+0000 mgr.y (mgr.25421) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:49:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:57 smithi016 bash[77892]: cluster 2023-10-09T08:49:57.304309+0000 mgr.y (mgr.25421) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:00.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:49:59 smithi094 bash[59677]: cluster 2023-10-09T08:49:59.304912+0000 mgr.y (mgr.25421) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:00.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:49:59 smithi016 bash[75179]: cluster 2023-10-09T08:49:59.304912+0000 mgr.y (mgr.25421) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:00.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:49:59 smithi016 bash[77892]: cluster 2023-10-09T08:49:59.304912+0000 mgr.y (mgr.25421) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:00.909 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:01.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:00 smithi094 bash[59677]: cluster 2023-10-09T08:50:00.000125+0000 mon.a (mon.0) 491 : cluster 1 overall HEALTH_OK 2023-10-09T08:50:01.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:00 smithi016 bash[75179]: cluster 2023-10-09T08:50:00.000125+0000 mon.a (mon.0) 491 : cluster 1 overall HEALTH_OK 2023-10-09T08:50:01.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:00 smithi016 bash[77892]: cluster 2023-10-09T08:50:00.000125+0000 mon.a (mon.0) 491 : cluster 1 overall HEALTH_OK 2023-10-09T08:50:02.177 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:01 smithi016 bash[75179]: cluster 2023-10-09T08:50:01.305891+0000 mgr.y (mgr.25421) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:02.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:01 smithi016 bash[77892]: cluster 2023-10-09T08:50:01.305891+0000 mgr.y (mgr.25421) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:02.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:01 smithi094 bash[59677]: cluster 2023-10-09T08:50:01.305891+0000 mgr.y (mgr.25421) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:03.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:03 smithi094 bash[59677]: cluster 2023-10-09T08:50:03.306658+0000 mgr.y (mgr.25421) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:03.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:03 smithi016 bash[75179]: cluster 2023-10-09T08:50:03.306658+0000 mgr.y (mgr.25421) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:03.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:03 smithi016 bash[77892]: cluster 2023-10-09T08:50:03.306658+0000 mgr.y (mgr.25421) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:04.118 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a: {} 2023-10-09T08:50:04.134 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b: {} 2023-10-09T08:50:04.153 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c: {} 2023-10-09T08:50:04.155 INFO:teuthology.orchestra.run.smithi016.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T08:50:04.156 INFO:teuthology.orchestra.run.smithi016.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T08:50:04.156 INFO:teuthology.orchestra.run.smithi016.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T08:50:05.032 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T08:50:05.033 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T08:50:05.034 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T08:50:05.035 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T08:50:05.035 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T08:50:05.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T08:50:05.036 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T08:50:05.036 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:50:05.036 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:50:05.037 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T08:50:05.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-10-09T08:50:05.037 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T08:50:05.037 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:05.038 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T08:50:05.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:50:05.038 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T08:50:05.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T08:50:05.039 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T08:50:05.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T08:50:05.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:05.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:05.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:05.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:04 smithi016 bash[75179]: audit 2023-10-09T08:50:04.097447+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.16:0/3403141127' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:50:05.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:04 smithi016 bash[75179]: audit 2023-10-09T08:50:04.205282+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:05.186 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:04 smithi016 bash[77892]: audit 2023-10-09T08:50:04.097447+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.16:0/3403141127' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:50:05.187 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:04 smithi016 bash[77892]: audit 2023-10-09T08:50:04.205282+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:05.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:05.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:04 smithi094 bash[59677]: audit 2023-10-09T08:50:04.097447+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.16:0/3403141127' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:50:05.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:04 smithi094 bash[59677]: audit 2023-10-09T08:50:04.205282+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:05.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:05.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:05.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:05.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:05.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:05.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:05.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:05.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:05.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:05.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:05.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:05.846 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:05.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:05.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:05.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:05.996 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:50:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:50:05] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.33.4" 2023-10-09T08:50:05.996 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:05 smithi016 bash[75179]: cluster 2023-10-09T08:50:05.307258+0000 mgr.y (mgr.25421) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:05.996 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:05 smithi016 bash[77892]: cluster 2023-10-09T08:50:05.307258+0000 mgr.y (mgr.25421) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:06.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:06.120 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:06.135 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:05 smithi094 bash[59677]: cluster 2023-10-09T08:50:05.307258+0000 mgr.y (mgr.25421) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:06.158 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:06.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:06.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:06.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:06.398 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:06.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:06.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:06.537 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:06.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:06.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:06.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:06.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:06.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:06.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:06.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:06.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:06.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:07.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:07.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:07.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:07.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:07.224 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:07.260 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:07.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:07.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:07.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:07.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:07.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:07.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:07.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:07.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:07.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:07.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:07.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:07.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:07 smithi016 bash[75179]: cluster 2023-10-09T08:50:07.308095+0000 mgr.y (mgr.25421) 369 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:07.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:07 smithi016 bash[77892]: cluster 2023-10-09T08:50:07.308095+0000 mgr.y (mgr.25421) 369 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:07.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:07.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:07.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:07.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:07 smithi094 bash[59677]: cluster 2023-10-09T08:50:07.308095+0000 mgr.y (mgr.25421) 369 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:07.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:07.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:07.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:07.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:07.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:07.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:08.040 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:08.076 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:08.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:08.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:08.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:08.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:08.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:08.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:08.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:08.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:08.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:08.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:08.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:08.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:08.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:08.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:08.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:08.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:08.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:08.750 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:08.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:08.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:08.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:08.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:08.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:08.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:09.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:09.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:09.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:09.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:09.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:09.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:09.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:09.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:09.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:09.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:09.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:09.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:09.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:09.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:09.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:09.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:09.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:09.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:09.841 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:09.841 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:09.842 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:09.842 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:09.843 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:09.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:09.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:09.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:09.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:09.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:09 smithi016 bash[75179]: cluster 2023-10-09T08:50:09.308694+0000 mgr.y (mgr.25421) 370 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:09.968 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:09 smithi016 bash[77892]: cluster 2023-10-09T08:50:09.308694+0000 mgr.y (mgr.25421) 370 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:10.068 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:10.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:10.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:10.206 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:10.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:09 smithi094 bash[59677]: cluster 2023-10-09T08:50:09.308694+0000 mgr.y (mgr.25421) 370 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:10.238 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:10.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:10.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:10.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:10.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:10.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:10.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:10.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:10.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:10.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:10.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:10.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:10.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:10.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:10.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:10.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:10.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:10.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:10.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:10.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:11.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:11.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:11.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:11.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:11.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:11.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:11.306 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:11.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:11.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:11.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:11.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:11.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:11 smithi094 bash[59677]: cluster 2023-10-09T08:50:11.309616+0000 mgr.y (mgr.25421) 371 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:11.542 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:11.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:11.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:11.709 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:11.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:11 smithi016 bash[75179]: cluster 2023-10-09T08:50:11.309616+0000 mgr.y (mgr.25421) 371 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:11.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:11 smithi016 bash[77892]: cluster 2023-10-09T08:50:11.309616+0000 mgr.y (mgr.25421) 371 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:11.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:11.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:11.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:11.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:11.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:12.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:12.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:12.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:12.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:12.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:12.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:12.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:12.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:12.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:12.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:12.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:12.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:12.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:12.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:12.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:12.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:12.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:12.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:12.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:12.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:12.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:12.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:12.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:12.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:12.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:12.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:13.012 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:13.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:13.146 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:13.182 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:13.285 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:13.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:13.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:13.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:13.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:13 smithi094 bash[59677]: cluster 2023-10-09T08:50:13.310424+0000 mgr.y (mgr.25421) 372 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:13.454 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:13.557 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:13.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:13.647 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:13 smithi016 bash[77892]: cluster 2023-10-09T08:50:13.310424+0000 mgr.y (mgr.25421) 372 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:13.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:13 smithi016 bash[75179]: cluster 2023-10-09T08:50:13.310424+0000 mgr.y (mgr.25421) 372 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:13.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:13.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:13.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:13.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:13.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:13.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:13.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:13.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:14.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:14.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:14.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:14.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:14.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:14.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:14.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:14.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:14.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:14.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:14.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:14.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:14.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:14.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:14.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:14.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:14.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:14.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:14.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:14.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:14.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:14.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:14.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:14.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:15.012 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:15.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:15.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:15.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:15.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:15.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:15.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:15.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:15.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:15.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:15 smithi016 bash[75179]: cluster 2023-10-09T08:50:15.311326+0000 mgr.y (mgr.25421) 373 : cluster 0 pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:15.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:15 smithi016 bash[77892]: cluster 2023-10-09T08:50:15.311326+0000 mgr.y (mgr.25421) 373 : cluster 0 pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:15.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:15.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:15.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:15.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:15.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:15.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:15.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:15 smithi094 bash[59677]: cluster 2023-10-09T08:50:15.311326+0000 mgr.y (mgr.25421) 373 : cluster 0 pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:15.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:15.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:15.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:15.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:15.931 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:50:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:50:15] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.33.4" 2023-10-09T08:50:15.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:15.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:15.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:16.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:16.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:16.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:16.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:16.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:16.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:16.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:16.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:16.353 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:16.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:16.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:16.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:16.493 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:16.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:16.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:16.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:16.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:16.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:16.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:16.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:16.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:16.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:16.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:16.990 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:17.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:17.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:17.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:17.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:17.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:17.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:17.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:17.533 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:17 smithi094 bash[59677]: cluster 2023-10-09T08:50:17.312224+0000 mgr.y (mgr.25421) 374 : cluster 0 pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:17.564 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:17.654 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:17 smithi016 bash[75179]: cluster 2023-10-09T08:50:17.312224+0000 mgr.y (mgr.25421) 374 : cluster 0 pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:17.655 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:17 smithi016 bash[77892]: cluster 2023-10-09T08:50:17.312224+0000 mgr.y (mgr.25421) 374 : cluster 0 pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:17.668 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.697 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:17.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:17.801 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:17.832 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:17.936 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:17.968 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:17.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:18.072 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:18.104 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:18.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:18.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:18.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:18.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:18.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:18.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:18.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:18.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:18.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:18.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:18.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:18.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:18.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:18.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:18.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:18.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:18.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:18.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:18.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:18.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:18.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:18.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:18.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:19.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:19.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:19.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:19.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:19.169 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:50:19.169 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T08:50:19.169 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-10-09T08:50:19.170 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:50:19.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:19.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:19.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:19.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:19.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:19.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:19.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:19.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:19.650 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:19.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:19.754 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:19.785 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:19.888 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:19.918 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:19.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:20.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:20.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:20.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:20.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:20.162 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:19 smithi016 bash[75179]: audit 2023-10-09T08:50:19.205276+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:20.162 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:19 smithi016 bash[75179]: cluster 2023-10-09T08:50:19.312719+0000 mgr.y (mgr.25421) 375 : cluster 0 pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:20.162 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:19 smithi016 bash[77892]: audit 2023-10-09T08:50:19.205276+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:20.162 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:19 smithi016 bash[77892]: cluster 2023-10-09T08:50:19.312719+0000 mgr.y (mgr.25421) 375 : cluster 0 pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:20.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:20.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:20.238 INFO:teuthology.orchestra.run.smithi016.stdout:moved 48 / 219 (21.9178%) 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:avg 27.375 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:stddev 4.8202 -> 10.0491 (expected baseline 4.89419) 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:min osd.2 with 19 -> 26 pgs (0.694064 -> 0.949772 * mean) 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:max osd.4 with 33 -> 24 pgs (1.20548 -> 0.876712 * mean) 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:oload 110 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:max_change 3 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:max_change_osds 4 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:average_utilization 0.0018 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:overload_utilization 0.0020 2023-10-09T08:50:20.239 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 weight 1.0000 -> 0.5437 2023-10-09T08:50:20.240 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 weight 1.0000 -> 0.5447 2023-10-09T08:50:20.240 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 weight 1.0000 -> 0.5595 2023-10-09T08:50:20.240 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 weight 1.0000 -> 0.5661 2023-10-09T08:50:20.242 INFO:teuthology.orchestra.run.smithi016.stderr:no change 2023-10-09T08:50:20.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:19 smithi094 bash[59677]: audit 2023-10-09T08:50:19.205276+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:20.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:19 smithi094 bash[59677]: cluster 2023-10-09T08:50:19.312719+0000 mgr.y (mgr.25421) 375 : cluster 0 pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:20.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:20.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:20.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:20.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:20.434 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:20.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:20.501 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:50:20 smithi016 bash[71225]: debug 2023-10-09T08:50:20.231+0000 7f5681439700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-09T08:50:20.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:20.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:20.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:20.703 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:20.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:20.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:20.844 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:20.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:20.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:20.961 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:20 smithi094 bash[59677]: audit 2023-10-09T08:50:20.234768+0000 mgr.y (mgr.25421) 376 : audit 0 from='client.34737 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:50:20.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:20.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:20.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:21.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:21.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:21.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:21.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:21.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:21.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:20 smithi016 bash[75179]: audit 2023-10-09T08:50:20.234768+0000 mgr.y (mgr.25421) 376 : audit 0 from='client.34737 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:50:21.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:20 smithi016 bash[77892]: audit 2023-10-09T08:50:20.234768+0000 mgr.y (mgr.25421) 376 : audit 0 from='client.34737 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:50:21.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:21.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:21.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T08:50:21.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:21.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:21.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:21.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:21.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:21.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:21.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:21.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:21.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:21.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:21.746 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:21.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:21.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:21.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:21.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:21.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:21.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:22.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:22.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:22.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:22.151 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:21 smithi094 bash[59677]: cluster 2023-10-09T08:50:21.313411+0000 mgr.y (mgr.25421) 377 : cluster 0 pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:22.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:22.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:21 smithi016 bash[75179]: cluster 2023-10-09T08:50:21.313411+0000 mgr.y (mgr.25421) 377 : cluster 0 pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:22.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:21 smithi016 bash[77892]: cluster 2023-10-09T08:50:21.313411+0000 mgr.y (mgr.25421) 377 : cluster 0 pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:22.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:22.308 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:22.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:22.445 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:22.549 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:22.586 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:22.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:22.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:22.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:22.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:22.848 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:50:22.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:22.859 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T08:50:22.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-10-09T08:50:22.875 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:50:22.915 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:50:22.940 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:50:22.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:22.963 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:50:22.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:22.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:22.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:22 smithi094 bash[59677]: audit 2023-10-09T08:50:22.815771+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.16:0/223167840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:50:23.004 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:50:23.034 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:50:23.065 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:50:23.066 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.067 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.067 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.067 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.067 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.067 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.067 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.067 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:50:23.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:23.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:23.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:23.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:23.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:22 smithi016 bash[75179]: audit 2023-10-09T08:50:22.815771+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.16:0/223167840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:50:23.237 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:22 smithi016 bash[77892]: audit 2023-10-09T08:50:22.815771+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.16:0/223167840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:50:23.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:23.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:23.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:23.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:23.384 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:23.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:23.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:23.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:23.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:23.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:23.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:23.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:23.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:23.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:23.749 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:23.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:23.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:23.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:23.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:23.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:23.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:23.938 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T08:50:23.939 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_0' created 2023-10-09T08:50:24.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:24.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:24.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:24.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:24.162 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:23 smithi016 bash[75179]: audit 2023-10-09T08:50:23.197201+0000 mon.a (mon.0) 494 : audit 1 from='client.? 172.21.15.16:0/2255174051' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:50:24.162 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:23 smithi016 bash[75179]: audit 2023-10-09T08:50:23.233719+0000 mon.a (mon.0) 495 : audit 1 from='client.? 172.21.15.16:0/411050417' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T08:50:24.162 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:23 smithi016 bash[75179]: cluster 2023-10-09T08:50:23.313948+0000 mgr.y (mgr.25421) 378 : cluster 0 pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:24.163 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:23 smithi016 bash[77892]: audit 2023-10-09T08:50:23.197201+0000 mon.a (mon.0) 494 : audit 1 from='client.? 172.21.15.16:0/2255174051' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:50:24.163 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:23 smithi016 bash[77892]: audit 2023-10-09T08:50:23.233719+0000 mon.a (mon.0) 495 : audit 1 from='client.? 172.21.15.16:0/411050417' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T08:50:24.163 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:23 smithi016 bash[77892]: cluster 2023-10-09T08:50:23.313948+0000 mgr.y (mgr.25421) 378 : cluster 0 pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:24.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:23 smithi094 bash[59677]: audit 2023-10-09T08:50:23.197201+0000 mon.a (mon.0) 494 : audit 1 from='client.? 172.21.15.16:0/2255174051' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:50:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:23 smithi094 bash[59677]: audit 2023-10-09T08:50:23.233719+0000 mon.a (mon.0) 495 : audit 1 from='client.? 172.21.15.16:0/411050417' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T08:50:24.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:23 smithi094 bash[59677]: cluster 2023-10-09T08:50:23.313948+0000 mgr.y (mgr.25421) 378 : cluster 0 pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:24.284 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:24.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:24.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:24.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:24.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:24.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:24.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:24.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:24.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:24.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:24.818 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:24.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:24.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:24.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:24.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:24.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:25.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: cluster 2023-10-09T08:50:23.916682+0000 mon.a (mon.0) 496 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:50:25.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: cluster 2023-10-09T08:50:23.926289+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: audit 2023-10-09T08:50:23.929981+0000 mon.a (mon.0) 497 : audit 1 from='client.? 172.21.15.16:0/2255174051' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: audit 2023-10-09T08:50:23.930214+0000 mon.a (mon.0) 498 : audit 1 from='client.? 172.21.15.16:0/411050417' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: cluster 2023-10-09T08:50:23.930312+0000 mon.a (mon.0) 499 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: cluster 2023-10-09T08:50:23.933448+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: cluster 2023-10-09T08:50:23.934525+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[75179]: cluster 2023-10-09T08:50:23.934673+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: cluster 2023-10-09T08:50:23.916682+0000 mon.a (mon.0) 496 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: cluster 2023-10-09T08:50:23.926289+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: audit 2023-10-09T08:50:23.929981+0000 mon.a (mon.0) 497 : audit 1 from='client.? 172.21.15.16:0/2255174051' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:50:25.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: audit 2023-10-09T08:50:23.930214+0000 mon.a (mon.0) 498 : audit 1 from='client.? 172.21.15.16:0/411050417' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T08:50:25.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: cluster 2023-10-09T08:50:23.930312+0000 mon.a (mon.0) 499 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-10-09T08:50:25.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: cluster 2023-10-09T08:50:23.933448+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: cluster 2023-10-09T08:50:23.934525+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:24 smithi016 bash[77892]: cluster 2023-10-09T08:50:23.934673+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:25.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:25.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:25.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:25.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:25.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: cluster 2023-10-09T08:50:23.916682+0000 mon.a (mon.0) 496 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: cluster 2023-10-09T08:50:23.926289+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: audit 2023-10-09T08:50:23.929981+0000 mon.a (mon.0) 497 : audit 1 from='client.? 172.21.15.16:0/2255174051' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: audit 2023-10-09T08:50:23.930214+0000 mon.a (mon.0) 498 : audit 1 from='client.? 172.21.15.16:0/411050417' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: cluster 2023-10-09T08:50:23.930312+0000 mon.a (mon.0) 499 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: cluster 2023-10-09T08:50:23.933448+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: cluster 2023-10-09T08:50:23.934525+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:24 smithi094 bash[59677]: cluster 2023-10-09T08:50:23.934673+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T08:50:25.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:25.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:25.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:25.480 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:25.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:25.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:25.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:25.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:25.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:25.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T08:50:25.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:25.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:25.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:25.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:25.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:25.892 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:50:25] "GET /metrics HTTP/1.1" 200 33877 "" "Prometheus/2.33.4" 2023-10-09T08:50:25.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:25.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:25.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:25.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:50:26.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:25 smithi094 bash[59677]: cluster 2023-10-09T08:50:24.931767+0000 mon.a (mon.0) 500 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-09T08:50:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:25 smithi094 bash[59677]: cluster 2023-10-09T08:50:24.932684+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:25 smithi094 bash[59677]: cluster 2023-10-09T08:50:24.932764+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:25 smithi094 bash[59677]: cluster 2023-10-09T08:50:24.933643+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:25 smithi094 bash[59677]: cluster 2023-10-09T08:50:25.314911+0000 mgr.y (mgr.25421) 379 : cluster 0 pgmap v313: 89 pgs: 4 creating+activating, 5 unknown, 80 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:25 smithi094 bash[59677]: cluster 2023-10-09T08:50:25.380850+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:26.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:26.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:26.164 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[77892]: cluster 2023-10-09T08:50:24.931767+0000 mon.a (mon.0) 500 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-09T08:50:26.165 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[77892]: cluster 2023-10-09T08:50:24.932684+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.165 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[77892]: cluster 2023-10-09T08:50:24.932764+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.165 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[77892]: cluster 2023-10-09T08:50:24.933643+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.165 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[77892]: cluster 2023-10-09T08:50:25.314911+0000 mgr.y (mgr.25421) 379 : cluster 0 pgmap v313: 89 pgs: 4 creating+activating, 5 unknown, 80 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:26.166 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[77892]: cluster 2023-10-09T08:50:25.380850+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.166 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[75179]: cluster 2023-10-09T08:50:24.931767+0000 mon.a (mon.0) 500 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-09T08:50:26.166 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[75179]: cluster 2023-10-09T08:50:24.932684+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.166 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[75179]: cluster 2023-10-09T08:50:24.932764+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[75179]: cluster 2023-10-09T08:50:24.933643+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[75179]: cluster 2023-10-09T08:50:25.314911+0000 mgr.y (mgr.25421) 379 : cluster 0 pgmap v313: 89 pgs: 4 creating+activating, 5 unknown, 80 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:26.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:25 smithi016 bash[75179]: cluster 2023-10-09T08:50:25.380850+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T08:50:26.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:26.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:26.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:26.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:26.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:26.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:26.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:26.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:26.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:26.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:26.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:26.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:26.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:26.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:26.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:26.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:26.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:26.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:26.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:26.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:26.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:26.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:26.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:26.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:26 smithi094 bash[59677]: cluster 2023-10-09T08:50:25.945642+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:26.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:26 smithi094 bash[59677]: cluster 2023-10-09T08:50:25.945981+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:26.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:26 smithi094 bash[59677]: cluster 2023-10-09T08:50:25.949102+0000 mon.a (mon.0) 501 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-10-09T08:50:26.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:27.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:27.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:27.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:26 smithi016 bash[75179]: cluster 2023-10-09T08:50:25.945642+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:27.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:26 smithi016 bash[75179]: cluster 2023-10-09T08:50:25.945981+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:27.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:26 smithi016 bash[75179]: cluster 2023-10-09T08:50:25.949102+0000 mon.a (mon.0) 501 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-10-09T08:50:27.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:26 smithi016 bash[77892]: cluster 2023-10-09T08:50:25.945642+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:27.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:26 smithi016 bash[77892]: cluster 2023-10-09T08:50:25.945981+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:27.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:26 smithi016 bash[77892]: cluster 2023-10-09T08:50:25.949102+0000 mon.a (mon.0) 501 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-10-09T08:50:27.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:27.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:27.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:27.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:27.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:27.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:27.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:27.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:27.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:27.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:27.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:27.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:27.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:27.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T08:50:27.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:27.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:27.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:27.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:27.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:27.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:28.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:28.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:27 smithi016 bash[75179]: cluster 2023-10-09T08:50:27.290721+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:28.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:27 smithi016 bash[75179]: cluster 2023-10-09T08:50:27.315776+0000 mgr.y (mgr.25421) 380 : cluster 0 pgmap v315: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:28.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:27 smithi016 bash[77892]: cluster 2023-10-09T08:50:27.290721+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:28.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:27 smithi016 bash[77892]: cluster 2023-10-09T08:50:27.315776+0000 mgr.y (mgr.25421) 380 : cluster 0 pgmap v315: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:28.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:28.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:28.184 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:28.210 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:28.211 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:28.212 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:28.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:27 smithi094 bash[59677]: cluster 2023-10-09T08:50:27.290721+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T08:50:28.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:27 smithi094 bash[59677]: cluster 2023-10-09T08:50:27.315776+0000 mgr.y (mgr.25421) 380 : cluster 0 pgmap v315: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:28.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:28.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:28.348 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:28.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:28.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:28.483 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:28.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:28.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:28.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:28.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:28.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:28.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:28.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:28.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:28.849 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:28.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:28.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:28.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:29.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:29.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:29.152 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:29.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:29.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:29.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:29.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:29.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:29.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:29.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:29.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:29.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:29.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:29.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:29.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:29.920 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:29.933 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:29 smithi094 bash[59677]: cluster 2023-10-09T08:50:29.316429+0000 mgr.y (mgr.25421) 381 : cluster 0 pgmap v316: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:29.954 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:29.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:30.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:30.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:30.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:30.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:30.130 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:29 smithi016 bash[75179]: cluster 2023-10-09T08:50:29.316429+0000 mgr.y (mgr.25421) 381 : cluster 0 pgmap v316: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:30.131 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:29 smithi016 bash[77892]: cluster 2023-10-09T08:50:29.316429+0000 mgr.y (mgr.25421) 381 : cluster 0 pgmap v316: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:30.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:30.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:30.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:30.242 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:30.244 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:30.329 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:30.359 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:30.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:30.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:30.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:30.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:30.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:30.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:30.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:30.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:30.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:30.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:30.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:30.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:30.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:30.890 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:30.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:30.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:30.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:31.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:31.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:31.162 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:31.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:31.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:31.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:31.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:31.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:31.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:31.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:31.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:31.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:31 smithi016 bash[75179]: cluster 2023-10-09T08:50:31.317529+0000 mgr.y (mgr.25421) 382 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:31.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:31 smithi016 bash[77892]: cluster 2023-10-09T08:50:31.317529+0000 mgr.y (mgr.25421) 382 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:31.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:31.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:31.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:31 smithi094 bash[59677]: cluster 2023-10-09T08:50:31.317529+0000 mgr.y (mgr.25421) 382 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:31.794 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:31.830 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:31.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:31.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:31.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:31.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:32.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:32.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:32.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:32.190 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:32.226 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:32.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:32.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:32.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:32.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:32.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:32.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:32.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:32.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:32.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:32.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:32.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:32.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:32.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:32.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:32.818 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:32.846 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:32.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:32.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:32.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:32.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:33.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:33.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:33.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:33.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.282 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:33.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.412 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:33.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:33.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:33.648 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:33.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:33 smithi094 bash[59677]: cluster 2023-10-09T08:50:33.318378+0000 mgr.y (mgr.25421) 383 : cluster 0 pgmap v318: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:33.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:33.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:33.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:33 smithi016 bash[75179]: cluster 2023-10-09T08:50:33.318378+0000 mgr.y (mgr.25421) 383 : cluster 0 pgmap v318: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:33.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:33 smithi016 bash[77892]: cluster 2023-10-09T08:50:33.318378+0000 mgr.y (mgr.25421) 383 : cluster 0 pgmap v318: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:33.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:33.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:33.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:33.913 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:33.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:33.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:33.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:34.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:34.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:34.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:34.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:34.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:34.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:34.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:34.434 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:34.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:34.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:34.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:34.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:34.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:34.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:34.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:34.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:34.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:34.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:34.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:34.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:35.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:35.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:35.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:35.226 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:35.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:35.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:35.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:35.375 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:35 smithi016 bash[75179]: audit 2023-10-09T08:50:34.211576+0000 mon.a (mon.0) 502 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:35.375 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:35 smithi016 bash[75179]: audit 2023-10-09T08:50:34.213280+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:35.375 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:35 smithi016 bash[75179]: audit 2023-10-09T08:50:34.351387+0000 mon.a (mon.0) 504 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:35.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:35 smithi016 bash[77892]: audit 2023-10-09T08:50:34.211576+0000 mon.a (mon.0) 502 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:35.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:35 smithi016 bash[77892]: audit 2023-10-09T08:50:34.213280+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:35.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:35 smithi016 bash[77892]: audit 2023-10-09T08:50:34.351387+0000 mon.a (mon.0) 504 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:35.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:35.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:35.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:35 smithi094 bash[59677]: audit 2023-10-09T08:50:34.211576+0000 mon.a (mon.0) 502 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:35.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:35 smithi094 bash[59677]: audit 2023-10-09T08:50:34.213280+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:35.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:35 smithi094 bash[59677]: audit 2023-10-09T08:50:34.351387+0000 mon.a (mon.0) 504 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:35.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:35.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:35.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:35.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:35.660 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:35.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:35.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:35.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:35.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:35.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:35.899 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:35.931 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:35.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:36.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:50:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:50:35] "GET /metrics HTTP/1.1" 200 36234 "" "Prometheus/2.33.4" 2023-10-09T08:50:36.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:36.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:36.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:36.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:36.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:36.224 INFO:teuthology.orchestra.run.smithi016.stdout:moved 4 / 267 (1.49813%) 2023-10-09T08:50:36.224 INFO:teuthology.orchestra.run.smithi016.stdout:avg 33.375 2023-10-09T08:50:36.224 INFO:teuthology.orchestra.run.smithi016.stdout:stddev 5.09749 -> 5.1463 (expected baseline 5.40399) 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:min osd.5 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:max osd.3 with 41 -> 43 pgs (1.22846 -> 1.28839 * mean) 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:oload 110 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:max_change 0.05 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:max_change_osds 4 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:average_utilization 0.0018 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:overload_utilization 0.0020 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 weight 1.0000 -> 0.9500 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 weight 1.0000 -> 0.9500 2023-10-09T08:50:36.225 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 weight 1.0000 -> 0.9500 2023-10-09T08:50:36.226 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 weight 1.0000 -> 0.9500 2023-10-09T08:50:36.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:36.312 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:36 smithi016 bash[75179]: cluster 2023-10-09T08:50:35.319385+0000 mgr.y (mgr.25421) 384 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:36.312 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:36 smithi016 bash[75179]: audit 2023-10-09T08:50:35.422544+0000 mon.a (mon.0) 505 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]: dispatch 2023-10-09T08:50:36.312 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:36 smithi016 bash[77892]: cluster 2023-10-09T08:50:35.319385+0000 mgr.y (mgr.25421) 384 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:36.312 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:36 smithi016 bash[77892]: audit 2023-10-09T08:50:35.422544+0000 mon.a (mon.0) 505 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]: dispatch 2023-10-09T08:50:36.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:36.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:36.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:36 smithi094 bash[59677]: cluster 2023-10-09T08:50:35.319385+0000 mgr.y (mgr.25421) 384 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:36 smithi094 bash[59677]: audit 2023-10-09T08:50:35.422544+0000 mon.a (mon.0) 505 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]: dispatch 2023-10-09T08:50:36.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:36.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:36.679 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:36.713 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:36.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:36.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:36.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:36.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:36.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:37.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:37.117 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:37.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:37.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:37.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:37.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:37.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:37.349 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:37.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:37 smithi094 bash[59677]: audit 2023-10-09T08:50:35.421128+0000 mgr.y (mgr.25421) 385 : audit 0 from='client.44663 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:50:37.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:37 smithi094 bash[59677]: cluster 2023-10-09T08:50:36.218546+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:37 smithi094 bash[59677]: audit 2023-10-09T08:50:36.221022+0000 mon.a (mon.0) 506 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]': finished 2023-10-09T08:50:37.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:37 smithi094 bash[59677]: cluster 2023-10-09T08:50:36.221106+0000 mon.a (mon.0) 507 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T08:50:37.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:37 smithi094 bash[59677]: cluster 2023-10-09T08:50:36.225136+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:37 smithi094 bash[59677]: cluster 2023-10-09T08:50:36.226386+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:37 smithi094 bash[59677]: cluster 2023-10-09T08:50:36.227671+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.382 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:37.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[75179]: audit 2023-10-09T08:50:35.421128+0000 mgr.y (mgr.25421) 385 : audit 0 from='client.44663 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[75179]: cluster 2023-10-09T08:50:36.218546+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[75179]: audit 2023-10-09T08:50:36.221022+0000 mon.a (mon.0) 506 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]': finished 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[75179]: cluster 2023-10-09T08:50:36.221106+0000 mon.a (mon.0) 507 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[75179]: cluster 2023-10-09T08:50:36.225136+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[75179]: cluster 2023-10-09T08:50:36.226386+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[75179]: cluster 2023-10-09T08:50:36.227671+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[77892]: audit 2023-10-09T08:50:35.421128+0000 mgr.y (mgr.25421) 385 : audit 0 from='client.44663 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:50:37.458 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[77892]: cluster 2023-10-09T08:50:36.218546+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.458 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[77892]: audit 2023-10-09T08:50:36.221022+0000 mon.a (mon.0) 506 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]': finished 2023-10-09T08:50:37.458 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[77892]: cluster 2023-10-09T08:50:36.221106+0000 mon.a (mon.0) 507 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T08:50:37.458 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[77892]: cluster 2023-10-09T08:50:36.225136+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.458 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[77892]: cluster 2023-10-09T08:50:36.226386+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.458 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:37 smithi016 bash[77892]: cluster 2023-10-09T08:50:36.227671+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T08:50:37.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:37.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:37.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:37.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:37.519 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:50:37.520 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:50:37.607 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:37.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:37.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:37.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:37.782 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:37.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:37.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:37.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:37.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:37.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:38.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:38.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:38.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:38.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:38.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:38.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:38.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:38.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:38.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:38.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[75179]: cluster 2023-10-09T08:50:37.222618+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[75179]: cluster 2023-10-09T08:50:37.226788+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[75179]: cluster 2023-10-09T08:50:37.227434+0000 mon.a (mon.0) 508 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T08:50:38.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[75179]: cluster 2023-10-09T08:50:37.229470+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[75179]: cluster 2023-10-09T08:50:37.305923+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[75179]: cluster 2023-10-09T08:50:37.319988+0000 mgr.y (mgr.25421) 386 : cluster 0 pgmap v322: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:38.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[77892]: cluster 2023-10-09T08:50:37.222618+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.415 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[77892]: cluster 2023-10-09T08:50:37.226788+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.415 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[77892]: cluster 2023-10-09T08:50:37.227434+0000 mon.a (mon.0) 508 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T08:50:38.415 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[77892]: cluster 2023-10-09T08:50:37.229470+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.415 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[77892]: cluster 2023-10-09T08:50:37.305923+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.415 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:38 smithi016 bash[77892]: cluster 2023-10-09T08:50:37.319988+0000 mgr.y (mgr.25421) 386 : cluster 0 pgmap v322: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:38.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:38.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:38.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:38 smithi094 bash[59677]: cluster 2023-10-09T08:50:37.222618+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:38 smithi094 bash[59677]: cluster 2023-10-09T08:50:37.226788+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:38 smithi094 bash[59677]: cluster 2023-10-09T08:50:37.227434+0000 mon.a (mon.0) 508 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T08:50:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:38 smithi094 bash[59677]: cluster 2023-10-09T08:50:37.229470+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:38 smithi094 bash[59677]: cluster 2023-10-09T08:50:37.305923+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T08:50:38.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:38 smithi094 bash[59677]: cluster 2023-10-09T08:50:37.319988+0000 mgr.y (mgr.25421) 386 : cluster 0 pgmap v322: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:38.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:38.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:38.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:38.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:38.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:38.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:38.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:38.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:38.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:38.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:38.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:38.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:39.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:39.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:39.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:39.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:39.334 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:39.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:39.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:39.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:39.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:39.592 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:39.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:39.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:39.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:39.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:39.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:39.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:39.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:39.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:39.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:40.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:39 smithi016 bash[77892]: cluster 2023-10-09T08:50:39.320654+0000 mgr.y (mgr.25421) 387 : cluster 0 pgmap v323: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:40.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:39 smithi016 bash[75179]: cluster 2023-10-09T08:50:39.320654+0000 mgr.y (mgr.25421) 387 : cluster 0 pgmap v323: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:40.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:40.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:40.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:40.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:40.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:39 smithi094 bash[59677]: cluster 2023-10-09T08:50:39.320654+0000 mgr.y (mgr.25421) 387 : cluster 0 pgmap v323: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T08:50:40.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:40.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:40.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:40.414 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:40.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:40.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:40.593 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T08:50:40.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-10-09T08:50:40.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:40.678 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:40.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:40.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T08:50:40.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:40.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:40.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:40.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:40.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:41.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:41.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:41.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:41.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:41.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:41.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:40 smithi094 bash[59677]: audit 2023-10-09T08:50:40.890198+0000 mon.a (mon.0) 509 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:50:41.191 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:41.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:41.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:40 smithi016 bash[75179]: audit 2023-10-09T08:50:40.890198+0000 mon.a (mon.0) 509 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:50:41.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:40 smithi016 bash[77892]: audit 2023-10-09T08:50:40.890198+0000 mon.a (mon.0) 509 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:50:41.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:41.329 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:41.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:41.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:41.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:41.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:41.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:41.570 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:41.598 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:41.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:41.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:41.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:41.731 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:41.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:41.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:41.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:41.870 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:41.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:41.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:42.005 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:42.109 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:42.122 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:41 smithi094 bash[59677]: cluster 2023-10-09T08:50:41.321769+0000 mgr.y (mgr.25421) 388 : cluster 0 pgmap v324: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:42.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:42.182 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:41 smithi016 bash[75179]: cluster 2023-10-09T08:50:41.321769+0000 mgr.y (mgr.25421) 388 : cluster 0 pgmap v324: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:42.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:41 smithi016 bash[77892]: cluster 2023-10-09T08:50:41.321769+0000 mgr.y (mgr.25421) 388 : cluster 0 pgmap v324: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:42.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:42.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:42.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:42.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:42.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:42.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:42.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:42.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:42.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:42.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:42.673 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:50:42.702 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:50:42.724 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:50:42.749 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:50:42.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:42.777 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:50:42.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:42.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:42.803 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:50:42.833 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:50:42.860 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.862 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:50:42.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:42.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:42.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:42.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:42.934 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T08:50:43.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:43.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:42 smithi016 bash[75179]: audit 2023-10-09T08:50:42.643348+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.16:0/500051505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:50:43.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:42 smithi016 bash[75179]: audit 2023-10-09T08:50:42.653416+0000 mon.a (mon.0) 510 : audit 1 from='client.? 172.21.15.16:0/4170789787' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:50:43.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:42 smithi016 bash[77892]: audit 2023-10-09T08:50:42.643348+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.16:0/500051505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:50:43.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:42 smithi016 bash[77892]: audit 2023-10-09T08:50:42.653416+0000 mon.a (mon.0) 510 : audit 1 from='client.? 172.21.15.16:0/4170789787' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:50:43.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:43.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:43.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:43.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:42 smithi094 bash[59677]: audit 2023-10-09T08:50:42.643348+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.16:0/500051505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:50:43.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:42 smithi094 bash[59677]: audit 2023-10-09T08:50:42.653416+0000 mon.a (mon.0) 510 : audit 1 from='client.? 172.21.15.16:0/4170789787' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:50:43.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:43.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:43.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:43.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:43.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:43.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:43.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:43.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:43.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:43.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:43.815 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:43.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:43.949 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T08:50:43.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:43.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:43.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:43.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:44.086 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:44.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:43 smithi094 bash[59677]: cluster 2023-10-09T08:50:42.925039+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:44.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:43 smithi094 bash[59677]: audit 2023-10-09T08:50:42.927208+0000 mon.a (mon.0) 511 : audit 1 from='client.? 172.21.15.16:0/4170789787' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:50:44.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:43 smithi094 bash[59677]: cluster 2023-10-09T08:50:42.927330+0000 mon.a (mon.0) 512 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-10-09T08:50:44.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:43 smithi094 bash[59677]: audit 2023-10-09T08:50:43.092550+0000 mon.a (mon.0) 513 : audit 1 from='client.? 172.21.15.16:0/2009246009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:50:44.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:43 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.322455+0000 mgr.y (mgr.25421) 389 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:44.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:44.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:44.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:44.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:44.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[75179]: cluster 2023-10-09T08:50:42.925039+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:44.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[75179]: audit 2023-10-09T08:50:42.927208+0000 mon.a (mon.0) 511 : audit 1 from='client.? 172.21.15.16:0/4170789787' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:50:44.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[75179]: cluster 2023-10-09T08:50:42.927330+0000 mon.a (mon.0) 512 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-10-09T08:50:44.235 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[75179]: audit 2023-10-09T08:50:43.092550+0000 mon.a (mon.0) 513 : audit 1 from='client.? 172.21.15.16:0/2009246009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:50:44.235 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.322455+0000 mgr.y (mgr.25421) 389 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:44.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[77892]: cluster 2023-10-09T08:50:42.925039+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:44.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[77892]: audit 2023-10-09T08:50:42.927208+0000 mon.a (mon.0) 511 : audit 1 from='client.? 172.21.15.16:0/4170789787' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:50:44.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[77892]: cluster 2023-10-09T08:50:42.927330+0000 mon.a (mon.0) 512 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-10-09T08:50:44.236 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[77892]: audit 2023-10-09T08:50:43.092550+0000 mon.a (mon.0) 513 : audit 1 from='client.? 172.21.15.16:0/2009246009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:50:44.236 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:43 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.322455+0000 mgr.y (mgr.25421) 389 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:44.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:44.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:44.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:44.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:44.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:44.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:44.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:44.488 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:44.522 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:44.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:44.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:44.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:44.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:44.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:44.758 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:44.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:44.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:44.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:44.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:44.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:44.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:45.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:45.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:45.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:45.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.845244+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.845379+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.845384+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.929673+0000 mon.a (mon.0) 514 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.938185+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.939146+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: audit 2023-10-09T08:50:43.940754+0000 mon.a (mon.0) 515 : audit 1 from='client.? 172.21.15.16:0/2009246009' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:50:45.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:44 smithi094 bash[59677]: cluster 2023-10-09T08:50:43.940831+0000 mon.a (mon.0) 516 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-09T08:50:45.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:45.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:45.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:45.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.845244+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.845379+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.845384+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.929673+0000 mon.a (mon.0) 514 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.938185+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.939146+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: audit 2023-10-09T08:50:43.940754+0000 mon.a (mon.0) 515 : audit 1 from='client.? 172.21.15.16:0/2009246009' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[75179]: cluster 2023-10-09T08:50:43.940831+0000 mon.a (mon.0) 516 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.845244+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.845379+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.845384+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T08:50:45.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.929673+0000 mon.a (mon.0) 514 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:50:45.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.938185+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T08:50:45.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.939146+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T08:50:45.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: audit 2023-10-09T08:50:43.940754+0000 mon.a (mon.0) 515 : audit 1 from='client.? 172.21.15.16:0/2009246009' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:50:45.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:44 smithi016 bash[77892]: cluster 2023-10-09T08:50:43.940831+0000 mon.a (mon.0) 516 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-09T08:50:45.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:45.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:45.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:45.397 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T08:50:45.400 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T08:50:45.400 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T08:50:45.446 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-10-09T08:50:45.446 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T08:50:45.446 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_105291 2023-10-09T08:50:45.447 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:50:45.447 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-10-09T08:50:45.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:45.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:45.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:45.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:45.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:45.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:45.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:45.727 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:45.757 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:45.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:45.864 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:45.910 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:45.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:45.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:45.944 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:50:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:50:45] "GET /metrics HTTP/1.1" 200 36234 "" "Prometheus/2.33.4" 2023-10-09T08:50:45.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:45 smithi094 bash[59677]: cluster 2023-10-09T08:50:45.323688+0000 mgr.y (mgr.25421) 390 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:46.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:46.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:46.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:46.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:46.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:46.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:46.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:46.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:45 smithi016 bash[75179]: cluster 2023-10-09T08:50:45.323688+0000 mgr.y (mgr.25421) 390 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:46.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:45 smithi016 bash[77892]: cluster 2023-10-09T08:50:45.323688+0000 mgr.y (mgr.25421) 390 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:46.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:46.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:46.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:46.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:50:46.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:46.447 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2783 2767 172.928 172.938 0.00569157 0.00574989 2023-10-09T08:50:46.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:46.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:46.553 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:46.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:46.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:46.688 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:46.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:46.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:46.820 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:46.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:46.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:46.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:46.916 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:46.916 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:46.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:47.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:47.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T08:50:47.110 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:47.136 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:47.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:47.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:47.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:47.420 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:47.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:47.447 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 4960 4944 154.455 136.062 0.00545672 0.00645342 2023-10-09T08:50:47.526 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:47.553 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:47.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:47.657 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:47.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:47.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:47.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:47.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:47.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:47.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:47.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:48.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:48.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:48.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:48.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:48.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:48.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:48.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:48.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:48.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:48.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:48.448 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7074 7058 146.966 132.125 0.00702558 0.0067873 2023-10-09T08:50:48.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:48 smithi094 bash[59677]: audit 2023-10-09T08:50:47.237676+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:48 smithi094 bash[59677]: audit 2023-10-09T08:50:47.251822+0000 mon.a (mon.0) 518 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:48.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:48 smithi094 bash[59677]: cluster 2023-10-09T08:50:47.327902+0000 mgr.y (mgr.25421) 391 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s wr, 158 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:48.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:48 smithi016 bash[75179]: audit 2023-10-09T08:50:47.237676+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:48.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:48 smithi016 bash[75179]: audit 2023-10-09T08:50:47.251822+0000 mon.a (mon.0) 518 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:48.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:48 smithi016 bash[75179]: cluster 2023-10-09T08:50:47.327902+0000 mgr.y (mgr.25421) 391 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s wr, 158 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:48.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:48 smithi016 bash[77892]: audit 2023-10-09T08:50:47.237676+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:48.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:48 smithi016 bash[77892]: audit 2023-10-09T08:50:47.251822+0000 mon.a (mon.0) 518 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:48.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:48 smithi016 bash[77892]: cluster 2023-10-09T08:50:47.327902+0000 mgr.y (mgr.25421) 391 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s wr, 158 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T08:50:48.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:48.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:48.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:48.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:48.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:48.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:48.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:48.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:48.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:48.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:48.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:48.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:48.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:48.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:49.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:49.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:49.173 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:49.201 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:49.306 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:49.335 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:49.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:49.448 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9430 9414 147.033 147.25 0.00521441 0.00678926 2023-10-09T08:50:49.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:49.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:49 smithi094 bash[59677]: audit 2023-10-09T08:50:48.353478+0000 mon.a (mon.0) 519 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:49 smithi094 bash[59677]: audit 2023-10-09T08:50:48.367094+0000 mon.a (mon.0) 520 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:49 smithi094 bash[59677]: audit 2023-10-09T08:50:49.047780+0000 mon.a (mon.0) 521 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:50:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:49 smithi094 bash[59677]: audit 2023-10-09T08:50:49.049337+0000 mon.a (mon.0) 522 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:50:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:49 smithi094 bash[59677]: audit 2023-10-09T08:50:49.060290+0000 mon.a (mon.0) 523 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:49 smithi094 bash[59677]: audit 2023-10-09T08:50:49.206601+0000 mon.a (mon.0) 524 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:49 smithi094 bash[59677]: cluster 2023-10-09T08:50:49.328549+0000 mgr.y (mgr.25421) 392 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s wr, 158 op/s 2023-10-09T08:50:49.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:49.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[75179]: audit 2023-10-09T08:50:48.353478+0000 mon.a (mon.0) 519 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[75179]: audit 2023-10-09T08:50:48.367094+0000 mon.a (mon.0) 520 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[75179]: audit 2023-10-09T08:50:49.047780+0000 mon.a (mon.0) 521 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:50:49.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[75179]: audit 2023-10-09T08:50:49.049337+0000 mon.a (mon.0) 522 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:50:49.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[75179]: audit 2023-10-09T08:50:49.060290+0000 mon.a (mon.0) 523 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[75179]: audit 2023-10-09T08:50:49.206601+0000 mon.a (mon.0) 524 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:49.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[75179]: cluster 2023-10-09T08:50:49.328549+0000 mgr.y (mgr.25421) 392 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s wr, 158 op/s 2023-10-09T08:50:49.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[77892]: audit 2023-10-09T08:50:48.353478+0000 mon.a (mon.0) 519 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[77892]: audit 2023-10-09T08:50:48.367094+0000 mon.a (mon.0) 520 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[77892]: audit 2023-10-09T08:50:49.047780+0000 mon.a (mon.0) 521 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:50:49.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[77892]: audit 2023-10-09T08:50:49.049337+0000 mon.a (mon.0) 522 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:50:49.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[77892]: audit 2023-10-09T08:50:49.060290+0000 mon.a (mon.0) 523 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:49.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[77892]: audit 2023-10-09T08:50:49.206601+0000 mon.a (mon.0) 524 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:50:49.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:49 smithi016 bash[77892]: cluster 2023-10-09T08:50:49.328549+0000 mgr.y (mgr.25421) 392 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s wr, 158 op/s 2023-10-09T08:50:49.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:49.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:49.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:49.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:49.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:49.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:49.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:49.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:49.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:50.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:50.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:50.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:50.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:50.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:50.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:50.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:50.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:50.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:50.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:50.448 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 11897 11881 148.46 154.188 0.00297358 0.00671412 2023-10-09T08:50:50.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:50.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:50.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:50.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:50.614 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:50 smithi094 bash[59677]: audit 2023-10-09T08:50:49.374411+0000 mon.a (mon.0) 525 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:50.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:50.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:50.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:50.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:50 smithi016 bash[75179]: audit 2023-10-09T08:50:49.374411+0000 mon.a (mon.0) 525 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:50.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:50 smithi016 bash[77892]: audit 2023-10-09T08:50:49.374411+0000 mon.a (mon.0) 525 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:50:50.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:50.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:50.864 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:50.890 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:50.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:50.993 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:50.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:50.994 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:50:51.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:51.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:51.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:51.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:51.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:51.387 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:51 smithi016 bash[77892]: cluster 2023-10-09T08:50:51.329161+0000 mgr.y (mgr.25421) 393 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 551 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T08:50:51.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:51.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:51.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 14300 14284 148.745 150.188 0.00533308 0.00671634 2023-10-09T08:50:51.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:51.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:51.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:51.664 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:51 smithi016 bash[75179]: cluster 2023-10-09T08:50:51.329161+0000 mgr.y (mgr.25421) 393 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 551 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T08:50:51.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:51.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:51 smithi094 bash[59677]: cluster 2023-10-09T08:50:51.329161+0000 mgr.y (mgr.25421) 393 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 551 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T08:50:51.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:51.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:51.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:51.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:51.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:51.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:52.040 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:52.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:52.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:52.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:52.185 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:52.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:52.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:52.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:52.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:52.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:52.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:52.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 16391 16375 146.164 130.688 0.016341 0.00683432 2023-10-09T08:50:52.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:52.487 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:52.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:52.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:52.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:52.616 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:52.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:52.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:52.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:52.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:52.854 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:52.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:52.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:52.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:53.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:53.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:53.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:53.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:53.253 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:53.281 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:53.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:53.384 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:53.414 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:53.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 18096 18080 141.213 106.562 0.00572777 0.00707514 2023-10-09T08:50:53.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:53.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:53.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:53.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:53.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:53 smithi094 bash[59677]: cluster 2023-10-09T08:50:53.329928+0000 mgr.y (mgr.25421) 394 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 773 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T08:50:53.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:53.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:53 smithi016 bash[75179]: cluster 2023-10-09T08:50:53.329928+0000 mgr.y (mgr.25421) 394 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 773 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T08:50:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:53 smithi016 bash[77892]: cluster 2023-10-09T08:50:53.329928+0000 mgr.y (mgr.25421) 394 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 773 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T08:50:53.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:53.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:53.913 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:53.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:53.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:53.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:54.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:54.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:54.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:54.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:54.311 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:54.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:54.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:54.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 20369 20353 141.306 142.062 0.00511827 0.0070697 2023-10-09T08:50:54.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:54.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:54.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:54.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:54.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:54.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:54.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:54.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:54.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:54.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:54.988 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:54.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:54.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:55.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:55.117 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:55.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:55.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:55.221 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:55.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:55.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:55.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:55.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:55.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:55.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:55.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:55.406 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:55 smithi016 bash[77892]: cluster 2023-10-09T08:50:55.330423+0000 mgr.y (mgr.25421) 395 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T08:50:55.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 22352 22336 139.568 123.938 0.0102718 0.00715808 2023-10-09T08:50:55.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:55.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:55 smithi094 bash[59677]: cluster 2023-10-09T08:50:55.330423+0000 mgr.y (mgr.25421) 395 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T08:50:55.515 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:55.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:55.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:55.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:55.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:55.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:55.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:55 smithi016 bash[75179]: cluster 2023-10-09T08:50:55.330423+0000 mgr.y (mgr.25421) 395 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T08:50:55.743 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:50:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:50:55] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.33.4" 2023-10-09T08:50:55.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:55.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:55.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:55.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:55.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:55.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:55.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:56.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:56.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:56.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:56.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:56.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:56.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:56.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:56.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:56.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:56.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 24363 24347 138.305 125.688 0.00966144 0.00722398 2023-10-09T08:50:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:56.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:56.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:56.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:56.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:56.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:56.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:56.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:56.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:56.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:56.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:56.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:57.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:57.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:57.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:57.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:57.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:57.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:57.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:57.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:57.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:57.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:57.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 25820 25804 134.368 91.0625 0.0102832 0.00743524 2023-10-09T08:50:57.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:57.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:57.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:57.619 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:57.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:57.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:57 smithi094 bash[59677]: cluster 2023-10-09T08:50:57.330964+0000 mgr.y (mgr.25421) 396 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T08:50:57.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:57.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:57 smithi016 bash[75179]: cluster 2023-10-09T08:50:57.330964+0000 mgr.y (mgr.25421) 396 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T08:50:57.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:57 smithi016 bash[77892]: cluster 2023-10-09T08:50:57.330964+0000 mgr.y (mgr.25421) 396 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T08:50:57.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:57.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:57.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:57.892 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:57.918 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:57.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:57.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:58.020 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:58.051 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:58.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:58.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:58.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:58.291 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:58.319 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:58.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:58.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 27348 27332 131.378 95.5 0.00254418 0.00756087 2023-10-09T08:50:58.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:58.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:58.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:58.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:50:58.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:50:58.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:58.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:50:58.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:50:58.834 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:58.861 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:50:58.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:50:58.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:58.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:58.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:58.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:59.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:59.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:50:59.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:59.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:50:59.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:59.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:50:59.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:59.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:50:59.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:59.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:50:59.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:59.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:50:59.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 29032 29016 129.511 105.25 0.00451716 0.00771505 2023-10-09T08:50:59.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:59.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:59.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:59.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:59.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:50:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:59.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:50:59.726 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:59.751 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:59.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:59.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:59.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:59.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:59.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:50:59.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:50:59.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:50:59.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:50:59 smithi094 bash[59677]: cluster 2023-10-09T08:50:59.331404+0000 mgr.y (mgr.25421) 397 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T08:51:00.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:00.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:00.116 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:00.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:00.248 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:00.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:50:59 smithi016 bash[75179]: cluster 2023-10-09T08:50:59.331404+0000 mgr.y (mgr.25421) 397 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T08:51:00.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:50:59 smithi016 bash[77892]: cluster 2023-10-09T08:50:59.331404+0000 mgr.y (mgr.25421) 397 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T08:51:00.279 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:00.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:00.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:00.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:00.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:00.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 30740 30724 127.993 106.75 0.0120584 0.00780659 2023-10-09T08:51:00.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:00.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:00.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:00.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:00.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:00.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:00.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:00.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:00.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:00.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:01.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:01.070 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:01.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:01.176 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:01.204 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:01.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:01.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:01.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:01.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 32179 32163 125.614 89.9375 0.00956839 0.00794864 2023-10-09T08:51:01.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:01.562 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:01.589 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:01.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:01.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:01.702 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:01 smithi016 bash[75179]: cluster 2023-10-09T08:51:01.332178+0000 mgr.y (mgr.25421) 398 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T08:51:01.702 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:01 smithi016 bash[77892]: cluster 2023-10-09T08:51:01.332178+0000 mgr.y (mgr.25421) 398 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T08:51:01.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:01.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:01 smithi094 bash[59677]: cluster 2023-10-09T08:51:01.332178+0000 mgr.y (mgr.25421) 398 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T08:51:01.820 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:01.847 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:01.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:01.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:01.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:01.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:02.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:02.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:02.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:02.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:02.206 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:02.238 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:02.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:02.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:02.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:02.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:02.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:02.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 33446 33430 122.883 79.1875 0.00428772 0.00813107 2023-10-09T08:51:02.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:02.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:02.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:02.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:02.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:02.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:02.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:02.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:02.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:02.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:02.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:02.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:02.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:02.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:02.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:03.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:03.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:03.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:03.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:03.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:03.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:03.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:03.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:03.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:03.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:03.449 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 35061 35045 121.663 100.938 0.0026745 0.00820158 2023-10-09T08:51:03.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:03.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:03.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:03 smithi016 bash[75179]: cluster 2023-10-09T08:51:03.332664+0000 mgr.y (mgr.25421) 399 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T08:51:03.560 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:03 smithi016 bash[77892]: cluster 2023-10-09T08:51:03.332664+0000 mgr.y (mgr.25421) 399 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T08:51:03.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:03.684 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:03.694 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:03 smithi094 bash[59677]: cluster 2023-10-09T08:51:03.332664+0000 mgr.y (mgr.25421) 399 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T08:51:03.711 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:03.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:03.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:03.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:03.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:03.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:04.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:04.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:04.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:04.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:04.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:04.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:04.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:04.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:04.376 INFO:teuthology.orchestra.run.smithi016.stdout:moved 16 / 267 (5.99251%) 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:avg 33.375 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:stddev 5.1463 -> 4.2408 (expected baseline 5.40399) 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:min osd.5 with 26 -> 29 pgs (0.779026 -> 0.868914 * mean) 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:max osd.3 with 43 -> 35 pgs (1.28839 -> 1.04869 * mean) 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:oload 110 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:max_change 3 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:max_change_osds 4 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:average_utilization 0.0096 2023-10-09T08:51:04.377 INFO:teuthology.orchestra.run.smithi016.stdout:overload_utilization 0.0105 2023-10-09T08:51:04.378 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 weight 1.0000 -> 0.7234 2023-10-09T08:51:04.378 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 weight 1.0000 -> 0.8446 2023-10-09T08:51:04.378 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 weight 0.9500 -> 0.8517 2023-10-09T08:51:04.378 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 weight 0.9500 -> 0.9740 2023-10-09T08:51:04.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:04.383 INFO:teuthology.orchestra.run.smithi016.stderr:no change 2023-10-09T08:51:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:04.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:04.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 36451 36435 119.832 86.875 0.0117413 0.00833801 2023-10-09T08:51:04.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:04.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:04.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:04.620 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:04.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:04.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:04.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:04.751 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:51:04 smithi016 bash[71225]: debug 2023-10-09T08:51:04.370+0000 7f5681439700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-09T08:51:04.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:04.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:04.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:04.834 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:51:04.834 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T08:51:04.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T08:51:04.834 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T08:51:04.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.834 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:15395 6.0 6:0c5eea51:::benchmark_data_smithi016_105291_object15394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.834 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:51.959980+0000", 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.866249889000001, 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.088526305, 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 15395 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.959980+0000", 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.959980+0000", 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.959979+0000", 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.960001+0000", 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1648e-05 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.960011+0000", 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0040999999999999e-05 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.960018+0000", 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0339999999999999e-06 2023-10-09T08:51:04.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.960737+0000", 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00071916500000000002 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.960870+0000", 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000133196 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961002+0000", 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000131674 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.962868+0000", 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001866634 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.964825+0000", 2023-10-09T08:51:04.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001956916 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.048400+0000", 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.083574433000000004 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.048453+0000", 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3025999999999997e-05 2023-10-09T08:51:04.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.048506+0000", 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3353999999999998e-05 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:15401 6.0 6:0147ded3:::benchmark_data_smithi016_105291_object15400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.842 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:51.961565+0000", 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.864664909, 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.087075137999999996, 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 15401 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961565+0000", 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961565+0000", 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961563+0000", 2023-10-09T08:51:04.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961586+0000", 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2841e-05 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961598+0000", 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1508999999999999e-05 2023-10-09T08:51:04.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961605+0000", 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5820000000000004e-06 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961738+0000", 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000132884 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.961875+0000", 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013696300000000001 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.962017+0000", 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000141848 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.963476+0000", 2023-10-09T08:51:04.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0014591039999999999 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:51.964849+0000", 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0013728029999999999 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.048557+0000", 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.083708174999999996 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.048598+0000", 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0865000000000003e-05 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.048640+0000", 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2005000000000002e-05 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16764 6.0 6:0d8697c0:::benchmark_data_smithi016_105291_object16763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.641126+0000", 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.185104034, 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14470834099999999, 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16764 2023-10-09T08:51:04.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641126+0000", 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641126+0000", 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641124+0000", 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641149+0000", 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5009999999999999e-05 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641160+0000", 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0361e-05 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641168+0000", 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6720000000000004e-06 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641403+0000", 2023-10-09T08:51:04.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00023546400000000001 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641497+0000", 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3868000000000003e-05 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641691+0000", 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019386499999999999 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.643340+0000", 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0016495360000000001 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.737246+0000", 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.093906017999999994 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.785709+0000", 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.048462645999999998 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.785784+0000", 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4606999999999999e-05 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.785834+0000", 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0458999999999999e-05 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.857 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:24561 6.0 6:02b53e3e:::benchmark_data_smithi016_105291_object24560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:56.590827+0000", 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.2354022750000002, 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.094835450000000002, 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.858 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 24561 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.590827+0000", 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.590827+0000", 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.590826+0000", 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:51:04.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.590852+0000", 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5885999999999999e-05 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.590863+0000", 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1285e-05 2023-10-09T08:51:04.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.590871+0000", 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9219999999999997e-06 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.590922+0000", 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1474000000000002e-05 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.591050+0000", 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000127871 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.863 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.591191+0000", 2023-10-09T08:51:04.864 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014048700000000001 2023-10-09T08:51:04.864 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.864 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.864 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.864 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.592452+0000", 2023-10-09T08:51:04.864 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0012614639999999999 2023-10-09T08:51:04.864 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.596865+0000", 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004412582 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.685475+0000", 2023-10-09T08:51:04.865 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.088610778000000001 2023-10-09T08:51:04.866 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.866 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.866 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.866 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.685581+0000", 2023-10-09T08:51:04.866 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010506299999999999 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.685663+0000", 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2352e-05 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:24562 6.9 6:9ca4e25e:::benchmark_data_smithi016_105291_object24561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.867 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:56.591978+0000", 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.2342515130000002, 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.096132131999999995, 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 24562 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.868 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.591978+0000", 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.591978+0000", 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.869 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.591977+0000", 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.592008+0000", 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0514e-05 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.592025+0000", 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7564999999999999e-05 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.592034+0000", 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8780000000000004e-06 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.592105+0000", 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0939999999999995e-05 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.592226+0000", 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000120814 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.592379+0000", 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000153521 2023-10-09T08:51:04.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.594383+0000", 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020037259999999999 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.594630+0000", 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000246536 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.873 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688004+0000", 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.093374759000000002 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688057+0000", 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2394e-05 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688110+0000", 2023-10-09T08:51:04.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3417e-05 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:24571 6.9 6:906ae7cb:::benchmark_data_smithi016_105291_object24570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:56.595091+0000", 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.2311382210000001, 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.093176148, 2023-10-09T08:51:04.875 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 24571 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.876 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.877 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595091+0000", 2023-10-09T08:51:04.877 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.877 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595091+0000", 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595090+0000", 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595118+0000", 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8345999999999999e-05 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595134+0000", 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5571000000000001e-05 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595143+0000", 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4380000000000001e-06 2023-10-09T08:51:04.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595198+0000", 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4685999999999998e-05 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595354+0000", 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015568299999999999 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.595682+0000", 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00032818700000000002 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.597052+0000", 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0013705080000000001 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.598247+0000", 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011943609999999999 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688161+0000", 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.089913982000000003 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688202+0000", 2023-10-09T08:51:04.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1471999999999999e-05 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688268+0000", 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5395000000000007e-05 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:24581 6.9 6:98d990ca:::benchmark_data_smithi016_105291_object24580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.883 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:56.601385+0000", 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.2248447890000005, 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.087026483000000002, 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 24581 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601385+0000", 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601385+0000", 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601383+0000", 2023-10-09T08:51:04.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601413+0000", 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9420999999999998e-05 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601429+0000", 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5809999999999999e-05 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601438+0000", 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9509999999999995e-06 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601480+0000", 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2895000000000002e-05 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601582+0000", 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000101153 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.601693+0000", 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011121299999999999 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.602708+0000", 2023-10-09T08:51:04.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010149759999999999 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.603236+0000", 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00052845400000000003 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688324+0000", 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.085088121000000003 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688371+0000", 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6427000000000001e-05 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:56.688411+0000", 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0620000000000001e-05 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.890 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:26654 6.0 6:096ae588:::benchmark_data_smithi016_105291_object26653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:57.924957+0000", 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.9012723009999997, 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.090436368000000003, 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 26654 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.924957+0000", 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.924957+0000", 2023-10-09T08:51:04.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.924957+0000", 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.924992+0000", 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4925999999999997e-05 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.925003+0000", 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1588e-05 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.925014+0000", 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1112e-05 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.925058+0000", 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3739000000000003e-05 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.925172+0000", 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000114413 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.925300+0000", 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012708000000000001 2023-10-09T08:51:04.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.927472+0000", 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021724600000000002 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.927761+0000", 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00028888500000000001 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015269+0000", 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.087508078000000003 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015338+0000", 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8811000000000002e-05 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015394+0000", 2023-10-09T08:51:04.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5996999999999999e-05 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:26659 6.9 6:97f4d2f6:::benchmark_data_smithi016_105291_object26658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:57.928600+0000", 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.8976299489999997, 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.087252753000000002, 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.898 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 26659 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928600+0000", 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928600+0000", 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928598+0000", 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928624+0000", 2023-10-09T08:51:04.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5999999999999998e-05 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928637+0000", 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2840000000000001e-05 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928645+0000", 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7940000000000003e-06 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928725+0000", 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0678000000000002e-05 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.928874+0000", 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000148849 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929025+0000", 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000151385 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.933852+0000", 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048262009999999996 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.014960+0000", 2023-10-09T08:51:04.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.081107879999999993 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.014966+0000", 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9879999999999998e-06 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015017+0000", 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0717999999999998e-05 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015852+0000", 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00083528300000000003 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:26661 6.0 6:0df673d4:::benchmark_data_smithi016_105291_object26660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:57.929930+0000", 2023-10-09T08:51:04.905 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.8962995950000003, 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.086378541000000003, 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 26661 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929930+0000", 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929930+0000", 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929929+0000", 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:51:04.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929973+0000", 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.401e-05 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929983+0000", 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0026e-05 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929988+0000", 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3639999999999998e-06 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930037+0000", 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8347999999999999e-05 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930182+0000", 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000145198 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930344+0000", 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016220300000000001 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.932571+0000", 2023-10-09T08:51:04.910 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022272160000000002 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015165+0000", 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.082594025000000001 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015442+0000", 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00027628899999999998 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015498+0000", 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6486000000000001e-05 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.016309+0000", 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00081037999999999998 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:26662 6.9 6:97fa1ef5:::benchmark_data_smithi016_105291_object26661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:57.929999+0000", 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.8962308199999995, 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.087932840999999998, 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 26662 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.913 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929999+0000", 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929999+0000", 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.929998+0000", 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930018+0000", 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9238000000000001e-05 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930023+0000", 2023-10-09T08:51:04.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3600000000000004e-06 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930027+0000", 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6490000000000001e-06 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.933877+0000", 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0038499110000000001 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.934002+0000", 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012520599999999999 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.934137+0000", 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000135155 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015991+0000", 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.081854355000000004 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.016274+0000", 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00028252600000000001 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.017799+0000", 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001524673 2023-10-09T08:51:04.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.017862+0000", 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3303e-05 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.017932+0000", 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9894000000000005e-05 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.919 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:26666 6.0 6:09109588:::benchmark_data_smithi016_105291_object26665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:57.930983+0000", 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.8952466890000004, 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.08772742, 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.920 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 26666 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930983+0000", 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930983+0000", 2023-10-09T08:51:04.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.930982+0000", 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.931004+0000", 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2473999999999999e-05 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.931123+0000", 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000118492 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.931131+0000", 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8140000000000001e-06 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.931375+0000", 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00024460900000000002 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.931509+0000", 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000133299 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:57.931625+0000", 2023-10-09T08:51:04.924 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000116673 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015090+0000", 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.083464868999999997 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.015217+0000", 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000126563 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.925 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.018599+0000", 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0033822700000000002 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.018657+0000", 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7738000000000002e-05 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.926 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.018710+0000", 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3730000000000002e-05 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:27335 6.0 6:0bea1cef:::benchmark_data_smithi016_105291_object27334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.364792+0000", 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.4614379529999999, 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.135978812, 2023-10-09T08:51:04.927 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27335 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364792+0000", 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.928 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364792+0000", 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364790+0000", 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.929 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364825+0000", 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5389999999999998e-05 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364847+0000", 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1761999999999999e-05 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.930 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364858+0000", 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1026000000000001e-05 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364974+0000", 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011663 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.931 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365118+0000", 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014366300000000001 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365258+0000", 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000139455 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366885+0000", 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001627414 2023-10-09T08:51:04.932 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.373667+0000", 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0067823739999999999 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500416+0000", 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.12674902599999999 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500515+0000", 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8659999999999994e-05 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500771+0000", 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000255567 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.934 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:27345 6.0 6:0bdd0c77:::benchmark_data_smithi016_105291_object27344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.372202+0000", 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.4540280799999996, 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13327161600000001, 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.935 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27345 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372202+0000", 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372202+0000", 2023-10-09T08:51:04.936 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372200+0000", 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372238+0000", 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8012000000000001e-05 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372252+0000", 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3560000000000001e-05 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372261+0000", 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9449999999999999e-06 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372315+0000", 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4282e-05 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372442+0000", 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000127422 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372558+0000", 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011592 2023-10-09T08:51:04.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.373956+0000", 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001397831 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.376008+0000", 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00205229 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505376+0000", 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.129367436 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505430+0000", 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4759000000000002e-05 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505473+0000", 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2781000000000003e-05 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33221 6.9 6:99c54d79:::benchmark_data_smithi016_105291_object33220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.076314+0000", 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.7499159309999999, 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.161054854, 2023-10-09T08:51:04.942 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33221 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076314+0000", 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.943 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076314+0000", 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076313+0000", 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.944 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076338+0000", 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4510000000000001e-05 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076345+0000", 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7060000000000004e-06 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076351+0000", 2023-10-09T08:51:04.945 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3020000000000002e-06 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.079361+0000", 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0030103700000000001 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.079529+0000", 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016808200000000001 2023-10-09T08:51:04.946 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.079685+0000", 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015617800000000001 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081122+0000", 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0014370909999999999 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.947 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083049+0000", 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001926138 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.237185+0000", 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.154136316 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.948 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.237242+0000", 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7139000000000003e-05 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.237369+0000", 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012664600000000001 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33237 6.0 6:090cd5be:::benchmark_data_smithi016_105291_object33236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.083055+0000", 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.743174556, 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15698177299999999, 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33237 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.950 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083055+0000", 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083055+0000", 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.952 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083054+0000", 2023-10-09T08:51:04.952 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:51:04.952 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.952 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.953 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.953 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083083+0000", 2023-10-09T08:51:04.953 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9247e-05 2023-10-09T08:51:04.953 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083095+0000", 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.187e-05 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083103+0000", 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8890000000000002e-06 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083570+0000", 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00046726500000000002 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083698+0000", 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012836299999999999 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083843+0000", 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000145222 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.085474+0000", 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0016303470000000001 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.956 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.087413+0000", 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0019395969999999999 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.239922+0000", 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.152509019 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.239991+0000", 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8560999999999996e-05 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.240037+0000", 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5918000000000003e-05 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.958 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33261 6.9 6:9a5f0e1d:::benchmark_data_smithi016_105291_object33260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.242423+0000", 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.5838062170000002, 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.11731966200000001, 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33261 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.959 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242423+0000", 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242423+0000", 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.960 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242421+0000", 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242470+0000", 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9437000000000001e-05 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242527+0000", 2023-10-09T08:51:04.961 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6229e-05 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242545+0000", 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8314000000000001e-05 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242740+0000", 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019496699999999999 2023-10-09T08:51:04.962 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.243115+0000", 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00037537900000000001 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.243494+0000", 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000378301 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.963 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.246490+0000", 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0029969290000000002 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.249285+0000", 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0027949070000000001 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359656+0000", 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.11037013399999999 2023-10-09T08:51:04.964 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359706+0000", 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9978999999999997e-05 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359743+0000", 2023-10-09T08:51:04.965 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7614000000000003e-05 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33269 6.9 6:928db727:::benchmark_data_smithi016_105291_object33268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.247172+0000", 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.5790581110000002, 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.112677965, 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.966 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33269 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247172+0000", 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.967 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247172+0000", 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247170+0000", 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247203+0000", 2023-10-09T08:51:04.968 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3665999999999999e-05 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247218+0000", 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5166e-05 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247227+0000", 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4880000000000007e-06 2023-10-09T08:51:04.969 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247294+0000", 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7034999999999998e-05 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247434+0000", 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000139763 2023-10-09T08:51:04.970 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247587+0000", 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015303800000000001 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.250197+0000", 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0026104639999999998 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.251762+0000", 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001564417 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359781+0000", 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.108019096 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.972 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359819+0000", 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7988000000000003e-05 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359850+0000", 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0834999999999999e-05 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.973 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33270 6.0 6:09625a96:::benchmark_data_smithi016_105291_object33269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.247246+0000", 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.5789835610000003, 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.112006252, 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33270 2023-10-09T08:51:04.974 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247246+0000", 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247246+0000", 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.975 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247246+0000", 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247272+0000", 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6517e-05 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.976 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247280+0000", 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3820000000000004e-06 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247286+0000", 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8510000000000004e-06 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247339+0000", 2023-10-09T08:51:04.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3201000000000001e-05 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247484+0000", 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014484 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247639+0000", 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015491699999999999 2023-10-09T08:51:04.978 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.250351+0000", 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0027117970000000002 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.251799+0000", 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00144769 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.979 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359164+0000", 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.107364944 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359214+0000", 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0024000000000001e-05 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.359252+0000", 2023-10-09T08:51:04.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8690999999999999e-05 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:35788 6.0 6:049420ee:::benchmark_data_smithi016_105291_object35787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:03.907252+0000", 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.91897812599999995, 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.086100267999999994, 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:51:04.981 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 35788 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907252+0000", 2023-10-09T08:51:04.982 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907252+0000", 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907249+0000", 2023-10-09T08:51:04.983 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907278+0000", 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8858999999999999e-05 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907295+0000", 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7104000000000001e-05 2023-10-09T08:51:04.984 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907305+0000", 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0088000000000001e-05 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907359+0000", 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3362000000000002e-05 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.985 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907488+0000", 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012932199999999999 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.907637+0000", 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014883299999999999 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:51:04.986 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.910737+0000", 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0031005820000000002 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.916398+0000", 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0056606180000000001 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.993215+0000", 2023-10-09T08:51:04.987 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.076817253000000002 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.993292+0000", 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7037000000000001e-05 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:03.993352+0000", 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9524999999999997e-05 2023-10-09T08:51:04.988 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.989 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.989 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.989 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:51:04.989 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:51:04.989 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:51:04.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:04.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:04 smithi094 bash[59677]: audit 2023-10-09T08:51:04.207998+0000 mon.a (mon.0) 526 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:05.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:05.117 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:05.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:05.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:04 smithi016 bash[75179]: audit 2023-10-09T08:51:04.207998+0000 mon.a (mon.0) 526 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:05.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:04 smithi016 bash[77892]: audit 2023-10-09T08:51:04.207998+0000 mon.a (mon.0) 526 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:05.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:05.381 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:05.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:51:05.447892+0000 min lat: 0.00138239 max lat: 0.265943 avg lat: 0.00841899 2023-10-09T08:51:05.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:51:05.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 37915 37899 118.415 91.5 0.00630305 0.00841899 2023-10-09T08:51:05.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:05.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:05.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:05.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:05.777 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.803 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:05.905 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.928 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:51:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:51:05] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.33.4" 2023-10-09T08:51:05.929 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:05 smithi016 bash[77892]: audit 2023-10-09T08:51:04.373148+0000 mgr.y (mgr.25421) 400 : audit 0 from='client.34794 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:05.929 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:05 smithi016 bash[77892]: cluster 2023-10-09T08:51:05.333382+0000 mgr.y (mgr.25421) 401 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T08:51:05.932 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:05.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:05.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:06.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:06.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:06.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:06.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:06.181 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:05 smithi094 bash[59677]: audit 2023-10-09T08:51:04.373148+0000 mgr.y (mgr.25421) 400 : audit 0 from='client.34794 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:06.182 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:05 smithi094 bash[59677]: cluster 2023-10-09T08:51:05.333382+0000 mgr.y (mgr.25421) 401 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T08:51:06.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:06.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:06.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:05 smithi016 bash[75179]: audit 2023-10-09T08:51:04.373148+0000 mgr.y (mgr.25421) 400 : audit 0 from='client.34794 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:06.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:05 smithi016 bash[75179]: cluster 2023-10-09T08:51:05.333382+0000 mgr.y (mgr.25421) 401 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T08:51:06.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:06.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:06.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:06.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:06.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:06.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 39416 39400 117.243 93.8125 0.0131363 0.00852363 2023-10-09T08:51:06.470 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:06.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:06.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:06.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:06.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:06.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:06.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:06.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:06.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:06.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:06.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:06.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:06.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:07.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:07.113 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:07.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:07.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:07.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:07.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:07.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:07.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 40950 40934 116.271 95.875 0.00632332 0.00859402 2023-10-09T08:51:07.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:07.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:07.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:07.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:07.666 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.671 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:07 smithi094 bash[59677]: cluster 2023-10-09T08:51:07.333944+0000 mgr.y (mgr.25421) 402 : cluster 0 pgmap v339: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T08:51:07.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:07.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T08:51:07.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:07 smithi016 bash[75179]: cluster 2023-10-09T08:51:07.333944+0000 mgr.y (mgr.25421) 402 : cluster 0 pgmap v339: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T08:51:07.730 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:07 smithi016 bash[77892]: cluster 2023-10-09T08:51:07.333944+0000 mgr.y (mgr.25421) 402 : cluster 0 pgmap v339: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T08:51:07.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:07.794 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T08:51:07.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T08:51:07.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:07.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:07.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:07.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:08.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:08.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:08.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:08.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:08.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:08.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:08.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:08.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:08.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:08.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 42323 42307 114.947 85.8125 0.00502798 0.00869267 2023-10-09T08:51:08.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:08.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:08.599 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:08.635 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:51:08.639 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:08 smithi016 bash[75179]: audit 2023-10-09T08:51:08.580574+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.16:0/4123169330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:08.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:08 smithi016 bash[77892]: audit 2023-10-09T08:51:08.580574+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.16:0/4123169330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:08.682 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:51:08.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:08.711 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:08 smithi094 bash[59677]: audit 2023-10-09T08:51:08.580574+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.16:0/4123169330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:08.726 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:51:08.728 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:08.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:08.787 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:51:08.828 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:51:08.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:08.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:08.868 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:51:08.906 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:51:08.960 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:51:08.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:08.974 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.974 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.975 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.975 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.975 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.975 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.975 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.975 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:51:08.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:08.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:09.099 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:09.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:09.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:09.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:09.366 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.393 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:09.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:09.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 43661 43645 113.641 83.625 0.0104382 0.00879341 2023-10-09T08:51:09.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:09.522 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:09.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:09.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.658 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:09.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:09.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:09.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:09.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:09.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:09.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:09.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:09.926 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:09.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:09.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:09.937 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T08:51:10.029 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:10.055 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:10.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:10.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:10.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:10.170 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:09 smithi094 bash[59677]: audit 2023-10-09T08:51:09.008356+0000 mon.a (mon.0) 527 : audit 1 from='client.? 172.21.15.16:0/321392165' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:51:10.170 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:09 smithi094 bash[59677]: cluster 2023-10-09T08:51:09.334409+0000 mgr.y (mgr.25421) 403 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T08:51:10.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:10.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:10.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:10.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:09 smithi016 bash[75179]: audit 2023-10-09T08:51:09.008356+0000 mon.a (mon.0) 527 : audit 1 from='client.? 172.21.15.16:0/321392165' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:51:10.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:09 smithi016 bash[75179]: cluster 2023-10-09T08:51:09.334409+0000 mgr.y (mgr.25421) 403 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T08:51:10.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:09 smithi016 bash[77892]: audit 2023-10-09T08:51:09.008356+0000 mon.a (mon.0) 527 : audit 1 from='client.? 172.21.15.16:0/321392165' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:51:10.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:09 smithi016 bash[77892]: cluster 2023-10-09T08:51:09.334409+0000 mgr.y (mgr.25421) 403 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T08:51:10.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:10.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:10.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:10.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:10.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:10.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 44979 44963 112.39 82.375 0.00494354 0.00889153 2023-10-09T08:51:10.463 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:10.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:10.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:10.599 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:10.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:10.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:10.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:10.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:10.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:10.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:10.867 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:10.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[75179]: cluster 2023-10-09T08:51:09.912180+0000 mon.a (mon.0) 528 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[75179]: cluster 2023-10-09T08:51:09.922504+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[75179]: cluster 2023-10-09T08:51:09.922564+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[75179]: audit 2023-10-09T08:51:09.928706+0000 mon.a (mon.0) 529 : audit 1 from='client.? 172.21.15.16:0/321392165' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[75179]: cluster 2023-10-09T08:51:09.929089+0000 mon.a (mon.0) 530 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[75179]: cluster 2023-10-09T08:51:09.939417+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[77892]: cluster 2023-10-09T08:51:09.912180+0000 mon.a (mon.0) 528 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:51:10.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[77892]: cluster 2023-10-09T08:51:09.922504+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[77892]: cluster 2023-10-09T08:51:09.922564+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[77892]: audit 2023-10-09T08:51:09.928706+0000 mon.a (mon.0) 529 : audit 1 from='client.? 172.21.15.16:0/321392165' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:51:10.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[77892]: cluster 2023-10-09T08:51:09.929089+0000 mon.a (mon.0) 530 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-09T08:51:10.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:10 smithi016 bash[77892]: cluster 2023-10-09T08:51:09.939417+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:10.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:10 smithi094 bash[59677]: cluster 2023-10-09T08:51:09.912180+0000 mon.a (mon.0) 528 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:51:10.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:10 smithi094 bash[59677]: cluster 2023-10-09T08:51:09.922504+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:10 smithi094 bash[59677]: cluster 2023-10-09T08:51:09.922564+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:10.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:10 smithi094 bash[59677]: audit 2023-10-09T08:51:09.928706+0000 mon.a (mon.0) 529 : audit 1 from='client.? 172.21.15.16:0/321392165' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:51:10.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:10 smithi094 bash[59677]: cluster 2023-10-09T08:51:09.929089+0000 mon.a (mon.0) 530 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-09T08:51:10.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:10 smithi094 bash[59677]: cluster 2023-10-09T08:51:09.939417+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T08:51:11.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:11.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:11.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:11.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:11.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:11.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:11.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:11.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:11.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:11.450 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 46346 46330 111.353 85.4375 0.00895611 0.0089742 2023-10-09T08:51:11.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:11.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:11.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:11.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:11.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:11.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:11.822 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:11.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:11.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:11 smithi016 bash[75179]: cluster 2023-10-09T08:51:11.335050+0000 mgr.y (mgr.25421) 404 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T08:51:11.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:11.953 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:11.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:12.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:12.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:12.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:12.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:12.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:11 smithi094 bash[59677]: cluster 2023-10-09T08:51:11.335050+0000 mgr.y (mgr.25421) 404 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T08:51:12.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:11 smithi016 bash[77892]: cluster 2023-10-09T08:51:11.335050+0000 mgr.y (mgr.25421) 404 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T08:51:12.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:12.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:12.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:12.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:12.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:12.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:12.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:12.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 47773 47757 110.532 89.1875 0.00790475 0.0090408 2023-10-09T08:51:12.509 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:12.541 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:12.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:12.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:12.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:12.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:12.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:12.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:51:12.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:12.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:12.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:12.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:12.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:12.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:12.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:13.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:13.083 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:13.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:13.187 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:13.224 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:13.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:13.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:13.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:13.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:13.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:13.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:13.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T08:51:13.452 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 15 49044 49029 109.424 79.5 0.0109921 0.00913062 2023-10-09T08:51:13.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:13.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:13.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:13.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:13.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:13.617 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:13 smithi016 bash[75179]: cluster 2023-10-09T08:51:13.335589+0000 mgr.y (mgr.25421) 405 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T08:51:13.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:13 smithi016 bash[77892]: cluster 2023-10-09T08:51:13.335589+0000 mgr.y (mgr.25421) 405 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T08:51:13.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:13.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:13 smithi094 bash[59677]: cluster 2023-10-09T08:51:13.335589+0000 mgr.y (mgr.25421) 405 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T08:51:13.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:13.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:13.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:13.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:13.914 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:13.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:14.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:14.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:14.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:14.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:14.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:14.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:14.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:14.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:14.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:14.426 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:14.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 50379 50363 108.525 83.375 0.0125409 0.009208 2023-10-09T08:51:14.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:14.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:14.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:14.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:14.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:14.822 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:14.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:14.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:14.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:14.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:14.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:14.982 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:14.982 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:15.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:15.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:15.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:15.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:15.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:15.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:15.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:15.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:15.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:15.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 51664 51648 107.585 80.3125 0.019784 0.00928858 2023-10-09T08:51:15.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:15.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:15.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:15.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:15.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:15.646 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:15 smithi016 bash[75179]: cluster 2023-10-09T08:51:15.336291+0000 mgr.y (mgr.25421) 406 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T08:51:15.646 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:15 smithi016 bash[77892]: cluster 2023-10-09T08:51:15.336291+0000 mgr.y (mgr.25421) 406 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T08:51:15.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:15.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:15.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:15 smithi094 bash[59677]: cluster 2023-10-09T08:51:15.336291+0000 mgr.y (mgr.25421) 406 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T08:51:15.769 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:15.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:15.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:15.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:15.917 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:51:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:51:15] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.33.4" 2023-10-09T08:51:15.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:15.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:15.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:16.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:16.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:16.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:16.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:16.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:16.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:16.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:16.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:16.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:16.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:16.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:16.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 52793 52777 106.39 70.5625 0.00533865 0.00939325 2023-10-09T08:51:16.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:16.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:16.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:16.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:16.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:16.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:16.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:16.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:16.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:16.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:16.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:16.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:16.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:16.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:16.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:17.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:17.023 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:17.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:17.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:17.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:17.240 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:17.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:17.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:17.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:17.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:17.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 54120 54104 105.657 82.9375 0.0128725 0.00945916 2023-10-09T08:51:17.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:17.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:17.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:17.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:17.670 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:17.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:17 smithi094 bash[59677]: cluster 2023-10-09T08:51:17.336822+0000 mgr.y (mgr.25421) 407 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T08:51:17.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:17.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:17 smithi016 bash[75179]: cluster 2023-10-09T08:51:17.336822+0000 mgr.y (mgr.25421) 407 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T08:51:17.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:17 smithi016 bash[77892]: cluster 2023-10-09T08:51:17.336822+0000 mgr.y (mgr.25421) 407 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T08:51:17.775 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:17.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:17.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:17.932 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:17.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:18.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:18.045 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:18.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:18.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:18.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:18.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:18.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:18.309 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:18.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:18.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:18.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:18.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 55416 55400 104.91 81 0.0126223 0.00952598 2023-10-09T08:51:18.474 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:18.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:18.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:18.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:18.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:18.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:18.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:18.748 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:18.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:18.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:18.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:18.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:18.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:19.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:19.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:19.158 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:19.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:19.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:19.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:19.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:19.416 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:19.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 56626 56610 104.048 75.625 0.00727437 0.00960452 2023-10-09T08:51:19.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:19.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:19.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:19.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:19.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:19.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:19.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:19.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:19.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:19.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:20.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:20.059 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:19 smithi094 bash[59677]: audit 2023-10-09T08:51:19.207163+0000 mon.a (mon.0) 531 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:20.060 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:19 smithi094 bash[59677]: cluster 2023-10-09T08:51:19.337307+0000 mgr.y (mgr.25421) 408 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T08:51:20.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:20.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:20.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:19 smithi016 bash[75179]: audit 2023-10-09T08:51:19.207163+0000 mon.a (mon.0) 531 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:20.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:19 smithi016 bash[75179]: cluster 2023-10-09T08:51:19.337307+0000 mgr.y (mgr.25421) 408 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T08:51:20.187 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:19 smithi016 bash[77892]: audit 2023-10-09T08:51:19.207163+0000 mon.a (mon.0) 531 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:20.187 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:19 smithi016 bash[77892]: cluster 2023-10-09T08:51:19.337307+0000 mgr.y (mgr.25421) 408 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T08:51:20.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:20.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:20.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:20.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:20.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:20.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 57785 57769 103.145 72.4375 0.0147606 0.00968965 2023-10-09T08:51:20.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:20.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:20.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:20.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:20.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:20.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:20.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:20.880 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:20.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:20.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:21.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:21.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:21.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:21.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:21.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:21.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:21.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:21.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:21.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:21.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:21.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:21.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 59026 59010 102.434 77.5625 0.0114621 0.00975653 2023-10-09T08:51:21.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:21 smithi016 bash[75179]: cluster 2023-10-09T08:51:21.338416+0000 mgr.y (mgr.25421) 409 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T08:51:21.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:21 smithi016 bash[77892]: cluster 2023-10-09T08:51:21.338416+0000 mgr.y (mgr.25421) 409 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T08:51:21.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:21.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:21.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:21 smithi094 bash[59677]: cluster 2023-10-09T08:51:21.338416+0000 mgr.y (mgr.25421) 409 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T08:51:21.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:21.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:21.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:22.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:22.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:22.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:22.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:22.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:22.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:22.311 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:22.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:22.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:22.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:22.452 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 60205 60189 101.657 73.6875 0.00799518 0.00983137 2023-10-09T08:51:22.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:22.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:22.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:22.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:22.597 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:22.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:22.701 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:22.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:22.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:22.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:22.841 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:22.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:22.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:22.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:23.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:23.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:23.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:23.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:23.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:23.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:23.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:23.366 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:23.393 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:23.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:23.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:23 smithi016 bash[75179]: cluster 2023-10-09T08:51:23.339460+0000 mgr.y (mgr.25421) 410 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T08:51:23.451 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 61374 61358 100.904 73.0625 0.0123919 0.00990451 2023-10-09T08:51:23.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:23.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:23.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:23.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:23.667 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:23 smithi016 bash[77892]: cluster 2023-10-09T08:51:23.339460+0000 mgr.y (mgr.25421) 410 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T08:51:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:23.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:23 smithi094 bash[59677]: cluster 2023-10-09T08:51:23.339460+0000 mgr.y (mgr.25421) 410 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T08:51:23.769 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:23.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:23.898 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:23.933 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:23.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:24.036 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:24.062 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:24.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:24.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:24.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:24.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:24.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:24.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:24.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:24.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:24.454 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 62453 62437 100.046 67.4375 0.0114651 0.00998929 2023-10-09T08:51:24.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:24.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:24.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:24.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:24.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:24.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:24.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:24.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:24.734 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:24.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:24.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:24.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:24.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:24.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:24.971 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:24.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:25.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:25.103 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:25.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:25.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:25.276 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:25.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:25.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:25.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:25.452 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:51:25.449863+0000 min lat: 0.00138239 max lat: 0.265943 avg lat: 0.0100411 2023-10-09T08:51:25.452 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:51:25.452 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 63725 63709 99.5323 79.5 0.0101352 0.0100411 2023-10-09T08:51:25.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:25.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:25.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:25.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:25 smithi094 bash[59677]: cluster 2023-10-09T08:51:25.340115+0000 mgr.y (mgr.25421) 411 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T08:51:25.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:25.741 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:51:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:51:25] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.33.4" 2023-10-09T08:51:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:25 smithi016 bash[75179]: cluster 2023-10-09T08:51:25.340115+0000 mgr.y (mgr.25421) 411 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T08:51:25.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:25 smithi016 bash[77892]: cluster 2023-10-09T08:51:25.340115+0000 mgr.y (mgr.25421) 411 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T08:51:25.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:25.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:25.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:25.922 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:25.947 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:25.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:25.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:26.052 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:26.076 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:26.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:26.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:26.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:26.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:26.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:26.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:26.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:26.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:26.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:26.452 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 64946 64930 98.9644 76.3125 0.00811664 0.0100983 2023-10-09T08:51:26.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:26.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:26.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:26.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:26.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:26.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:26.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:26.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:26.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:26.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:26.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:26.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:26.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:27.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:27.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:27.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:27.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:27.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:27.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:27.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:27.395 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:27.427 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:27.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 66013 65997 98.1958 66.6875 0.00592443 0.0101776 2023-10-09T08:51:27.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:27.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:27 smithi016 bash[75179]: cluster 2023-10-09T08:51:27.340682+0000 mgr.y (mgr.25421) 412 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T08:51:27.541 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:27 smithi016 bash[77892]: cluster 2023-10-09T08:51:27.340682+0000 mgr.y (mgr.25421) 412 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T08:51:27.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:27.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:27.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:27 smithi094 bash[59677]: cluster 2023-10-09T08:51:27.340682+0000 mgr.y (mgr.25421) 412 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T08:51:27.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:27.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:27.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:27.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:27.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:27.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:28.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:28.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:28.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:28.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:28.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:28.347 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:28.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:28.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:28.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 67197 67181 97.633 74 0.018461 0.0102366 2023-10-09T08:51:28.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:28.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:28.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:28.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:28.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:28.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:28.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:28.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:28.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:28.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:28.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:29.097 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:29.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:29.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:29.225 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:29.258 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:29.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:29.361 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:29.390 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:29.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:29.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 68349 68333 97.0503 72 0.0144813 0.0102981 2023-10-09T08:51:29.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:29.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:29.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:29.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:29.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:29.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:29.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:29.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:29.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:29.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:29.910 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:29.932 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:29.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:30.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:30.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:30.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:30.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:30.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:29 smithi016 bash[77892]: cluster 2023-10-09T08:51:29.342240+0000 mgr.y (mgr.25421) 413 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T08:51:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:30.196 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:29 smithi016 bash[75179]: cluster 2023-10-09T08:51:29.342240+0000 mgr.y (mgr.25421) 413 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T08:51:30.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:29 smithi094 bash[59677]: cluster 2023-10-09T08:51:29.342240+0000 mgr.y (mgr.25421) 413 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T08:51:30.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:30.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:30.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:30.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:30.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 69466 69450 96.4446 69.8125 0.0346986 0.0103637 2023-10-09T08:51:30.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:30.560 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:30.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:30.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:30.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:30.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:30.841 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:30.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:30.957 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:51:30.957 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:51:30.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:30.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:31.086 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:31.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:31.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:31.217 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:31.243 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:31.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:31.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:31.376 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:51:31.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:31.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:31.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:31.427 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:51:31.428 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T08:51:31.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 70310 70294 95.4946 52.75 0.00868332 0.0104623 2023-10-09T08:51:31.476 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:51:31.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:31.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[75179]: audit 2023-10-09T08:51:31.322242+0000 mon.a (mon.0) 532 : audit 0 from='client.? 172.21.15.16:0/101077903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:31.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[75179]: cluster 2023-10-09T08:51:31.343112+0000 mgr.y (mgr.25421) 414 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T08:51:31.520 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[75179]: audit 2023-10-09T08:51:31.353652+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.16:0/2674459454' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:51:31.520 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[75179]: audit 2023-10-09T08:51:31.353837+0000 mon.a (mon.0) 533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:51:31.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[77892]: audit 2023-10-09T08:51:31.322242+0000 mon.a (mon.0) 532 : audit 0 from='client.? 172.21.15.16:0/101077903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:31.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[77892]: cluster 2023-10-09T08:51:31.343112+0000 mgr.y (mgr.25421) 414 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T08:51:31.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[77892]: audit 2023-10-09T08:51:31.353652+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.16:0/2674459454' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:51:31.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:31 smithi016 bash[77892]: audit 2023-10-09T08:51:31.353837+0000 mon.a (mon.0) 533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:51:31.522 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:51:31.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:31.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:31.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:31.568 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:51:31.607 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:51:31.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:31.649 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:51:31.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:31.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:31.689 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:51:31.703 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.704 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.704 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.704 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.704 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.704 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.705 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.705 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:51:31.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:31 smithi094 bash[59677]: audit 2023-10-09T08:51:31.322242+0000 mon.a (mon.0) 532 : audit 0 from='client.? 172.21.15.16:0/101077903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:31 smithi094 bash[59677]: cluster 2023-10-09T08:51:31.343112+0000 mgr.y (mgr.25421) 414 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T08:51:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:31 smithi094 bash[59677]: audit 2023-10-09T08:51:31.353652+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.16:0/2674459454' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:51:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:31 smithi094 bash[59677]: audit 2023-10-09T08:51:31.353837+0000 mon.a (mon.0) 533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:51:31.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:31.798 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:31.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:31.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:31.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:31.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:31.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:32.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:32.060 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:32.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:32.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:32.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:32.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:32.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:32.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:32.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:32.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:32.422 INFO:teuthology.orchestra.run.smithi016.stdout:moved 16 / 267 (5.99251%) 2023-10-09T08:51:32.422 INFO:teuthology.orchestra.run.smithi016.stdout:avg 33.375 2023-10-09T08:51:32.422 INFO:teuthology.orchestra.run.smithi016.stdout:stddev 5.1463 -> 5.4758 (expected baseline 5.40399) 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:min osd.5 with 26 -> 27 pgs (0.779026 -> 0.808989 * mean) 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:max osd.3 with 43 -> 31 pgs (1.28839 -> 0.928839 * mean) 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:oload 110 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:max_change 3 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:max_change_osds 4 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:average_utilization 0.0192 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:overload_utilization 0.0211 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 weight 1.0000 -> 0.6379 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 weight 1.0000 -> 0.8266 2023-10-09T08:51:32.423 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 weight 0.9500 -> 1.0000 2023-10-09T08:51:32.424 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 weight 0.9500 -> 1.0000 2023-10-09T08:51:32.440 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:32.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 71328 71312 94.8164 63.625 0.0475422 0.0105396 2023-10-09T08:51:32.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:32.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:32.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: cluster 2023-10-09T08:51:31.404319+0000 mon.a (mon.0) 534 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:51:32.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: cluster 2023-10-09T08:51:31.404364+0000 mon.a (mon.0) 535 : cluster 1 Cluster is now healthy 2023-10-09T08:51:32.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: cluster 2023-10-09T08:51:31.413794+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:32.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: audit 2023-10-09T08:51:31.417008+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:51:32.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: cluster 2023-10-09T08:51:31.417321+0000 mon.a (mon.0) 537 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T08:51:32.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: cluster 2023-10-09T08:51:31.422172+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:32.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: audit 2023-10-09T08:51:31.969336+0000 mgr.y (mgr.25421) 415 : audit 0 from='client.54589 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:32.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[75179]: audit 2023-10-09T08:51:31.970077+0000 mon.a (mon.0) 538 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54175','3':'41807','4':'65536','6':'65536'}"}]: dispatch 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: cluster 2023-10-09T08:51:31.404319+0000 mon.a (mon.0) 534 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: cluster 2023-10-09T08:51:31.404364+0000 mon.a (mon.0) 535 : cluster 1 Cluster is now healthy 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: cluster 2023-10-09T08:51:31.413794+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: audit 2023-10-09T08:51:31.417008+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: cluster 2023-10-09T08:51:31.417321+0000 mon.a (mon.0) 537 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: cluster 2023-10-09T08:51:31.422172+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: audit 2023-10-09T08:51:31.969336+0000 mgr.y (mgr.25421) 415 : audit 0 from='client.54589 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:32.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:32 smithi016 bash[77892]: audit 2023-10-09T08:51:31.970077+0000 mon.a (mon.0) 538 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54175','3':'41807','4':'65536','6':'65536'}"}]: dispatch 2023-10-09T08:51:32.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:32.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:32.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:32.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: cluster 2023-10-09T08:51:31.404319+0000 mon.a (mon.0) 534 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: cluster 2023-10-09T08:51:31.404364+0000 mon.a (mon.0) 535 : cluster 1 Cluster is now healthy 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: cluster 2023-10-09T08:51:31.413794+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: audit 2023-10-09T08:51:31.417008+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: cluster 2023-10-09T08:51:31.417321+0000 mon.a (mon.0) 537 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: cluster 2023-10-09T08:51:31.422172+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: audit 2023-10-09T08:51:31.969336+0000 mgr.y (mgr.25421) 415 : audit 0 from='client.54589 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:32.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:32 smithi094 bash[59677]: audit 2023-10-09T08:51:31.970077+0000 mon.a (mon.0) 538 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54175','3':'41807','4':'65536','6':'65536'}"}]: dispatch 2023-10-09T08:51:32.734 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:32.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:32.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:32.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:32.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:32.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:33.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:33.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:33.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:33.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:33.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:33.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:33.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:33.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:33.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 71391 71375 92.9232 3.9375 0.00386536 0.0105383 2023-10-09T08:51:33.500 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:33.515 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:33 smithi094 bash[59677]: cluster 2023-10-09T08:51:31.432135+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:33.515 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:33 smithi094 bash[59677]: cluster 2023-10-09T08:51:32.417147+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T08:51:33.515 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:33 smithi094 bash[59677]: audit 2023-10-09T08:51:32.418943+0000 mon.a (mon.0) 539 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'54175','3':'41807','4':'65536','6':'65536'}"}]': finished 2023-10-09T08:51:33.515 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:33 smithi094 bash[59677]: cluster 2023-10-09T08:51:32.419110+0000 mon.a (mon.0) 540 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T08:51:33.515 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:33 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.343701+0000 mgr.y (mgr.25421) 416 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-10-09T08:51:33.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:33.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:33.671 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:33.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[75179]: cluster 2023-10-09T08:51:31.432135+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[75179]: cluster 2023-10-09T08:51:32.417147+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[75179]: audit 2023-10-09T08:51:32.418943+0000 mon.a (mon.0) 539 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'54175','3':'41807','4':'65536','6':'65536'}"}]': finished 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[75179]: cluster 2023-10-09T08:51:32.419110+0000 mon.a (mon.0) 540 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.343701+0000 mgr.y (mgr.25421) 416 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[77892]: cluster 2023-10-09T08:51:31.432135+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[77892]: cluster 2023-10-09T08:51:32.417147+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[77892]: audit 2023-10-09T08:51:32.418943+0000 mon.a (mon.0) 539 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'54175','3':'41807','4':'65536','6':'65536'}"}]': finished 2023-10-09T08:51:33.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[77892]: cluster 2023-10-09T08:51:32.419110+0000 mon.a (mon.0) 540 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T08:51:33.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:33 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.343701+0000 mgr.y (mgr.25421) 416 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-10-09T08:51:33.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:33.805 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:33.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:33.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:33.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:34.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:34.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:34.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:34.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T08:51:34.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:34.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:34.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:34.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:34.261 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T08:51:34.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-10-09T08:51:34.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:34.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:34.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:34.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:34.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 71391 71375 91.0268 0 - 0.0105383 2023-10-09T08:51:34.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: cluster 2023-10-09T08:51:32.416370+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.422377+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.424208+0000 mon.a (mon.0) 541 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.424884+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.427821+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.431208+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.431652+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: audit 2023-10-09T08:51:34.211638+0000 mon.a (mon.0) 542 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:34.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[75179]: audit 2023-10-09T08:51:34.212803+0000 mon.a (mon.0) 543 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: cluster 2023-10-09T08:51:32.416370+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.422377+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.424208+0000 mon.a (mon.0) 541 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.424884+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.427821+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.431208+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.431652+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: audit 2023-10-09T08:51:34.211638+0000 mon.a (mon.0) 542 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:34.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:34 smithi016 bash[77892]: audit 2023-10-09T08:51:34.212803+0000 mon.a (mon.0) 543 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:34.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:34.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:34.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:34.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:34.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:34.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:34.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:34.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: cluster 2023-10-09T08:51:32.416370+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.422377+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.424208+0000 mon.a (mon.0) 541 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.424884+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.427821+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.431208+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.431652+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: audit 2023-10-09T08:51:34.211638+0000 mon.a (mon.0) 542 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:34 smithi094 bash[59677]: audit 2023-10-09T08:51:34.212803+0000 mon.a (mon.0) 543 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:34.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:34.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:34.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:34.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:34.893 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:34.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:34.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:35.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:35.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:35.134 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:35.165 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:35.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:35.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:35.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:35.406 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:35.435 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:35.453 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 72023 72007 89.9962 19.75 0.0445995 0.0111027 2023-10-09T08:51:35.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:35.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:35.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:35.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.425968+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.428681+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:33.430703+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:34.426735+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:34.428252+0000 mon.a (mon.0) 544 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T08:51:35.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:34.428366+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:34.430186+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:34.457157+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[77892]: cluster 2023-10-09T08:51:35.344536+0000 mgr.y (mgr.25421) 417 : cluster 0 pgmap v358: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s; 4576/214461 objects degraded (2.134%); 4419/214461 objects misplaced (2.061%); 288 KiB/s, 4 objects/s recovering 2023-10-09T08:51:35.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.425968+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.428681+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:33.430703+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:34.426735+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:34.428252+0000 mon.a (mon.0) 544 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T08:51:35.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:34.428366+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:34.430186+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:34.457157+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[75179]: cluster 2023-10-09T08:51:35.344536+0000 mgr.y (mgr.25421) 417 : cluster 0 pgmap v358: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s; 4576/214461 objects degraded (2.134%); 4419/214461 objects misplaced (2.061%); 288 KiB/s, 4 objects/s recovering 2023-10-09T08:51:35.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:35.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:35.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.425968+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.428681+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:33.430703+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:34.426735+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:34.428252+0000 mon.a (mon.0) 544 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:34.428366+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:34.430186+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:34.457157+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:35.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:35 smithi094 bash[59677]: cluster 2023-10-09T08:51:35.344536+0000 mgr.y (mgr.25421) 417 : cluster 0 pgmap v358: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s; 4576/214461 objects degraded (2.134%); 4419/214461 objects misplaced (2.061%); 288 KiB/s, 4 objects/s recovering 2023-10-09T08:51:35.818 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:35.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:35.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:35.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:35.973 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:51:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:51:35] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.33.4" 2023-10-09T08:51:35.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:35.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:35.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:36.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:36.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:36.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:36.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:36.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:36.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:36.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:36.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:51:36.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T08:51:36.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:36.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:36.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:36.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:36.454 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 72943 72927 89.359 57.5 0.0124803 0.0111852 2023-10-09T08:51:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:36 smithi094 bash[59677]: cluster 2023-10-09T08:51:34.606214+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:36 smithi094 bash[59677]: cluster 2023-10-09T08:51:34.611279+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:36 smithi094 bash[59677]: cluster 2023-10-09T08:51:35.422436+0000 mon.a (mon.0) 545 : cluster 3 Health check failed: Degraded data redundancy: 4576/214461 objects degraded (2.134%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:51:36.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:36.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:36.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:36.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:36.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:36 smithi016 bash[75179]: cluster 2023-10-09T08:51:34.606214+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:36.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:36 smithi016 bash[75179]: cluster 2023-10-09T08:51:34.611279+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:36.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:36 smithi016 bash[75179]: cluster 2023-10-09T08:51:35.422436+0000 mon.a (mon.0) 545 : cluster 3 Health check failed: Degraded data redundancy: 4576/214461 objects degraded (2.134%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:51:36.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:36 smithi016 bash[77892]: cluster 2023-10-09T08:51:34.606214+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:36.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:36 smithi016 bash[77892]: cluster 2023-10-09T08:51:34.611279+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T08:51:36.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:36 smithi016 bash[77892]: cluster 2023-10-09T08:51:35.422436+0000 mon.a (mon.0) 545 : cluster 3 Health check failed: Degraded data redundancy: 4576/214461 objects degraded (2.134%), 1 pg degraded (PG_DEGRADED) 2023-10-09T08:51:36.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:36.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:36.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:36.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:36.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:36.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:36.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:36.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:37.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:37.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:37.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:37.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:37.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:37.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:37.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:37.317 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:37.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:37.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:37.454 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 73850 73834 88.7306 56.6875 0.0101319 0.0112628 2023-10-09T08:51:37.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:37.467 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:37 smithi094 bash[59677]: cluster 2023-10-09T08:51:37.345157+0000 mgr.y (mgr.25421) 418 : cluster 0 pgmap v359: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 9064/216924 objects degraded (4.178%); 4419/216924 objects misplaced (2.037%); 194 KiB/s, 3 objects/s recovering 2023-10-09T08:51:37.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:37.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:37.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:37.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:37.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:37.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:37 smithi016 bash[75179]: cluster 2023-10-09T08:51:37.345157+0000 mgr.y (mgr.25421) 418 : cluster 0 pgmap v359: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 9064/216924 objects degraded (4.178%); 4419/216924 objects misplaced (2.037%); 194 KiB/s, 3 objects/s recovering 2023-10-09T08:51:37.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:37 smithi016 bash[77892]: cluster 2023-10-09T08:51:37.345157+0000 mgr.y (mgr.25421) 418 : cluster 0 pgmap v359: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 9064/216924 objects degraded (4.178%); 4419/216924 objects misplaced (2.037%); 194 KiB/s, 3 objects/s recovering 2023-10-09T08:51:37.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:37.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:37.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:37.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:37.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:37.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:37.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:38.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:38.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:38.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:38.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:38.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:38.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:38.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:38.300 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:38.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:38.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:38.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:38.458 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 74767 74751 88.1377 57.3125 0.0138438 0.0113402 2023-10-09T08:51:38.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:38.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:38.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:38.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:38.606 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:38.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:38.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:38.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:38.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:38.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:38.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:38.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:38.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:38.955 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:38.958 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:38.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:39.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:39.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:39.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:39.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:39.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:39.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:39.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:39.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:39.454 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 75665 75649 87.5448 56.125 0.0176821 0.011417 2023-10-09T08:51:39.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:39.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:39.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:39.664 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:39.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:39.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:39.797 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:39.823 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:39.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:39.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:39.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:40.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:40.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:40.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:40.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:40.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:40.209 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:39 smithi094 bash[59677]: cluster 2023-10-09T08:51:39.345661+0000 mgr.y (mgr.25421) 419 : cluster 0 pgmap v360: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 24 MiB/s wr, 377 op/s; 9064/216924 objects degraded (4.178%); 4419/216924 objects misplaced (2.037%); 166 KiB/s, 2 objects/s recovering 2023-10-09T08:51:40.230 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:40.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:40.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:39 smithi016 bash[75179]: cluster 2023-10-09T08:51:39.345661+0000 mgr.y (mgr.25421) 419 : cluster 0 pgmap v360: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 24 MiB/s wr, 377 op/s; 9064/216924 objects degraded (4.178%); 4419/216924 objects misplaced (2.037%); 166 KiB/s, 2 objects/s recovering 2023-10-09T08:51:40.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:39 smithi016 bash[77892]: cluster 2023-10-09T08:51:39.345661+0000 mgr.y (mgr.25421) 419 : cluster 0 pgmap v360: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 24 MiB/s wr, 377 op/s; 9064/216924 objects degraded (4.178%); 4419/216924 objects misplaced (2.037%); 166 KiB/s, 2 objects/s recovering 2023-10-09T08:51:40.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:40.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:40.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:40.454 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 76581 76565 86.9939 57.25 0.0122015 0.0114886 2023-10-09T08:51:40.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:40.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:40.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:40.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:40.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:40.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:40.770 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:40.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:40.876 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:40.902 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:40.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:40.997 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:41.001 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:51:41.005 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:41.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:41.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:41.136 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:41.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:41.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:41.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:41.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:41.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:41.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:41.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:41.454 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 77493 77477 86.4582 57 0.0114079 0.0115602 2023-10-09T08:51:41.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:41.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:41.679 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:41.688 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:41 smithi094 bash[59677]: cluster 2023-10-09T08:51:41.346259+0000 mgr.y (mgr.25421) 420 : cluster 0 pgmap v361: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 9064/226338 objects degraded (4.005%); 4287/226338 objects misplaced (1.894%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T08:51:41.689 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:41 smithi094 bash[59677]: cluster 2023-10-09T08:51:41.397377+0000 mon.a (mon.0) 546 : cluster 3 Health check update: Degraded data redundancy: 9064/226338 objects degraded (4.005%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:41.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:41.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:41.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:41 smithi016 bash[75179]: cluster 2023-10-09T08:51:41.346259+0000 mgr.y (mgr.25421) 420 : cluster 0 pgmap v361: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 9064/226338 objects degraded (4.005%); 4287/226338 objects misplaced (1.894%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T08:51:41.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:41 smithi016 bash[75179]: cluster 2023-10-09T08:51:41.397377+0000 mon.a (mon.0) 546 : cluster 3 Health check update: Degraded data redundancy: 9064/226338 objects degraded (4.005%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:41.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:41 smithi016 bash[77892]: cluster 2023-10-09T08:51:41.346259+0000 mgr.y (mgr.25421) 420 : cluster 0 pgmap v361: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 9064/226338 objects degraded (4.005%); 4287/226338 objects misplaced (1.894%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T08:51:41.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:41 smithi016 bash[77892]: cluster 2023-10-09T08:51:41.397377+0000 mon.a (mon.0) 546 : cluster 3 Health check update: Degraded data redundancy: 9064/226338 objects degraded (4.005%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:41.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:41.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:41.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:41.969 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:41.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:42.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:42.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:42.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:42.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:42.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:42.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:42.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:42.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:42.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:42.454 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 78384 78368 85.9183 55.6875 0.00990076 0.0116327 2023-10-09T08:51:42.488 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:42.513 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:42.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:42.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:42.616 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:42.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:42.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:42.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:42.746 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:42.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:42.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:42.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:42.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:42.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:42.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:42.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:43.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:43.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:43.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:43.173 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:43.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:43.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:43.277 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:43.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:43.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:43.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:43.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:43.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:43.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:43.455 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 79282 79266 85.4025 56.125 0.0349829 0.0117029 2023-10-09T08:51:43.549 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:43.557 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:43 smithi094 bash[59677]: cluster 2023-10-09T08:51:43.346915+0000 mgr.y (mgr.25421) 421 : cluster 0 pgmap v362: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s; 9064/230616 objects degraded (3.930%); 4287/230616 objects misplaced (1.859%); 967 KiB/s, 15 objects/s recovering 2023-10-09T08:51:43.573 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:43.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:43.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:43.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:43.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:43 smithi016 bash[75179]: cluster 2023-10-09T08:51:43.346915+0000 mgr.y (mgr.25421) 421 : cluster 0 pgmap v362: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s; 9064/230616 objects degraded (3.930%); 4287/230616 objects misplaced (1.859%); 967 KiB/s, 15 objects/s recovering 2023-10-09T08:51:43.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:43 smithi016 bash[77892]: cluster 2023-10-09T08:51:43.346915+0000 mgr.y (mgr.25421) 421 : cluster 0 pgmap v362: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s; 9064/230616 objects degraded (3.930%); 4287/230616 objects misplaced (1.859%); 967 KiB/s, 15 objects/s recovering 2023-10-09T08:51:43.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:43.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:43.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:43.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:43.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:43.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:43.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:44.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:44.108 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:44.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:44.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:44.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:44.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:44.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:44.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:44.408 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:44.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:44.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 80157 80141 84.8818 54.6875 0.0082195 0.0117755 2023-10-09T08:51:44.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:44.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:44.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:44.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:44.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:44.826 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:44.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:44.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:44.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:44.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:44.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:45.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:45.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:45.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:45.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:45.230 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:45.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:45.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:45.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:45.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:45.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:45.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:45.421 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:45 smithi016 bash[75179]: cluster 2023-10-09T08:51:45.347824+0000 mgr.y (mgr.25421) 422 : cluster 0 pgmap v363: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 9064/239931 objects degraded (3.778%); 4163/239931 objects misplaced (1.735%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:51:45.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:51:45.453951+0000 min lat: 0.00138239 max lat: 2.36633 avg lat: 0.0118483 2023-10-09T08:51:45.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:51:45.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 81010 80994 84.3556 53.3125 0.0468251 0.0118483 2023-10-09T08:51:45.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:45.478 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:45 smithi094 bash[59677]: cluster 2023-10-09T08:51:45.347824+0000 mgr.y (mgr.25421) 422 : cluster 0 pgmap v363: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 9064/239931 objects degraded (3.778%); 4163/239931 objects misplaced (1.735%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:51:45.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:45.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:45.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:45.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:45.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:45.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:45 smithi016 bash[77892]: cluster 2023-10-09T08:51:45.347824+0000 mgr.y (mgr.25421) 422 : cluster 0 pgmap v363: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 9064/239931 objects degraded (3.778%); 4163/239931 objects misplaced (1.735%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:51:45.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:45.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:45.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:45.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:45.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:45.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:45.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:45.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:45.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:45.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:46.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:51:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:51:45] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.33.4" 2023-10-09T08:51:46.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:46.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:46.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:46.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:46.168 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:46.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:46.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:46.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:46.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:46.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:46.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 81864 81848 83.8477 53.375 0.00799434 0.0119192 2023-10-09T08:51:46.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:46.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:46.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:46.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:46.693 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:46.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:46.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:46.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:46.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:46.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:46.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:47.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:47.082 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:47.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:47.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:47.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:47.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:47.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:47.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:47.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:47.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:47.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 82731 82715 83.3692 54.1875 0.0090016 0.0119884 2023-10-09T08:51:47.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:47.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:47.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:47.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:47.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:47 smithi016 bash[75179]: cluster 2023-10-09T08:51:47.348431+0000 mgr.y (mgr.25421) 423 : cluster 0 pgmap v364: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 9064/243885 objects degraded (3.717%); 4163/243885 objects misplaced (1.707%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:51:47.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:47 smithi016 bash[75179]: cluster 2023-10-09T08:51:47.399232+0000 mon.a (mon.0) 547 : cluster 3 Health check update: Degraded data redundancy: 9064/243885 objects degraded (3.717%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:47.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:47 smithi016 bash[77892]: cluster 2023-10-09T08:51:47.348431+0000 mgr.y (mgr.25421) 423 : cluster 0 pgmap v364: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 9064/243885 objects degraded (3.717%); 4163/243885 objects misplaced (1.707%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:51:47.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:47 smithi016 bash[77892]: cluster 2023-10-09T08:51:47.399232+0000 mon.a (mon.0) 547 : cluster 3 Health check update: Degraded data redundancy: 9064/243885 objects degraded (3.717%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:47.595 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:47.608 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:47 smithi094 bash[59677]: cluster 2023-10-09T08:51:47.348431+0000 mgr.y (mgr.25421) 423 : cluster 0 pgmap v364: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 9064/243885 objects degraded (3.717%); 4163/243885 objects misplaced (1.707%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:51:47.608 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:47 smithi094 bash[59677]: cluster 2023-10-09T08:51:47.399232+0000 mon.a (mon.0) 547 : cluster 3 Health check update: Degraded data redundancy: 9064/243885 objects degraded (3.717%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:47.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:47.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:47.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:47.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:47.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:47.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:47.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:47.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:47.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:47.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:47.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:48.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:48.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:48.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:48.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:48.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:48.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:48.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:48.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:48.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:48.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 83579 83563 82.8871 53 0.0133305 0.0120581 2023-10-09T08:51:48.529 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:48.559 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:48.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:48.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:48.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:48.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:48.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:48.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:48.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:48.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:48.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:48.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:49.073 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:49.100 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:49.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:49.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:49.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:49.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:49.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:49.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:49.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:49.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:49.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:49.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 84401 84385 82.3947 51.375 0.0119105 0.0121296 2023-10-09T08:51:49.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:49.510 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:49.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:49.614 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:49.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:49.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:49.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:49.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:49.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:49.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:49.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:49.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:49.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:50.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[75179]: audit 2023-10-09T08:51:49.144630+0000 mon.a (mon.0) 548 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[75179]: audit 2023-10-09T08:51:49.226616+0000 mon.a (mon.0) 549 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[75179]: audit 2023-10-09T08:51:49.228485+0000 mon.a (mon.0) 550 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[75179]: cluster 2023-10-09T08:51:49.348920+0000 mgr.y (mgr.25421) 424 : cluster 0 pgmap v365: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 9064/243885 objects degraded (3.717%); 4163/243885 objects misplaced (1.707%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[77892]: audit 2023-10-09T08:51:49.144630+0000 mon.a (mon.0) 548 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[77892]: audit 2023-10-09T08:51:49.226616+0000 mon.a (mon.0) 549 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[77892]: audit 2023-10-09T08:51:49.228485+0000 mon.a (mon.0) 550 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:50.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:49 smithi016 bash[77892]: cluster 2023-10-09T08:51:49.348920+0000 mgr.y (mgr.25421) 424 : cluster 0 pgmap v365: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 9064/243885 objects degraded (3.717%); 4163/243885 objects misplaced (1.707%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:51:50.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:50.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:50.195 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:49 smithi094 bash[59677]: audit 2023-10-09T08:51:49.144630+0000 mon.a (mon.0) 548 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:51:50.195 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:49 smithi094 bash[59677]: audit 2023-10-09T08:51:49.226616+0000 mon.a (mon.0) 549 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:50.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:49 smithi094 bash[59677]: audit 2023-10-09T08:51:49.228485+0000 mon.a (mon.0) 550 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:51:50.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:49 smithi094 bash[59677]: cluster 2023-10-09T08:51:49.348920+0000 mgr.y (mgr.25421) 424 : cluster 0 pgmap v365: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 9064/243885 objects degraded (3.717%); 4163/243885 objects misplaced (1.707%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:51:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:50.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:50.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:50.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:50.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:50.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 85224 85208 81.9184 51.4375 0.013876 0.0122015 2023-10-09T08:51:50.459 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:50.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:50.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:50.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:50.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:50.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:50.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:50.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:50.964 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:50.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:50.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:51.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:51.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:51.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:51.221 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:51.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:51.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:51.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:51.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:51.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:51.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:51.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 86035 86019 81.4452 50.6875 0.0232084 0.0122722 2023-10-09T08:51:51.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:51 smithi094 bash[59677]: cluster 2023-10-09T08:51:51.349668+0000 mgr.y (mgr.25421) 425 : cluster 0 pgmap v366: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 9064/252666 objects degraded (3.587%); 4039/252666 objects misplaced (1.599%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:51:51.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:51.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:51.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:51.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:51.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:51.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:51.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:51.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:51 smithi016 bash[75179]: cluster 2023-10-09T08:51:51.349668+0000 mgr.y (mgr.25421) 425 : cluster 0 pgmap v366: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 9064/252666 objects degraded (3.587%); 4039/252666 objects misplaced (1.599%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:51:51.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:51.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:51 smithi016 bash[77892]: cluster 2023-10-09T08:51:51.349668+0000 mgr.y (mgr.25421) 425 : cluster 0 pgmap v366: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 9064/252666 objects degraded (3.587%); 4039/252666 objects misplaced (1.599%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:51:51.690 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:51:51.691 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T08:51:51.691 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T08:51:51.691 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T08:51:51.691 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:51:51.710 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:51:51.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:51.778 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:51:51.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:51.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:51.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:51.823 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:51:51.876 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:51:51.891 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:51.910 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:51:51.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:51.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:51.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:51.950 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:51:51.985 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:51:52.023 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:51:52.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:52.036 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.037 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.037 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.037 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.037 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.037 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.037 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.037 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:51:52.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:52.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:52.201 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:52.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:52.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:52.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:52.428 INFO:teuthology.orchestra.run.smithi016.stdout:moved 6 / 267 (2.24719%) 2023-10-09T08:51:52.428 INFO:teuthology.orchestra.run.smithi016.stdout:avg 33.375 2023-10-09T08:51:52.428 INFO:teuthology.orchestra.run.smithi016.stdout:stddev 5.4758 -> 6.30352 (expected baseline 5.40399) 2023-10-09T08:51:52.428 INFO:teuthology.orchestra.run.smithi016.stdout:min osd.2 with 25 -> 24 pgs (0.749064 -> 0.719101 * mean) 2023-10-09T08:51:52.428 INFO:teuthology.orchestra.run.smithi016.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T08:51:52.428 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:51:52.428 INFO:teuthology.orchestra.run.smithi016.stdout:oload 110 2023-10-09T08:51:52.429 INFO:teuthology.orchestra.run.smithi016.stdout:max_change 0.05 2023-10-09T08:51:52.429 INFO:teuthology.orchestra.run.smithi016.stdout:max_change_osds 4 2023-10-09T08:51:52.429 INFO:teuthology.orchestra.run.smithi016.stdout:average_utilization 0.0235 2023-10-09T08:51:52.429 INFO:teuthology.orchestra.run.smithi016.stdout:overload_utilization 0.0259 2023-10-09T08:51:52.429 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 weight 0.6379 -> 0.5879 2023-10-09T08:51:52.429 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 weight 0.8266 -> 0.7767 2023-10-09T08:51:52.429 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 weight 0.9500 -> 1.0000 2023-10-09T08:51:52.433 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:52 smithi094 bash[59677]: audit 2023-10-09T08:51:51.638639+0000 mon.a (mon.0) 551 : audit 0 from='client.? 172.21.15.16:0/2458726672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:52.433 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:52 smithi094 bash[59677]: audit 2023-10-09T08:51:52.338149+0000 mgr.y (mgr.25421) 426 : audit 0 from='client.54595 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:52.433 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:52 smithi094 bash[59677]: audit 2023-10-09T08:51:52.339096+0000 mon.a (mon.0) 552 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'50899','3':'38531','5':'65536'}"}]: dispatch 2023-10-09T08:51:52.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:52.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 86746 86730 80.8928 44.4375 0.0414041 0.0123544 2023-10-09T08:51:52.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:52.570 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:52.594 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:52.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:52.728 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:52.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:52.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:52 smithi016 bash[75179]: audit 2023-10-09T08:51:51.638639+0000 mon.a (mon.0) 551 : audit 0 from='client.? 172.21.15.16:0/2458726672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:52.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:52 smithi016 bash[75179]: audit 2023-10-09T08:51:52.338149+0000 mgr.y (mgr.25421) 426 : audit 0 from='client.54595 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:52.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:52 smithi016 bash[75179]: audit 2023-10-09T08:51:52.339096+0000 mon.a (mon.0) 552 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'50899','3':'38531','5':'65536'}"}]: dispatch 2023-10-09T08:51:52.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:52 smithi016 bash[77892]: audit 2023-10-09T08:51:51.638639+0000 mon.a (mon.0) 551 : audit 0 from='client.? 172.21.15.16:0/2458726672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:51:52.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:52 smithi016 bash[77892]: audit 2023-10-09T08:51:52.338149+0000 mgr.y (mgr.25421) 426 : audit 0 from='client.54595 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:51:52.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:52 smithi016 bash[77892]: audit 2023-10-09T08:51:52.339096+0000 mon.a (mon.0) 552 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'50899','3':'38531','5':'65536'}"}]: dispatch 2023-10-09T08:51:52.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:52.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:52.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:52.993 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:52.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:53.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:53.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:53.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:53.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:53.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:53.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:53.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:53.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:53.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:53.451 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T08:51:53.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 86831 86815 79.7814 5.3125 0.00393701 0.0123523 2023-10-09T08:51:53.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:53.508 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:53.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:53.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:53.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:53.632 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:53.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:53.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:53.735 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:53 smithi094 bash[59677]: cluster 2023-10-09T08:51:52.422079+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T08:51:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:53 smithi094 bash[59677]: audit 2023-10-09T08:51:52.424485+0000 mon.a (mon.0) 553 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'50899','3':'38531','5':'65536'}"}]': finished 2023-10-09T08:51:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:53 smithi094 bash[59677]: cluster 2023-10-09T08:51:52.424563+0000 mon.a (mon.0) 554 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T08:51:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:53 smithi094 bash[59677]: cluster 2023-10-09T08:51:52.432732+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T08:51:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:53 smithi094 bash[59677]: audit 2023-10-09T08:51:52.451733+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.16:0/265456269' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:51:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:53 smithi094 bash[59677]: audit 2023-10-09T08:51:52.453026+0000 mon.a (mon.0) 555 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:51:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:53 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.350411+0000 mgr.y (mgr.25421) 427 : cluster 0 pgmap v368: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 9064/256503 objects degraded (3.534%); 4039/256503 objects misplaced (1.575%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T08:51:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[77892]: cluster 2023-10-09T08:51:52.422079+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T08:51:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[77892]: audit 2023-10-09T08:51:52.424485+0000 mon.a (mon.0) 553 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'50899','3':'38531','5':'65536'}"}]': finished 2023-10-09T08:51:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[77892]: cluster 2023-10-09T08:51:52.424563+0000 mon.a (mon.0) 554 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T08:51:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[77892]: cluster 2023-10-09T08:51:52.432732+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T08:51:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[77892]: audit 2023-10-09T08:51:52.451733+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.16:0/265456269' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:51:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[77892]: audit 2023-10-09T08:51:52.453026+0000 mon.a (mon.0) 555 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:51:53.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.350411+0000 mgr.y (mgr.25421) 427 : cluster 0 pgmap v368: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 9064/256503 objects degraded (3.534%); 4039/256503 objects misplaced (1.575%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T08:51:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[75179]: cluster 2023-10-09T08:51:52.422079+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T08:51:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[75179]: audit 2023-10-09T08:51:52.424485+0000 mon.a (mon.0) 553 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'50899','3':'38531','5':'65536'}"}]': finished 2023-10-09T08:51:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[75179]: cluster 2023-10-09T08:51:52.424563+0000 mon.a (mon.0) 554 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T08:51:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[75179]: cluster 2023-10-09T08:51:52.432732+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T08:51:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[75179]: audit 2023-10-09T08:51:52.451733+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.16:0/265456269' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:51:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[75179]: audit 2023-10-09T08:51:52.453026+0000 mon.a (mon.0) 555 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:51:53.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:53 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.350411+0000 mgr.y (mgr.25421) 427 : cluster 0 pgmap v368: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 9064/256503 objects degraded (3.534%); 4039/256503 objects misplaced (1.575%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T08:51:53.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:53.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:53.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:53.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:53.901 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:53.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:54.005 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:54.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:54.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:54.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:54.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:54.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:54.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:54.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:54.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:54.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 86831 86815 78.6252 0 - 0.0123523 2023-10-09T08:51:54.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:54.535 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:54.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:54.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.429190+0000 mon.a (mon.0) 556 : cluster 3 Health check update: Degraded data redundancy: 9064/256503 objects degraded (3.534%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.429827+0000 mon.a (mon.0) 557 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.439436+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: audit 2023-10-09T08:51:53.444082+0000 mon.a (mon.0) 558 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.444282+0000 mon.a (mon.0) 559 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.444414+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.444500+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.444577+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.429190+0000 mon.a (mon.0) 556 : cluster 3 Health check update: Degraded data redundancy: 9064/256503 objects degraded (3.534%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:54.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.429827+0000 mon.a (mon.0) 557 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:51:54.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.439436+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: audit 2023-10-09T08:51:53.444082+0000 mon.a (mon.0) 558 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:51:54.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.444282+0000 mon.a (mon.0) 559 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T08:51:54.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.444414+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.444500+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:54 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.444577+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:54.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.429190+0000 mon.a (mon.0) 556 : cluster 3 Health check update: Degraded data redundancy: 9064/256503 objects degraded (3.534%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.429827+0000 mon.a (mon.0) 557 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:51:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.439436+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: audit 2023-10-09T08:51:53.444082+0000 mon.a (mon.0) 558 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:51:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.444282+0000 mon.a (mon.0) 559 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T08:51:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.444414+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.444500+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:54 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.444577+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:54.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:54.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:54.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:54.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:54.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:55.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:55.089 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:55.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T08:51:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:55.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:55.195 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:55.239 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:55.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:55.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:55.342 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:55.381 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:55.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:55.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 87288 87272 77.91 14.2812 0.00866472 0.0128274 2023-10-09T08:51:55.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:55.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:55.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:55.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:55.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:55.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:55.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:55.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[75179]: cluster 2023-10-09T08:51:53.447741+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:55.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[75179]: cluster 2023-10-09T08:51:54.445949+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[75179]: cluster 2023-10-09T08:51:54.447908+0000 mon.a (mon.0) 560 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T08:51:55.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[75179]: cluster 2023-10-09T08:51:54.448439+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[75179]: cluster 2023-10-09T08:51:54.450182+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[75179]: cluster 2023-10-09T08:51:55.351263+0000 mgr.y (mgr.25421) 428 : cluster 0 pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 23784/260805 objects degraded (9.119%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T08:51:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[77892]: cluster 2023-10-09T08:51:53.447741+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[77892]: cluster 2023-10-09T08:51:54.445949+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[77892]: cluster 2023-10-09T08:51:54.447908+0000 mon.a (mon.0) 560 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T08:51:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[77892]: cluster 2023-10-09T08:51:54.448439+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[77892]: cluster 2023-10-09T08:51:54.450182+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[77892]: cluster 2023-10-09T08:51:55.351263+0000 mgr.y (mgr.25421) 428 : cluster 0 pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 23784/260805 objects degraded (9.119%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T08:51:55.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:55 smithi094 bash[59677]: cluster 2023-10-09T08:51:53.447741+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T08:51:55.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:55 smithi094 bash[59677]: cluster 2023-10-09T08:51:54.445949+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:55 smithi094 bash[59677]: cluster 2023-10-09T08:51:54.447908+0000 mon.a (mon.0) 560 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T08:51:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:55 smithi094 bash[59677]: cluster 2023-10-09T08:51:54.448439+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:55 smithi094 bash[59677]: cluster 2023-10-09T08:51:54.450182+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:55.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:55 smithi094 bash[59677]: cluster 2023-10-09T08:51:55.351263+0000 mgr.y (mgr.25421) 428 : cluster 0 pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 23784/260805 objects degraded (9.119%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T08:51:55.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:55.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:55.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:55.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:55.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:55.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:55.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:56.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:51:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:51:55] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.33.4" 2023-10-09T08:51:56.032 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:56.058 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:56.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:56.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:56.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:56.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:56.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:56.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:56.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:56.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:56.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:56.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:56.456 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 87938 87922 77.3848 40.625 0.0516538 0.0129147 2023-10-09T08:51:56.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:56 smithi094 bash[59677]: cluster 2023-10-09T08:51:54.448577+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:56.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:56 smithi094 bash[59677]: cluster 2023-10-09T08:51:54.451419+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:56.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:56 smithi094 bash[59677]: audit 2023-10-09T08:51:55.785331+0000 mon.a (mon.0) 561 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:56.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:56 smithi094 bash[59677]: audit 2023-10-09T08:51:55.792377+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:56.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:56.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:56.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:56.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:56.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:56.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:56.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:56.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:56.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[75179]: cluster 2023-10-09T08:51:54.448577+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:56.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[75179]: cluster 2023-10-09T08:51:54.451419+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:56.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[75179]: audit 2023-10-09T08:51:55.785331+0000 mon.a (mon.0) 561 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:56.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[75179]: audit 2023-10-09T08:51:55.792377+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:56.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[77892]: cluster 2023-10-09T08:51:54.448577+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:56.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[77892]: cluster 2023-10-09T08:51:54.451419+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T08:51:56.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[77892]: audit 2023-10-09T08:51:55.785331+0000 mon.a (mon.0) 561 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:56.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:56 smithi016 bash[77892]: audit 2023-10-09T08:51:55.792377+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:51:56.757 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:56.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:56.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:56.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:56.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:56.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:56.990 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:57.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:57.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:57.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:57.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:57.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:57.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:57.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:57.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:57.323 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:57.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:57.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:57.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 88651 88635 76.9289 44.5625 0.00972298 0.0129917 2023-10-09T08:51:57.457 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:57.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:57.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:57.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:57 smithi094 bash[59677]: cluster 2023-10-09T08:51:57.351942+0000 mgr.y (mgr.25421) 429 : cluster 0 pgmap v372: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s; 24705/262251 objects degraded (9.420%); 4354/262251 objects misplaced (1.660%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:51:57.599 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:57.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:57.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:57 smithi016 bash[75179]: cluster 2023-10-09T08:51:57.351942+0000 mgr.y (mgr.25421) 429 : cluster 0 pgmap v372: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s; 24705/262251 objects degraded (9.420%); 4354/262251 objects misplaced (1.660%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:51:57.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:57 smithi016 bash[77892]: cluster 2023-10-09T08:51:57.351942+0000 mgr.y (mgr.25421) 429 : cluster 0 pgmap v372: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s; 24705/262251 objects degraded (9.420%); 4354/262251 objects misplaced (1.660%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:51:57.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:57.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:57.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:57.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:51:57.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:57.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:58.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:58.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T08:51:58.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:51:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:58.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:51:58.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:58.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:58.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:58.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:58.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:58.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:58.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 89341 89325 76.4655 43.125 0.0169105 0.0130715 2023-10-09T08:51:58.472 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:58.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:51:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:58.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:51:58.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:58.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:58.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:58.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:58.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:58.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:51:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:58.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:51:58.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:58.933 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:58.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:51:59.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:59.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:51:58 smithi016 bash[75179]: cluster 2023-10-09T08:51:58.923081+0000 mon.a (mon.0) 563 : cluster 3 Health check update: Degraded data redundancy: 24705/262251 objects degraded (9.420%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:59.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:51:58 smithi016 bash[77892]: cluster 2023-10-09T08:51:58.923081+0000 mon.a (mon.0) 563 : cluster 3 Health check update: Degraded data redundancy: 24705/262251 objects degraded (9.420%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:59.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:59.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:59.181 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:51:58 smithi094 bash[59677]: cluster 2023-10-09T08:51:58.923081+0000 mon.a (mon.0) 563 : cluster 3 Health check update: Degraded data redundancy: 24705/262251 objects degraded (9.420%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:51:59.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:51:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:51:59.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:59.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:51:59.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:51:59.433 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:59.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 15 90033 90018 76.0175 43.3125 0.0200341 0.0131491 2023-10-09T08:51:59.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:51:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:51:59.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:59.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:59.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:59.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:59.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:59.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:59.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:51:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:51:59.964 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:59.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:51:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:51:59.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:00.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:00.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[75179]: audit 2023-10-09T08:51:59.049388+0000 mon.a (mon.0) 564 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[75179]: audit 2023-10-09T08:51:59.060259+0000 mon.a (mon.0) 565 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[75179]: cluster 2023-10-09T08:51:59.352578+0000 mgr.y (mgr.25421) 430 : cluster 0 pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 18 MiB/s wr, 293 op/s; 24705/262611 objects degraded (9.407%); 4354/262611 objects misplaced (1.658%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:52:00.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[75179]: audit 2023-10-09T08:51:59.641122+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:52:00.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[75179]: audit 2023-10-09T08:51:59.643060+0000 mon.a (mon.0) 567 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:52:00.102 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[75179]: audit 2023-10-09T08:51:59.651015+0000 mon.a (mon.0) 568 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[77892]: audit 2023-10-09T08:51:59.049388+0000 mon.a (mon.0) 564 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[77892]: audit 2023-10-09T08:51:59.060259+0000 mon.a (mon.0) 565 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[77892]: cluster 2023-10-09T08:51:59.352578+0000 mgr.y (mgr.25421) 430 : cluster 0 pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 18 MiB/s wr, 293 op/s; 24705/262611 objects degraded (9.407%); 4354/262611 objects misplaced (1.658%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:52:00.103 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[77892]: audit 2023-10-09T08:51:59.641122+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:52:00.103 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[77892]: audit 2023-10-09T08:51:59.643060+0000 mon.a (mon.0) 567 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:52:00.103 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:00 smithi016 bash[77892]: audit 2023-10-09T08:51:59.651015+0000 mon.a (mon.0) 568 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:00.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:00.226 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:00.235 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:00 smithi094 bash[59677]: audit 2023-10-09T08:51:59.049388+0000 mon.a (mon.0) 564 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.235 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:00 smithi094 bash[59677]: audit 2023-10-09T08:51:59.060259+0000 mon.a (mon.0) 565 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.236 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:00 smithi094 bash[59677]: cluster 2023-10-09T08:51:59.352578+0000 mgr.y (mgr.25421) 430 : cluster 0 pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 18 MiB/s wr, 293 op/s; 24705/262611 objects degraded (9.407%); 4354/262611 objects misplaced (1.658%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:52:00.236 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:00 smithi094 bash[59677]: audit 2023-10-09T08:51:59.641122+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:52:00.236 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:00 smithi094 bash[59677]: audit 2023-10-09T08:51:59.643060+0000 mon.a (mon.0) 567 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:52:00.236 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:00 smithi094 bash[59677]: audit 2023-10-09T08:51:59.651015+0000 mon.a (mon.0) 568 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:00.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:00.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:00.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:00.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:00.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:00.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:00.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 90767 90751 75.6147 45.8125 0.0417067 0.0132181 2023-10-09T08:52:00.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:00.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:00.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:00.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:00.638 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:00.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:00.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:00.794 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:00.825 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:00.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:00.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:00.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:00.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:00.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:01.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:01.080 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:52:01.080 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:52:01.080 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T08:52:01.080 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T08:52:01.081 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T08:52:01.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T08:52:01.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:01.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:01.209 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:01.241 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:01.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:01.345 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:01.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:01.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:01.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:01.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:01 smithi016 bash[75179]: cluster 2023-10-09T08:52:01.353470+0000 mgr.y (mgr.25421) 431 : cluster 0 pgmap v374: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 24705/269709 objects degraded (9.160%); 4354/269709 objects misplaced (1.614%); 2.2 MiB/s, 36 objects/s recovering 2023-10-09T08:52:01.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 91507 91491 75.2283 46.25 0.0269643 0.0132861 2023-10-09T08:52:01.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:01.487 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:01 smithi094 bash[59677]: cluster 2023-10-09T08:52:01.353470+0000 mgr.y (mgr.25421) 431 : cluster 0 pgmap v374: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 24705/269709 objects degraded (9.160%); 4354/269709 objects misplaced (1.614%); 2.2 MiB/s, 36 objects/s recovering 2023-10-09T08:52:01.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:01.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:01.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:01.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:01.635 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:01.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:01.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:01.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:01.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:01.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:01.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:01.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:01 smithi016 bash[77892]: cluster 2023-10-09T08:52:01.353470+0000 mgr.y (mgr.25421) 431 : cluster 0 pgmap v374: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 24705/269709 objects degraded (9.160%); 4354/269709 objects misplaced (1.614%); 2.2 MiB/s, 36 objects/s recovering 2023-10-09T08:52:01.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:01.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:01.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:01.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:02.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:02.038 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:02.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:02.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:02.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:02.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:02.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:02.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:02.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:02.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:02.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:02.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:02.457 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 92252 92236 74.8559 46.5625 0.0273398 0.0133531 2023-10-09T08:52:02.553 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:02.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:02.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:02.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:02.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:02.688 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:02.690 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:02.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:02.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:02.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:02.836 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:02.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:02.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:02.972 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:03.005 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:03.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:03.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:03.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.275 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:03.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:03.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:03.408 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:03.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:03.458 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 93013 92997 74.506 47.5625 0.0215616 0.0134148 2023-10-09T08:52:03.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:03.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:03.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:03.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:03.683 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:03 smithi016 bash[77892]: cluster 2023-10-09T08:52:03.354242+0000 mgr.y (mgr.25421) 432 : cluster 0 pgmap v375: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 35 MiB/s wr, 560 op/s; 24705/273180 objects degraded (9.043%); 4224/273180 objects misplaced (1.546%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T08:52:03.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:03.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:03 smithi016 bash[75179]: cluster 2023-10-09T08:52:03.354242+0000 mgr.y (mgr.25421) 432 : cluster 0 pgmap v375: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 35 MiB/s wr, 560 op/s; 24705/273180 objects degraded (9.043%); 4224/273180 objects misplaced (1.546%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T08:52:03.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:03 smithi094 bash[59677]: cluster 2023-10-09T08:52:03.354242+0000 mgr.y (mgr.25421) 432 : cluster 0 pgmap v375: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 35 MiB/s wr, 560 op/s; 24705/273180 objects degraded (9.043%); 4224/273180 objects misplaced (1.546%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T08:52:03.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:03.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:03.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:03.918 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:03.953 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:03.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:04.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:04.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:04.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:04.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:04.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:04.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:04.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:04.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:04.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:04.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:04.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:04.431 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:04 smithi016 bash[75179]: cluster 2023-10-09T08:52:03.924232+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 24705/273180 objects degraded (9.043%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:04.431 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:04 smithi016 bash[75179]: audit 2023-10-09T08:52:04.216083+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:04.431 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:04 smithi016 bash[75179]: audit 2023-10-09T08:52:04.217667+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:04.458 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 93742 93726 74.1396 45.5625 0.0140952 0.0134814 2023-10-09T08:52:04.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:04.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:04.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:04.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:04.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:04.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:04 smithi094 bash[59677]: cluster 2023-10-09T08:52:03.924232+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 24705/273180 objects degraded (9.043%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:04.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:04 smithi094 bash[59677]: audit 2023-10-09T08:52:04.216083+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:04.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:04 smithi094 bash[59677]: audit 2023-10-09T08:52:04.217667+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:04.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:04.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:04.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:04.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:04.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:04.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:04.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:04.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:04 smithi016 bash[77892]: cluster 2023-10-09T08:52:03.924232+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 24705/273180 objects degraded (9.043%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:04.808 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:04 smithi016 bash[77892]: audit 2023-10-09T08:52:04.216083+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:52:04.808 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:04 smithi016 bash[77892]: audit 2023-10-09T08:52:04.217667+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:04.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:04.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:04.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:04.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:05.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:05.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:05.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:05.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:05.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:05.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:05.392 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:05.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:05.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:05.458 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:52:05.456172+0000 min lat: 0.00138239 max lat: 2.36633 avg lat: 0.0135535 2023-10-09T08:52:05.458 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:52:05.458 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 94417 94401 73.7401 42.1875 0.00676999 0.0135535 2023-10-09T08:52:05.530 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:05.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:05.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:05.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:05.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:05 smithi016 bash[75179]: cluster 2023-10-09T08:52:05.354965+0000 mgr.y (mgr.25421) 433 : cluster 0 pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 24705/280704 objects degraded (8.801%); 4224/280704 objects misplaced (1.505%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T08:52:05.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:05 smithi016 bash[77892]: cluster 2023-10-09T08:52:05.354965+0000 mgr.y (mgr.25421) 433 : cluster 0 pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 24705/280704 objects degraded (8.801%); 4224/280704 objects misplaced (1.505%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T08:52:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:05.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:05 smithi094 bash[59677]: cluster 2023-10-09T08:52:05.354965+0000 mgr.y (mgr.25421) 433 : cluster 0 pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 24705/280704 objects degraded (8.801%); 4224/280704 objects misplaced (1.505%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T08:52:05.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:05.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:05.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:05.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:05.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:05.957 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:52:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:52:05] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.33.4" 2023-10-09T08:52:05.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:05.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:06.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:06.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:06.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:06.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:06.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:06.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:06.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:06.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:06.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:06.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:06.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:06.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:06.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 95093 95077 73.3506 42.25 0.0183231 0.0136272 2023-10-09T08:52:06.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:06.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:06.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:06.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:06.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:06.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:06.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:06.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:06.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:06.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:06.861 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:06.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:06.964 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:06.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:07.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:07.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:07.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:07.273 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:07.378 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:07.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:07.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:07 smithi016 bash[75179]: cluster 2023-10-09T08:52:07.355552+0000 mgr.y (mgr.25421) 434 : cluster 0 pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 24705/283269 objects degraded (8.721%); 4089/283269 objects misplaced (1.444%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:52:07.459 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 95775 95759 72.9759 42.625 0.03858 0.0136972 2023-10-09T08:52:07.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:07.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:07.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:07.548 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:07.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:07.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:07.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:07.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:07 smithi094 bash[59677]: cluster 2023-10-09T08:52:07.355552+0000 mgr.y (mgr.25421) 434 : cluster 0 pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 24705/283269 objects degraded (8.721%); 4089/283269 objects misplaced (1.444%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:52:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:07 smithi016 bash[77892]: cluster 2023-10-09T08:52:07.355552+0000 mgr.y (mgr.25421) 434 : cluster 0 pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 24705/283269 objects degraded (8.721%); 4089/283269 objects misplaced (1.444%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:52:07.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:07.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:07.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:07.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:07.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:08.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:08.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:08.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:08.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:08.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:08.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:08.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:08.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:08.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:08.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:08.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:08.459 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 96489 96473 72.6343 44.625 0.0240933 0.0137609 2023-10-09T08:52:08.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:08.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:08.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:08.592 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:08.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:08.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:08.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:08.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:08.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:08.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:08.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:08.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:08.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:08.972 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:08.981 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:08 smithi016 bash[75179]: cluster 2023-10-09T08:52:08.925348+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 24705/283269 objects degraded (8.721%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:08.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:09.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:09.113 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:08 smithi094 bash[59677]: cluster 2023-10-09T08:52:08.925348+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 24705/283269 objects degraded (8.721%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:09.149 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:09.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:08 smithi016 bash[77892]: cluster 2023-10-09T08:52:08.925348+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 24705/283269 objects degraded (8.721%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:09.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:09.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:09.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:09.426 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:09.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 97189 97173 72.2904 43.75 0.0296522 0.0138257 2023-10-09T08:52:09.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:09.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:09.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:09.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:09.697 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:09.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:09.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:09.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:09.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:09.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:09.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:09.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:10.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:10.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:09 smithi016 bash[77892]: cluster 2023-10-09T08:52:09.356127+0000 mgr.y (mgr.25421) 435 : cluster 0 pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s; 24705/283950 objects degraded (8.700%); 4089/283950 objects misplaced (1.440%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:10.083 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:09 smithi016 bash[75179]: cluster 2023-10-09T08:52:09.356127+0000 mgr.y (mgr.25421) 435 : cluster 0 pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s; 24705/283950 objects degraded (8.700%); 4089/283950 objects misplaced (1.440%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:10.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:10.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:10.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:10.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:10.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:10.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:10.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:09 smithi094 bash[59677]: cluster 2023-10-09T08:52:09.356127+0000 mgr.y (mgr.25421) 435 : cluster 0 pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s; 24705/283950 objects degraded (8.700%); 4089/283950 objects misplaced (1.440%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:10.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:10.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:10.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:10.459 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 97862 97846 71.9347 42.0625 0.0312912 0.013895 2023-10-09T08:52:10.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:10.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:10.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:10.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:10.620 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:10.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:10.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:10.724 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:10.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:10.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:10.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:10.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:10.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:10.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:10.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:11.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:11.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:11.123 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:11.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:11.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:11.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:11.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:11.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:11.459 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 98551 98535 71.599 43.0625 0.0166784 0.0139597 2023-10-09T08:52:11.529 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:11.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:11.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:11.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:11 smithi016 bash[75179]: cluster 2023-10-09T08:52:11.356785+0000 mgr.y (mgr.25421) 436 : cluster 0 pgmap v379: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s; 24705/291015 objects degraded (8.489%); 4089/291015 objects misplaced (1.405%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:11.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:11 smithi016 bash[77892]: cluster 2023-10-09T08:52:11.356785+0000 mgr.y (mgr.25421) 436 : cluster 0 pgmap v379: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s; 24705/291015 objects degraded (8.489%); 4089/291015 objects misplaced (1.405%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:11.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:11.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:11 smithi094 bash[59677]: cluster 2023-10-09T08:52:11.356785+0000 mgr.y (mgr.25421) 436 : cluster 0 pgmap v379: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s; 24705/291015 objects degraded (8.489%); 4089/291015 objects misplaced (1.405%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:11.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:11.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:11.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:11.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:11.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:11.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:12.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:12.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:12.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:12.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:12.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:12.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:12.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:12.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:12.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:12.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:12.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:12.459 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 99234 99218 71.2666 42.6875 0.0201912 0.0140257 2023-10-09T08:52:12.469 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:12.504 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:12.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:12.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:12.650 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:12.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:12.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:12.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:12.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:12.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:12.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:12.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:12.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:13.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:13.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:13.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:13.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:13.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:13.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:13.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:13.284 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:13.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:13.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:13.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:13.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:13.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:13.459 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 99836 99820 70.8843 37.625 0.0165451 0.0140999 2023-10-09T08:52:13.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:13.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:13.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:13.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:13.691 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:13 smithi094 bash[59677]: cluster 2023-10-09T08:52:13.357517+0000 mgr.y (mgr.25421) 437 : cluster 0 pgmap v380: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 24705/294372 objects degraded (8.392%); 3953/294372 objects misplaced (1.343%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:52:13.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:13.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:13.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:13 smithi016 bash[77892]: cluster 2023-10-09T08:52:13.357517+0000 mgr.y (mgr.25421) 437 : cluster 0 pgmap v380: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 24705/294372 objects degraded (8.392%); 3953/294372 objects misplaced (1.343%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:52:13.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:13 smithi016 bash[75179]: cluster 2023-10-09T08:52:13.357517+0000 mgr.y (mgr.25421) 437 : cluster 0 pgmap v380: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 24705/294372 objects degraded (8.392%); 3953/294372 objects misplaced (1.343%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:52:13.833 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:13.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:13.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:13.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:13.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:14.009 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:14.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:14.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:14.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:14.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:14.276 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:52:14.276 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:52:14.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:14.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:14.459 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 100500 100484 70.5541 41.5 0.025841 0.014162 2023-10-09T08:52:14.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:14.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:14.617 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:14 smithi016 bash[75179]: cluster 2023-10-09T08:52:13.926406+0000 mon.a (mon.0) 573 : cluster 3 Health check update: Degraded data redundancy: 24705/294372 objects degraded (8.392%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:14.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:14 smithi016 bash[77892]: cluster 2023-10-09T08:52:13.926406+0000 mon.a (mon.0) 573 : cluster 3 Health check update: Degraded data redundancy: 24705/294372 objects degraded (8.392%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:14.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:14.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:14.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:14.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:14 smithi094 bash[59677]: cluster 2023-10-09T08:52:13.926406+0000 mon.a (mon.0) 573 : cluster 3 Health check update: Degraded data redundancy: 24705/294372 objects degraded (8.392%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:14.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:14.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:14.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:14.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:14.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:14.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:15.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:15.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:15.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:15.191 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:15.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:15.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:15.335 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:15.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:15.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:15 smithi094 bash[59677]: cluster 2023-10-09T08:52:15.358638+0000 mgr.y (mgr.25421) 438 : cluster 0 pgmap v381: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 24705/301164 objects degraded (8.203%); 3953/301164 objects misplaced (1.313%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:15.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:15.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 3 101124 101121 70.2125 39.8125 0.0182501 0.0142365 2023-10-09T08:52:15.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:15.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:15.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:15.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:15.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:15.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:15 smithi016 bash[75179]: cluster 2023-10-09T08:52:15.358638+0000 mgr.y (mgr.25421) 438 : cluster 0 pgmap v381: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 24705/301164 objects degraded (8.203%); 3953/301164 objects misplaced (1.313%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:15.711 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:15 smithi016 bash[77892]: cluster 2023-10-09T08:52:15.358638+0000 mgr.y (mgr.25421) 438 : cluster 0 pgmap v381: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 24705/301164 objects degraded (8.203%); 3953/301164 objects misplaced (1.313%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:15.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:15.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:15.832 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:52:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:52:15] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.33.4" 2023-10-09T08:52:15.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:15.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:15.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:15.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:15.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:15.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:16.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:16.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:16.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:16.211 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:16.240 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:16.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:16.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:16.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:16.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:16.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:16.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0259 2023-10-09T08:52:16.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 101124 2023-10-09T08:52:16.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-10-09T08:52:16.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-10-09T08:52:16.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 70.2048 2023-10-09T08:52:16.460 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 34.6657 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 172.938 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 0 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1123 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 554.661 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2767 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 0 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0142371 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0419387 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 2.36633 2023-10-09T08:52:16.461 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00138239 2023-10-09T08:52:16.467 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T08:52:16.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:16.507 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:16.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:16.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:16.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:16.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:16.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:16.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:16.786 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:16.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:16.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:16.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:16.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:16.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:17.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:17.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:17.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:17.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:17.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:17.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:17.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:17.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:17.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:17.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:17.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:17.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:17.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:17.697 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:17 smithi016 bash[75179]: cluster 2023-10-09T08:52:17.359371+0000 mgr.y (mgr.25421) 439 : cluster 0 pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 621 op/s; 24705/302433 objects degraded (8.169%); 3769/302433 objects misplaced (1.246%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T08:52:17.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:17 smithi016 bash[77892]: cluster 2023-10-09T08:52:17.359371+0000 mgr.y (mgr.25421) 439 : cluster 0 pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 621 op/s; 24705/302433 objects degraded (8.169%); 3769/302433 objects misplaced (1.246%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T08:52:17.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:17.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:17 smithi094 bash[59677]: cluster 2023-10-09T08:52:17.359371+0000 mgr.y (mgr.25421) 439 : cluster 0 pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 621 op/s; 24705/302433 objects degraded (8.169%); 3769/302433 objects misplaced (1.246%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T08:52:17.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:17.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:17.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:17.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T08:52:17.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:17.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:17.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:18.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:18.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:18.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:18.226 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:18.255 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:52:18.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:18.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:18.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:18.284 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:52:18.330 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:52:18.356 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:52:18.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:18.394 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:52:18.404 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:18.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:18.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:18.436 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:52:18.479 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:52:18.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:18 smithi094 bash[59677]: audit 2023-10-09T08:52:18.215366+0000 mon.a (mon.0) 574 : audit 0 from='client.? 172.21.15.16:0/1380268700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:52:18.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:18.525 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:52:18.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:18.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:18.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:18.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:18.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:18.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:18.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:18 smithi016 bash[75179]: audit 2023-10-09T08:52:18.215366+0000 mon.a (mon.0) 574 : audit 0 from='client.? 172.21.15.16:0/1380268700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:52:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:18 smithi016 bash[77892]: audit 2023-10-09T08:52:18.215366+0000 mon.a (mon.0) 574 : audit 0 from='client.? 172.21.15.16:0/1380268700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:52:18.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:18.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:18.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:18.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:18.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:18.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:18.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:19.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:19.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:19.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:19.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:19.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:19.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:19.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:19.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:19.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:19.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:19.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:19.453 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:19 smithi094 bash[59677]: cluster 2023-10-09T08:52:18.927388+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 24705/302433 objects degraded (8.169%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:19.453 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:19 smithi094 bash[59677]: audit 2023-10-09T08:52:19.100499+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.16:0/4238213008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:52:19.453 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:19 smithi094 bash[59677]: audit 2023-10-09T08:52:19.102239+0000 mon.a (mon.0) 576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:52:19.453 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:19 smithi094 bash[59677]: audit 2023-10-09T08:52:19.211779+0000 mon.a (mon.0) 577 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:19.453 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:19 smithi094 bash[59677]: cluster 2023-10-09T08:52:19.360019+0000 mgr.y (mgr.25421) 440 : cluster 0 pgmap v383: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 564 op/s; 24705/302682 objects degraded (8.162%); 3769/302682 objects misplaced (1.245%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:19.468 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:19.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:19.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:19.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:19.608 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:19.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:19.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[75179]: cluster 2023-10-09T08:52:18.927388+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 24705/302433 objects degraded (8.169%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:19.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[75179]: audit 2023-10-09T08:52:19.100499+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.16:0/4238213008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:52:19.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[75179]: audit 2023-10-09T08:52:19.102239+0000 mon.a (mon.0) 576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:52:19.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[75179]: audit 2023-10-09T08:52:19.211779+0000 mon.a (mon.0) 577 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:19.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[75179]: cluster 2023-10-09T08:52:19.360019+0000 mgr.y (mgr.25421) 440 : cluster 0 pgmap v383: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 564 op/s; 24705/302682 objects degraded (8.162%); 3769/302682 objects misplaced (1.245%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:19.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[77892]: cluster 2023-10-09T08:52:18.927388+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 24705/302433 objects degraded (8.169%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:19.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[77892]: audit 2023-10-09T08:52:19.100499+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.16:0/4238213008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:52:19.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[77892]: audit 2023-10-09T08:52:19.102239+0000 mon.a (mon.0) 576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:52:19.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[77892]: audit 2023-10-09T08:52:19.211779+0000 mon.a (mon.0) 577 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:19.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:19 smithi016 bash[77892]: cluster 2023-10-09T08:52:19.360019+0000 mgr.y (mgr.25421) 440 : cluster 0 pgmap v383: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 564 op/s; 24705/302682 objects degraded (8.162%); 3769/302682 objects misplaced (1.245%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:19.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:19.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:19.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:19.842 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:19.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:19.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:19.954 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T08:52:19.973 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:20.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:20.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:20.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:20.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:20.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:20.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:20.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:20.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:20.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:20.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:20.537 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:20.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:20.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:20 smithi016 bash[75179]: cluster 2023-10-09T08:52:19.932789+0000 mon.a (mon.0) 578 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:52:20.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:20 smithi016 bash[75179]: audit 2023-10-09T08:52:19.944937+0000 mon.a (mon.0) 579 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:52:20.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:20 smithi016 bash[75179]: cluster 2023-10-09T08:52:19.945055+0000 mon.a (mon.0) 580 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T08:52:20.628 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:20 smithi016 bash[77892]: cluster 2023-10-09T08:52:19.932789+0000 mon.a (mon.0) 578 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:52:20.628 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:20 smithi016 bash[77892]: audit 2023-10-09T08:52:19.944937+0000 mon.a (mon.0) 579 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:52:20.628 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:20 smithi016 bash[77892]: cluster 2023-10-09T08:52:19.945055+0000 mon.a (mon.0) 580 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T08:52:20.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:20.682 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:20 smithi094 bash[59677]: cluster 2023-10-09T08:52:19.932789+0000 mon.a (mon.0) 578 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:52:20.682 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:20 smithi094 bash[59677]: audit 2023-10-09T08:52:19.944937+0000 mon.a (mon.0) 579 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:52:20.682 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:20 smithi094 bash[59677]: cluster 2023-10-09T08:52:19.945055+0000 mon.a (mon.0) 580 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T08:52:20.712 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:20.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:20.851 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:20.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:20.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:20.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:21.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:21.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:21.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:21.224 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:21.255 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:21.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:21.362 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:21.401 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:21.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:21 smithi094 bash[59677]: cluster 2023-10-09T08:52:19.939611+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:21 smithi094 bash[59677]: cluster 2023-10-09T08:52:19.941350+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:21 smithi094 bash[59677]: cluster 2023-10-09T08:52:19.948773+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:21 smithi094 bash[59677]: cluster 2023-10-09T08:52:21.360733+0000 mgr.y (mgr.25421) 441 : cluster 0 pgmap v385: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 612 op/s; 24705/297507 objects degraded (8.304%); 3769/297507 objects misplaced (1.267%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:21.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:21.533 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:21.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:21.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:21.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:21.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:21.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:21.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[75179]: cluster 2023-10-09T08:52:19.939611+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[75179]: cluster 2023-10-09T08:52:19.941350+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[75179]: cluster 2023-10-09T08:52:19.948773+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[75179]: cluster 2023-10-09T08:52:21.360733+0000 mgr.y (mgr.25421) 441 : cluster 0 pgmap v385: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 612 op/s; 24705/297507 objects degraded (8.304%); 3769/297507 objects misplaced (1.267%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:21.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[77892]: cluster 2023-10-09T08:52:19.939611+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[77892]: cluster 2023-10-09T08:52:19.941350+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[77892]: cluster 2023-10-09T08:52:19.948773+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T08:52:21.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:21 smithi016 bash[77892]: cluster 2023-10-09T08:52:21.360733+0000 mgr.y (mgr.25421) 441 : cluster 0 pgmap v385: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 612 op/s; 24705/297507 objects degraded (8.304%); 3769/297507 objects misplaced (1.267%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:21.777 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:21.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:21.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:21.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:21.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:21.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:22.050 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:22.081 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:22.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:22.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:22.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:22.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:22.227 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:52:22.228 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T08:52:22.228 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:52:22.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:22.348 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:22.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:22.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:22.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:22.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:22.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:22.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:22.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:22.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:22.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:22.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:22.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:22.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:22.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:52:22.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:22.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:22.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:22.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:23.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:23.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:23.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:23.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:23.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:23.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:23.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:23.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:23.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:23.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:23.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:23.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:23.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:23.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:23.594 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:23 smithi094 bash[59677]: cluster 2023-10-09T08:52:23.361338+0000 mgr.y (mgr.25421) 442 : cluster 0 pgmap v386: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 606 op/s; 24705/294330 objects degraded (8.394%); 3636/294330 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:23.608 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:23.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:23.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:23.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:23.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:23 smithi016 bash[75179]: cluster 2023-10-09T08:52:23.361338+0000 mgr.y (mgr.25421) 442 : cluster 0 pgmap v386: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 606 op/s; 24705/294330 objects degraded (8.394%); 3636/294330 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:23 smithi016 bash[77892]: cluster 2023-10-09T08:52:23.361338+0000 mgr.y (mgr.25421) 442 : cluster 0 pgmap v386: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 606 op/s; 24705/294330 objects degraded (8.394%); 3636/294330 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:23.845 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:23.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:23.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:23.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:23.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:24.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T08:52:24.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:24.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:24.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:24.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:24.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:24.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:24.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:24.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:24.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:24.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:24.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:24.520 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:24.544 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:24 smithi016 bash[75179]: cluster 2023-10-09T08:52:23.928513+0000 mon.a (mon.0) 581 : cluster 3 Health check update: Degraded data redundancy: 24705/294330 objects degraded (8.394%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:24.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:24 smithi016 bash[77892]: cluster 2023-10-09T08:52:23.928513+0000 mon.a (mon.0) 581 : cluster 3 Health check update: Degraded data redundancy: 24705/294330 objects degraded (8.394%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:24.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:24.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:24.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:24.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:24.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:24 smithi094 bash[59677]: cluster 2023-10-09T08:52:23.928513+0000 mon.a (mon.0) 581 : cluster 3 Health check update: Degraded data redundancy: 24705/294330 objects degraded (8.394%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:24.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:24.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:24.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:24.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:24.980 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:24.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:24.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:25.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:25.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:25.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:25.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:25.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:25.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:25.361 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:25.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:25.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:25.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:25.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:25 smithi016 bash[75179]: cluster 2023-10-09T08:52:25.362102+0000 mgr.y (mgr.25421) 443 : cluster 0 pgmap v387: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 609 op/s; 24705/287430 objects degraded (8.595%); 3636/287430 objects misplaced (1.265%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:25.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:25 smithi016 bash[77892]: cluster 2023-10-09T08:52:25.362102+0000 mgr.y (mgr.25421) 443 : cluster 0 pgmap v387: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 609 op/s; 24705/287430 objects degraded (8.595%); 3636/287430 objects misplaced (1.265%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:25.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:25.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:25.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:25.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:25.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:25.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:25.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:25 smithi094 bash[59677]: cluster 2023-10-09T08:52:25.362102+0000 mgr.y (mgr.25421) 443 : cluster 0 pgmap v387: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 609 op/s; 24705/287430 objects degraded (8.595%); 3636/287430 objects misplaced (1.265%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T08:52:25.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:25.777 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:52:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:52:25] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.33.4" 2023-10-09T08:52:25.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:25.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:25.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:25.929 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:25.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:26.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:26.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:26.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:26.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:26.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:26.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:26.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:26.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:26.565 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:26.589 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:26.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:26.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:26.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:26.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:26.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:26.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:26.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:26.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:26.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:26.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:26.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:27.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:27.134 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:27.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:27.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:27.242 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:27.270 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:27.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:27.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:27.411 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:27.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:27.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:27.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:27.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:27 smithi094 bash[59677]: cluster 2023-10-09T08:52:27.362646+0000 mgr.y (mgr.25421) 444 : cluster 0 pgmap v388: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 635 op/s; 24705/284751 objects degraded (8.676%); 3516/284751 objects misplaced (1.235%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:52:27.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:27.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:27.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:27.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:27.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:27.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:27.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:27 smithi016 bash[77892]: cluster 2023-10-09T08:52:27.362646+0000 mgr.y (mgr.25421) 444 : cluster 0 pgmap v388: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 635 op/s; 24705/284751 objects degraded (8.676%); 3516/284751 objects misplaced (1.235%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:52:27.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:27 smithi016 bash[75179]: cluster 2023-10-09T08:52:27.362646+0000 mgr.y (mgr.25421) 444 : cluster 0 pgmap v388: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 635 op/s; 24705/284751 objects degraded (8.676%); 3516/284751 objects misplaced (1.235%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:52:27.772 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:27.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:27.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:27.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:27.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:27.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:27.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:28.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:28.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:28.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:28.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:28.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:28.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:28.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:28.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:28.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:28.531 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:52:28.531 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:52:28.531 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T08:52:28.531 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.756562 2023-10-09T08:52:28.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.7565623627591337 2023-10-09T08:52:28.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:28.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:28.598 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:28.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:28.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:28.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:28.819 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:28.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:28.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:28.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:29.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:29.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:29.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:29.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:29.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:29.195 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:28 smithi094 bash[59677]: cluster 2023-10-09T08:52:28.929577+0000 mon.a (mon.0) 582 : cluster 3 Health check update: Degraded data redundancy: 24705/284751 objects degraded (8.676%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:29.226 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:29.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:29.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:28 smithi016 bash[75179]: cluster 2023-10-09T08:52:28.929577+0000 mon.a (mon.0) 582 : cluster 3 Health check update: Degraded data redundancy: 24705/284751 objects degraded (8.676%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:29.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:28 smithi016 bash[77892]: cluster 2023-10-09T08:52:28.929577+0000 mon.a (mon.0) 582 : cluster 3 Health check update: Degraded data redundancy: 24705/284751 objects degraded (8.676%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:29.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:29.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:29.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:29.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:29.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:29.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:29.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:29.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:29.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:29.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:29.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:29.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:29.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:29.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:29.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:29.865 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:29.893 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:29.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:29.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:30.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:30.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:30.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:30.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:30.147 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:30 smithi094 bash[59677]: cluster 2023-10-09T08:52:29.363129+0000 mgr.y (mgr.25421) 445 : cluster 0 pgmap v389: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 640 KiB/s wr, 640 op/s; 24705/284073 objects degraded (8.697%); 3516/284073 objects misplaced (1.238%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:52:30.174 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:30.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:30.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:30.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:30 smithi016 bash[77892]: cluster 2023-10-09T08:52:29.363129+0000 mgr.y (mgr.25421) 445 : cluster 0 pgmap v389: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 640 KiB/s wr, 640 op/s; 24705/284073 objects degraded (8.697%); 3516/284073 objects misplaced (1.238%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:52:30.291 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:29 smithi016 bash[75179]: cluster 2023-10-09T08:52:29.363129+0000 mgr.y (mgr.25421) 445 : cluster 0 pgmap v389: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 640 KiB/s wr, 640 op/s; 24705/284073 objects degraded (8.697%); 3516/284073 objects misplaced (1.238%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:52:30.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:30.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:30.425 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:30.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:30.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:30.568 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:110694 6.8 e110/95 6:1bfd03b2:::benchmark_data_smithi016_105291_object9567:head v 110'7125, mlcod=110'7122)", 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:52:30.578891+0000", 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0034487559999999999, 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0034916790000000001, 2023-10-09T08:52:30.585 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578891+0000", 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578891+0000", 2023-10-09T08:52:30.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578890+0000", 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578895+0000", 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0259999999999998e-06 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:52:30.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578906+0000", 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0404999999999999e-05 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578911+0000", 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4829999999999999e-06 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:52:30.588 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578952+0000", 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0815000000000001e-05 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:52:30.578967+0000", 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4664999999999999e-05 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-10-09T08:52:30.589 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 1 2023-10-09T08:52:30.590 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:52:30.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:30.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:30.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:30.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:30.722 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:30.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:30.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:30.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:30.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:30.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:30.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:31.001 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:31.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:31.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:31.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:31.232 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:31.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:31.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:31.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:31 smithi094 bash[59677]: cluster 2023-10-09T08:52:31.363878+0000 mgr.y (mgr.25421) 446 : cluster 0 pgmap v390: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 B/s rd, 560 KiB/s wr, 758 op/s; 24705/277284 objects degraded (8.910%); 3516/277284 objects misplaced (1.268%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:31.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:31.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:31.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:31.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:31.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:31.734 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:31 smithi016 bash[75179]: cluster 2023-10-09T08:52:31.363878+0000 mgr.y (mgr.25421) 446 : cluster 0 pgmap v390: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 B/s rd, 560 KiB/s wr, 758 op/s; 24705/277284 objects degraded (8.910%); 3516/277284 objects misplaced (1.268%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:31.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:31 smithi016 bash[77892]: cluster 2023-10-09T08:52:31.363878+0000 mgr.y (mgr.25421) 446 : cluster 0 pgmap v390: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 B/s rd, 560 KiB/s wr, 758 op/s; 24705/277284 objects degraded (8.910%); 3516/277284 objects misplaced (1.268%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:31.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:31.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:31.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:31.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:31.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:32.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:32.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:32.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:32.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:32.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:32.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:32.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:32.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:32.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:32.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:32.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:32.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:32.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:32.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:32.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:32.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:32.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:32.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:32.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:32.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:32.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:32.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:33.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:33.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:33.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T08:52:33.133 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:33.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:33.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:33.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:33.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:33.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:33.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:33.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:33.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:33.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:33.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:33.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:33 smithi016 bash[75179]: cluster 2023-10-09T08:52:33.364570+0000 mgr.y (mgr.25421) 447 : cluster 0 pgmap v391: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 24705/274314 objects degraded (9.006%); 3378/274314 objects misplaced (1.231%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T08:52:33.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:33 smithi016 bash[77892]: cluster 2023-10-09T08:52:33.364570+0000 mgr.y (mgr.25421) 447 : cluster 0 pgmap v391: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 24705/274314 objects degraded (9.006%); 3378/274314 objects misplaced (1.231%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T08:52:33.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:33 smithi094 bash[59677]: cluster 2023-10-09T08:52:33.364570+0000 mgr.y (mgr.25421) 447 : cluster 0 pgmap v391: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 24705/274314 objects degraded (9.006%); 3378/274314 objects misplaced (1.231%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T08:52:33.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:33.853 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:33.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:33.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:33.988 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:33.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:34.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:34.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:34.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:34.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:34.255 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:34.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:34.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:34.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:34.393 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:34.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:34.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:34.432 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:34 smithi016 bash[75179]: cluster 2023-10-09T08:52:33.930890+0000 mon.a (mon.0) 583 : cluster 3 Health check update: Degraded data redundancy: 24705/274314 objects degraded (9.006%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:34.432 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:34 smithi016 bash[75179]: audit 2023-10-09T08:52:34.209991+0000 mon.a (mon.0) 584 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:34 smithi094 bash[59677]: cluster 2023-10-09T08:52:33.930890+0000 mon.a (mon.0) 583 : cluster 3 Health check update: Degraded data redundancy: 24705/274314 objects degraded (9.006%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:34.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:34 smithi094 bash[59677]: audit 2023-10-09T08:52:34.209991+0000 mon.a (mon.0) 584 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:34.498 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:34.538 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:34.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:34.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:34.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:34 smithi016 bash[77892]: cluster 2023-10-09T08:52:33.930890+0000 mon.a (mon.0) 583 : cluster 3 Health check update: Degraded data redundancy: 24705/274314 objects degraded (9.006%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T08:52:34.653 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:34 smithi016 bash[77892]: audit 2023-10-09T08:52:34.209991+0000 mon.a (mon.0) 584 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:34.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:34.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:34.775 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:34.813 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:34.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:34.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:34.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:34.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:35.052 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:35.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:35.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:35.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:35.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:35.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:35.317 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:35.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:35.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:35.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:35.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:35.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:35 smithi016 bash[75179]: cluster 2023-10-09T08:52:35.365798+0000 mgr.y (mgr.25421) 448 : cluster 0 pgmap v392: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 24705/267777 objects degraded (9.226%); 3378/267777 objects misplaced (1.261%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:52:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:35 smithi016 bash[77892]: cluster 2023-10-09T08:52:35.365798+0000 mgr.y (mgr.25421) 448 : cluster 0 pgmap v392: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 24705/267777 objects degraded (9.226%); 3378/267777 objects misplaced (1.261%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:52:35.600 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:35.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:35 smithi094 bash[59677]: cluster 2023-10-09T08:52:35.365798+0000 mgr.y (mgr.25421) 448 : cluster 0 pgmap v392: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 24705/267777 objects degraded (9.226%); 3378/267777 objects misplaced (1.261%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T08:52:35.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:35.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:35.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:35.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:35.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:35.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:35.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:35.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:35.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:35.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:35.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:36.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:52:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:52:35] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.33.4" 2023-10-09T08:52:36.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:36.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:36.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:36.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:36.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:36.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:36.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:36.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:36.350 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:36.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:36.443 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:36.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:36.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:36.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:36.684 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:36.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:36.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:36.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:36.859 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:36.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:36.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:36.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:37.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:37.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:37.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:37.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:37.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:37.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:37.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:37.357 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:37.394 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:37.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:37.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:37 smithi094 bash[59677]: cluster 2023-10-09T08:52:37.366453+0000 mgr.y (mgr.25421) 449 : cluster 0 pgmap v393: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24705/265329 objects degraded (9.311%); 3240/265329 objects misplaced (1.221%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T08:52:37.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:37.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:37.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:37.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:37.650 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:37.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:37.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:37 smithi016 bash[75179]: cluster 2023-10-09T08:52:37.366453+0000 mgr.y (mgr.25421) 449 : cluster 0 pgmap v393: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24705/265329 objects degraded (9.311%); 3240/265329 objects misplaced (1.221%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T08:52:37.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:37 smithi016 bash[77892]: cluster 2023-10-09T08:52:37.366453+0000 mgr.y (mgr.25421) 449 : cluster 0 pgmap v393: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24705/265329 objects degraded (9.311%); 3240/265329 objects misplaced (1.221%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T08:52:37.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:37.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:37.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:37.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:37.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:37.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:37.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:37.956 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:38.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:38.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:38.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:38.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:38.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:38.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:38.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:38.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:38.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:38.541 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:38.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:38.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:38.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:38.707 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:38.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:38.842 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:38.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:38.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:38.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:38.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:38.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:38 smithi094 bash[59677]: cluster 2023-10-09T08:52:38.931895+0000 mon.a (mon.0) 585 : cluster 3 Health check update: Degraded data redundancy: 24705/265329 objects degraded (9.311%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:39.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:39.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:39.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:39.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:39.196 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:39.204 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:38 smithi016 bash[77892]: cluster 2023-10-09T08:52:38.931895+0000 mon.a (mon.0) 585 : cluster 3 Health check update: Degraded data redundancy: 24705/265329 objects degraded (9.311%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:39.204 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:38 smithi016 bash[75179]: cluster 2023-10-09T08:52:38.931895+0000 mon.a (mon.0) 585 : cluster 3 Health check update: Degraded data redundancy: 24705/265329 objects degraded (9.311%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:39.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:39.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:39.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:39.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:39.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:39.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:39.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:39.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:39.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:39.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:39.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:39.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:39.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:39.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:39.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:39.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:39.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:39.842 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:39.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:39.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:39.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:40.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:40.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:40.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:40.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:39 smithi094 bash[59677]: cluster 2023-10-09T08:52:39.367074+0000 mgr.y (mgr.25421) 450 : cluster 0 pgmap v394: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24705/264717 objects degraded (9.333%); 3240/264717 objects misplaced (1.224%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:40.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:39 smithi016 bash[75179]: cluster 2023-10-09T08:52:39.367074+0000 mgr.y (mgr.25421) 450 : cluster 0 pgmap v394: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24705/264717 objects degraded (9.333%); 3240/264717 objects misplaced (1.224%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:40.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:39 smithi016 bash[77892]: cluster 2023-10-09T08:52:39.367074+0000 mgr.y (mgr.25421) 450 : cluster 0 pgmap v394: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24705/264717 objects degraded (9.333%); 3240/264717 objects misplaced (1.224%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:40.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:40.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:40.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:40.388 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:52:40.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:40.450 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:52:40.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:40.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:40.474 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:52:40.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:52:40.553 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:52:40.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:40.589 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:52:40.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:40.625 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:52:40.669 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:52:40.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:40.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:40.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:40.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:40.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:40.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:40.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:40.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:40.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:41.075 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:40 smithi016 bash[75179]: audit 2023-10-09T08:52:40.340084+0000 mon.a (mon.0) 586 : audit 0 from='client.? 172.21.15.16:0/1917668311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:52:41.077 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:40 smithi016 bash[77892]: audit 2023-10-09T08:52:40.340084+0000 mon.a (mon.0) 586 : audit 0 from='client.? 172.21.15.16:0/1917668311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:52:41.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:41.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:41.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:41.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:41.254 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:41.263 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:40 smithi094 bash[59677]: audit 2023-10-09T08:52:40.340084+0000 mon.a (mon.0) 586 : audit 0 from='client.? 172.21.15.16:0/1917668311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:52:41.293 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:41.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:41.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:41.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:41.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:41.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:41.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:41.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:41.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:41.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:41.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:41.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:41.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:41.857 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:41.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:41.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.001 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:42.023 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T08:52:42.105 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.115 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:42 smithi094 bash[59677]: audit 2023-10-09T08:52:41.073336+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.16:0/764207922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:52:42.115 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:42 smithi094 bash[59677]: audit 2023-10-09T08:52:41.073846+0000 mon.a (mon.0) 587 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:52:42.115 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:42 smithi094 bash[59677]: cluster 2023-10-09T08:52:41.368105+0000 mgr.y (mgr.25421) 451 : cluster 0 pgmap v395: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 24705/257787 objects degraded (9.583%); 3240/257787 objects misplaced (1.257%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:42.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:42.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:42 smithi016 bash[77892]: audit 2023-10-09T08:52:41.073336+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.16:0/764207922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:52:42.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:42 smithi016 bash[77892]: audit 2023-10-09T08:52:41.073846+0000 mon.a (mon.0) 587 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:52:42.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:42 smithi016 bash[77892]: cluster 2023-10-09T08:52:41.368105+0000 mgr.y (mgr.25421) 451 : cluster 0 pgmap v395: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 24705/257787 objects degraded (9.583%); 3240/257787 objects misplaced (1.257%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:42.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:42 smithi016 bash[75179]: audit 2023-10-09T08:52:41.073336+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.16:0/764207922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:52:42.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:42 smithi016 bash[75179]: audit 2023-10-09T08:52:41.073846+0000 mon.a (mon.0) 587 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:52:42.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:42 smithi016 bash[75179]: cluster 2023-10-09T08:52:41.368105+0000 mgr.y (mgr.25421) 451 : cluster 0 pgmap v395: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 24705/257787 objects degraded (9.583%); 3240/257787 objects misplaced (1.257%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T08:52:42.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:42.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:42.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:42.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:42.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:42.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:42.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:42.640 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:42.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:42.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:42.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.822 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:42.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:42.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:42.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:42.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:43.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:43.076 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:43.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:43.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: cluster 2023-10-09T08:52:41.999020+0000 mon.a (mon.0) 588 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:52:43.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: audit 2023-10-09T08:52:42.014493+0000 mon.a (mon.0) 589 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: cluster 2023-10-09T08:52:42.014565+0000 mon.a (mon.0) 590 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: cluster 2023-10-09T08:52:42.019835+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: cluster 2023-10-09T08:52:42.022581+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: cluster 2023-10-09T08:52:42.023757+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: cluster 2023-10-09T08:52:42.024962+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[75179]: cluster 2023-10-09T08:52:42.028667+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: cluster 2023-10-09T08:52:41.999020+0000 mon.a (mon.0) 588 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: audit 2023-10-09T08:52:42.014493+0000 mon.a (mon.0) 589 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:52:43.082 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: cluster 2023-10-09T08:52:42.014565+0000 mon.a (mon.0) 590 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T08:52:43.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: cluster 2023-10-09T08:52:42.019835+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: cluster 2023-10-09T08:52:42.022581+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: cluster 2023-10-09T08:52:42.023757+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: cluster 2023-10-09T08:52:42.024962+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:43 smithi016 bash[77892]: cluster 2023-10-09T08:52:42.028667+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:43.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:43.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:43.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:43.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:43.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:43.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:43.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:43.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: cluster 2023-10-09T08:52:41.999020+0000 mon.a (mon.0) 588 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: audit 2023-10-09T08:52:42.014493+0000 mon.a (mon.0) 589 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: cluster 2023-10-09T08:52:42.014565+0000 mon.a (mon.0) 590 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: cluster 2023-10-09T08:52:42.019835+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: cluster 2023-10-09T08:52:42.022581+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: cluster 2023-10-09T08:52:42.023757+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: cluster 2023-10-09T08:52:42.024962+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:43 smithi094 bash[59677]: cluster 2023-10-09T08:52:42.028667+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T08:52:43.487 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:43.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:43.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:43.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:43.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:43.732 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:43.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:43.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:43.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:43.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:43.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:43.990 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:44.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:44.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:44.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:44.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:44.282 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:44 smithi094 bash[59677]: cluster 2023-10-09T08:52:43.368761+0000 mgr.y (mgr.25421) 452 : cluster 0 pgmap v397: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 24705/254847 objects degraded (9.694%); 3112/254847 objects misplaced (1.221%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:44.282 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:44 smithi094 bash[59677]: cluster 2023-10-09T08:52:43.932725+0000 mon.a (mon.0) 591 : cluster 3 Health check update: Degraded data redundancy: 24705/257787 objects degraded (9.583%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:44.306 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:44 smithi016 bash[77892]: cluster 2023-10-09T08:52:43.368761+0000 mgr.y (mgr.25421) 452 : cluster 0 pgmap v397: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 24705/254847 objects degraded (9.694%); 3112/254847 objects misplaced (1.221%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:44.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:44 smithi016 bash[77892]: cluster 2023-10-09T08:52:43.932725+0000 mon.a (mon.0) 591 : cluster 3 Health check update: Degraded data redundancy: 24705/257787 objects degraded (9.583%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:44.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:44 smithi016 bash[75179]: cluster 2023-10-09T08:52:43.368761+0000 mgr.y (mgr.25421) 452 : cluster 0 pgmap v397: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 24705/254847 objects degraded (9.694%); 3112/254847 objects misplaced (1.221%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:44.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:44 smithi016 bash[75179]: cluster 2023-10-09T08:52:43.932725+0000 mon.a (mon.0) 591 : cluster 3 Health check update: Degraded data redundancy: 24705/257787 objects degraded (9.583%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:44.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:44.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:44.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:44.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:44.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:44.684 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:44.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:44.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:44.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:44.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:44.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:44.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:44.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:44.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:45.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:45.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:45.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:45.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:45.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:45.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:45.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:45.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:45.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:45.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:45.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:45.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:45.431 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:45 smithi094 bash[59677]: cluster 2023-10-09T08:52:45.369738+0000 mgr.y (mgr.25421) 453 : cluster 0 pgmap v398: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 24705/248352 objects degraded (9.948%); 3112/248352 objects misplaced (1.253%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:45.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:45.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:45 smithi016 bash[75179]: cluster 2023-10-09T08:52:45.369738+0000 mgr.y (mgr.25421) 453 : cluster 0 pgmap v398: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 24705/248352 objects degraded (9.948%); 3112/248352 objects misplaced (1.253%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:45.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:45 smithi016 bash[77892]: cluster 2023-10-09T08:52:45.369738+0000 mgr.y (mgr.25421) 453 : cluster 0 pgmap v398: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 24705/248352 objects degraded (9.948%); 3112/248352 objects misplaced (1.253%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T08:52:45.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:45.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:45.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:45.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:45.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:45.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:45.793 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:45.826 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:45.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:46.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:46.226 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:52:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:52:45] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.33.4" 2023-10-09T08:52:46.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:46.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:46.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:46.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:52:46.466 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:46.493 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:46.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:46.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:46.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:46.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:46.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:46.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:46.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:46.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:46.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:46.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:46.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:47.011 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:47.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:47.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:47.174 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:47.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:47.311 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:47.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:47.414 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:47.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:47.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:47.552 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:47 smithi094 bash[59677]: cluster 2023-10-09T08:52:47.370308+0000 mgr.y (mgr.25421) 454 : cluster 0 pgmap v399: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 24705/245871 objects degraded (10.048%); 2960/245871 objects misplaced (1.204%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:52:47.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:47.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:47.587 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:52:47.587 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:52:47.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:47 smithi016 bash[75179]: cluster 2023-10-09T08:52:47.370308+0000 mgr.y (mgr.25421) 454 : cluster 0 pgmap v399: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 24705/245871 objects degraded (10.048%); 2960/245871 objects misplaced (1.204%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:52:47.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:47 smithi016 bash[77892]: cluster 2023-10-09T08:52:47.370308+0000 mgr.y (mgr.25421) 454 : cluster 0 pgmap v399: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 24705/245871 objects degraded (10.048%); 2960/245871 objects misplaced (1.204%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:52:47.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:47.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:47.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:47.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T08:52:47.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:47.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:47.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:47.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:47.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:47.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:48.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:48.108 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:48.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:48.209 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:48.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:48.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:48.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:48.455 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.2 primary-affinity to 0.756562 (8495822) 2023-10-09T08:52:48.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:48.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:48.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:48.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:48.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:48.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:48.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:48 smithi094 bash[59677]: audit 2023-10-09T08:52:48.116459+0000 mon.a (mon.0) 592 : audit 1 from='client.? 172.21.15.16:0/921789182' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7565623627591337}]: dispatch 2023-10-09T08:52:48.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:48 smithi016 bash[75179]: audit 2023-10-09T08:52:48.116459+0000 mon.a (mon.0) 592 : audit 1 from='client.? 172.21.15.16:0/921789182' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7565623627591337}]: dispatch 2023-10-09T08:52:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:48 smithi016 bash[77892]: audit 2023-10-09T08:52:48.116459+0000 mon.a (mon.0) 592 : audit 1 from='client.? 172.21.15.16:0/921789182' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7565623627591337}]: dispatch 2023-10-09T08:52:48.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.788 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:48.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:48.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:48.947 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:48.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:49.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T08:52:49.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:49.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:49.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:49.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:49.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:49.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:49.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:49.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:49.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:49.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:49.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: audit 2023-10-09T08:52:48.441124+0000 mon.a (mon.0) 593 : audit 1 from='client.? 172.21.15.16:0/921789182' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7565623627591337}]': finished 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:48.441275+0000 mon.a (mon.0) 594 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:48.446274+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:48.452283+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:48.452769+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:48.456586+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:48.458532+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:48.933595+0000 mon.a (mon.0) 595 : cluster 3 Health check update: Degraded data redundancy: 24705/245871 objects degraded (10.048%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:49.444 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: audit 2023-10-09T08:52:49.212363+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:49.445 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[75179]: cluster 2023-10-09T08:52:49.370986+0000 mgr.y (mgr.25421) 455 : cluster 0 pgmap v401: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 24705/245277 objects degraded (10.072%); 2960/245277 objects misplaced (1.207%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T08:52:49.445 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: audit 2023-10-09T08:52:48.441124+0000 mon.a (mon.0) 593 : audit 1 from='client.? 172.21.15.16:0/921789182' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7565623627591337}]': finished 2023-10-09T08:52:49.445 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:48.441275+0000 mon.a (mon.0) 594 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T08:52:49.445 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:48.446274+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.445 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:48.452283+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.445 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:48.452769+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.445 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:48.456586+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:48.458532+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:48.933595+0000 mon.a (mon.0) 595 : cluster 3 Health check update: Degraded data redundancy: 24705/245871 objects degraded (10.048%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:49.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: audit 2023-10-09T08:52:49.212363+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:49.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:49 smithi016 bash[77892]: cluster 2023-10-09T08:52:49.370986+0000 mgr.y (mgr.25421) 455 : cluster 0 pgmap v401: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 24705/245277 objects degraded (10.072%); 2960/245277 objects misplaced (1.207%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T08:52:49.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:49.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:49.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:49.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:49.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:49.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:49.716 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:49.725 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: audit 2023-10-09T08:52:48.441124+0000 mon.a (mon.0) 593 : audit 1 from='client.? 172.21.15.16:0/921789182' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7565623627591337}]': finished 2023-10-09T08:52:49.725 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:48.441275+0000 mon.a (mon.0) 594 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T08:52:49.725 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:48.446274+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.725 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:48.452283+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.725 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:48.452769+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:48.456586+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:48.458532+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T08:52:49.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:48.933595+0000 mon.a (mon.0) 595 : cluster 3 Health check update: Degraded data redundancy: 24705/245871 objects degraded (10.048%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:49.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: audit 2023-10-09T08:52:49.212363+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:52:49.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:49 smithi094 bash[59677]: cluster 2023-10-09T08:52:49.370986+0000 mgr.y (mgr.25421) 455 : cluster 0 pgmap v401: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 24705/245277 objects degraded (10.072%); 2960/245277 objects misplaced (1.207%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T08:52:49.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:49.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:49.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:49.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:49.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:49.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:49.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:49.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:50.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:50.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:50.149 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:50.253 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:50.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:50.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:50.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:50.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:50.565 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:50.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:50.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:50.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:50.812 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:50.842 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:50.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:50.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:50.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:51.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:51.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:51.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:51.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:51.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:51.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:51.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:51.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:51.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:51.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:51.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:51.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:51.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:51.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:51.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:51.669 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:51.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:51.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:51.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:51 smithi094 bash[59677]: cluster 2023-10-09T08:52:51.371867+0000 mgr.y (mgr.25421) 456 : cluster 0 pgmap v402: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24705/238971 objects degraded (10.338%); 2960/238971 objects misplaced (1.239%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T08:52:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:51 smithi016 bash[75179]: cluster 2023-10-09T08:52:51.371867+0000 mgr.y (mgr.25421) 456 : cluster 0 pgmap v402: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24705/238971 objects degraded (10.338%); 2960/238971 objects misplaced (1.239%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T08:52:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:51 smithi016 bash[77892]: cluster 2023-10-09T08:52:51.371867+0000 mgr.y (mgr.25421) 456 : cluster 0 pgmap v402: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24705/238971 objects degraded (10.338%); 2960/238971 objects misplaced (1.239%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T08:52:51.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:51.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:51.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:51.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:51.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:51.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:51.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:51.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:52.038 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:52.070 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:52.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:52.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:52.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:52.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:52.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:52.339 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:52.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:52.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:52.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:52.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:52.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:52.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:52.619 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:52.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:52.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:52.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:52.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:52.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:52.860 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:52.891 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:52.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:52.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:53.036 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:53.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:53.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:53.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:53.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:53.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:53.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:53.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:53.444 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:53 smithi016 bash[77892]: cluster 2023-10-09T08:52:53.372782+0000 mgr.y (mgr.25421) 457 : cluster 0 pgmap v403: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 24705/235971 objects degraded (10.470%); 2803/235971 objects misplaced (1.188%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:52:53.468 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:53.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:53.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:53.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:53.690 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:53 smithi094 bash[59677]: cluster 2023-10-09T08:52:53.372782+0000 mgr.y (mgr.25421) 457 : cluster 0 pgmap v403: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 24705/235971 objects degraded (10.470%); 2803/235971 objects misplaced (1.188%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:52:53.711 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:53.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:53 smithi016 bash[75179]: cluster 2023-10-09T08:52:53.372782+0000 mgr.y (mgr.25421) 457 : cluster 0 pgmap v403: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 24705/235971 objects degraded (10.470%); 2803/235971 objects misplaced (1.188%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:52:53.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:53.842 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:53.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:53.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:53.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:54.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:54.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:54.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:54.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:54.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:54.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:54.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:54.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:54.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:54.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:54.489 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:54.490 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:52:54.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:54.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:54 smithi094 bash[59677]: cluster 2023-10-09T08:52:53.935086+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 24705/235971 objects degraded (10.470%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:54.523 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:54.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:54.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:54.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:54 smithi016 bash[75179]: cluster 2023-10-09T08:52:53.935086+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 24705/235971 objects degraded (10.470%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:54.636 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:54 smithi016 bash[77892]: cluster 2023-10-09T08:52:53.935086+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 24705/235971 objects degraded (10.470%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:54.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:54.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:54.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:54.758 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:54.797 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:54.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:54.901 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:54.929 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:54.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:54.948 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:52:54.948 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:52:54.948 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T08:52:54.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:52:55.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:55.069 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:55.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:55.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:55.171 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:55.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:55.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:55.304 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:55.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:55.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:55.431 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:55.463 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:55.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:55.569 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:55.577 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:55 smithi094 bash[59677]: cluster 2023-10-09T08:52:55.373990+0000 mgr.y (mgr.25421) 458 : cluster 0 pgmap v404: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 24705/229503 objects degraded (10.765%); 2803/229503 objects misplaced (1.221%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:52:55.612 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:55.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:55.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:55.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:55 smithi016 bash[75179]: cluster 2023-10-09T08:52:55.373990+0000 mgr.y (mgr.25421) 458 : cluster 0 pgmap v404: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 24705/229503 objects degraded (10.765%); 2803/229503 objects misplaced (1.221%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:52:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:55 smithi016 bash[77892]: cluster 2023-10-09T08:52:55.373990+0000 mgr.y (mgr.25421) 458 : cluster 0 pgmap v404: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 24705/229503 objects degraded (10.765%); 2803/229503 objects misplaced (1.221%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:52:55.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:55.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:55.859 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:55.914 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:55.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:56.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:52:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:52:55] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.33.4" 2023-10-09T08:52:56.020 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:56.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:56.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:56.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:56.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:56.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:56.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:56.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:56.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:56.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:56.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:56.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:56.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:56.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:56.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:56.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:56.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:56.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:56.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:56.728 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:56.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:52:56.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:52:56.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:56.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:56.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:57.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:57.046 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:57.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:57.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:57.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:57.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:52:57.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:52:57.433 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:57.438 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:57 smithi016 bash[75179]: cluster 2023-10-09T08:52:57.374642+0000 mgr.y (mgr.25421) 459 : cluster 0 pgmap v405: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 24705/227244 objects degraded (10.872%); 2663/227244 objects misplaced (1.172%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T08:52:57.438 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:57 smithi016 bash[77892]: cluster 2023-10-09T08:52:57.374642+0000 mgr.y (mgr.25421) 459 : cluster 0 pgmap v405: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 24705/227244 objects degraded (10.872%); 2663/227244 objects misplaced (1.172%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T08:52:57.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:57.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:57.568 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:57.577 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:57 smithi094 bash[59677]: cluster 2023-10-09T08:52:57.374642+0000 mgr.y (mgr.25421) 459 : cluster 0 pgmap v405: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 24705/227244 objects degraded (10.872%); 2663/227244 objects misplaced (1.172%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T08:52:57.599 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:57.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:57.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:57.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:57.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:57.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:57.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:57.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:57.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:57.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:58.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:58.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:58.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:58.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:58.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:58.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:58.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:58.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:58.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:58.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:58.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:58.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:58.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:58.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:58.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:58.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:58.652 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:58.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:58.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:58.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:58.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:52:58.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:58.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:52:58.888 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:58.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:52:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:58.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:52:59.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:59.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:58 smithi016 bash[75179]: cluster 2023-10-09T08:52:58.936415+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 24705/227244 objects degraded (10.872%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:59.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:52:58 smithi016 bash[77892]: cluster 2023-10-09T08:52:58.936415+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 24705/227244 objects degraded (10.872%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:59.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:52:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:59.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:52:59.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:59.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:59.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:59.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:59.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:52:58 smithi094 bash[59677]: cluster 2023-10-09T08:52:58.936415+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 24705/227244 objects degraded (10.872%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:52:59.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:59.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:59.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:59.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:59.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:59.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:59.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:59.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:59.601 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:52:59.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:59.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:52:59.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:59.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:52:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:59.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:52:59.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:59.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:52:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:52:59.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:52:59.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:00.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:00.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:00.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:00.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:00.299 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:59 smithi016 bash[75179]: cluster 2023-10-09T08:52:59.375314+0000 mgr.y (mgr.25421) 460 : cluster 0 pgmap v406: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 24705/226641 objects degraded (10.900%); 2663/226641 objects misplaced (1.175%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:53:00.299 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:52:59 smithi016 bash[75179]: audit 2023-10-09T08:52:59.725760+0000 mon.a (mon.0) 599 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:00.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:00 smithi016 bash[77892]: cluster 2023-10-09T08:52:59.375314+0000 mgr.y (mgr.25421) 460 : cluster 0 pgmap v406: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 24705/226641 objects degraded (10.900%); 2663/226641 objects misplaced (1.175%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:53:00.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:00 smithi016 bash[77892]: audit 2023-10-09T08:52:59.725760+0000 mon.a (mon.0) 599 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:00.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:00.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:00.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:00.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:00 smithi094 bash[59677]: cluster 2023-10-09T08:52:59.375314+0000 mgr.y (mgr.25421) 460 : cluster 0 pgmap v406: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 24705/226641 objects degraded (10.900%); 2663/226641 objects misplaced (1.175%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:53:00.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:00 smithi094 bash[59677]: audit 2023-10-09T08:52:59.725760+0000 mon.a (mon.0) 599 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:00.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:00.552 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:00.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:00.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:00.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:00.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:00.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:00.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:00.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:00.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:00.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:00.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:01.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:01.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:01.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:01.224 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:01.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:01.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:01.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:01.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:01.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:01.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:01.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:01 smithi094 bash[59677]: cluster 2023-10-09T08:53:01.376137+0000 mgr.y (mgr.25421) 461 : cluster 0 pgmap v407: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 24705/220326 objects degraded (11.213%); 2663/220326 objects misplaced (1.209%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T08:53:01.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:01.538 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:01.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:01.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:01.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:01.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:01.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:01.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:01 smithi016 bash[75179]: cluster 2023-10-09T08:53:01.376137+0000 mgr.y (mgr.25421) 461 : cluster 0 pgmap v407: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 24705/220326 objects degraded (11.213%); 2663/220326 objects misplaced (1.209%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T08:53:01.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:01 smithi016 bash[77892]: cluster 2023-10-09T08:53:01.376137+0000 mgr.y (mgr.25421) 461 : cluster 0 pgmap v407: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 24705/220326 objects degraded (11.213%); 2663/220326 objects misplaced (1.209%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T08:53:01.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:01.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:01.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:01.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:01.921 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:01.953 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:01.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:02.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:02.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:02.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:02.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:02.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:02.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:02.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:02.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:02.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:02.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:02.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:02.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:02.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:02.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:02.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:02.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:02.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:02.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:02.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:02.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:02.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:02.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:02.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:02.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:02.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:02.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:03.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:03.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:03.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:03.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:03.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:03.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:03.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:03.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:03.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:03.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:03.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:03 smithi094 bash[59677]: cluster 2023-10-09T08:53:03.376843+0000 mgr.y (mgr.25421) 462 : cluster 0 pgmap v408: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24705/217539 objects degraded (11.357%); 2499/217539 objects misplaced (1.149%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T08:53:03.556 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:03.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:03.661 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:03.667 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:03.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:03.694 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:03 smithi016 bash[75179]: cluster 2023-10-09T08:53:03.376843+0000 mgr.y (mgr.25421) 462 : cluster 0 pgmap v408: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24705/217539 objects degraded (11.357%); 2499/217539 objects misplaced (1.149%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T08:53:03.695 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:03 smithi016 bash[77892]: cluster 2023-10-09T08:53:03.376843+0000 mgr.y (mgr.25421) 462 : cluster 0 pgmap v408: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24705/217539 objects degraded (11.357%); 2499/217539 objects misplaced (1.149%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T08:53:03.790 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:03.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:03.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:03.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:03.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:04.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:04.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:04.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:04.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:04.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:04.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:04.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:04.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:04.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:04.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:04.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:04.461 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:53:04.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:04.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:04.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:04.487 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:53:04.513 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:53:04.538 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:53:04.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:04.580 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:04 smithi016 bash[75179]: cluster 2023-10-09T08:53:03.937529+0000 mon.a (mon.0) 600 : cluster 3 Health check update: Degraded data redundancy: 24705/217539 objects degraded (11.357%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:04.580 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:04 smithi016 bash[75179]: audit 2023-10-09T08:53:04.210603+0000 mon.a (mon.0) 601 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:04.581 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:04 smithi016 bash[75179]: audit 2023-10-09T08:53:04.425411+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.16:0/3171722431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:04.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:04 smithi016 bash[77892]: cluster 2023-10-09T08:53:03.937529+0000 mon.a (mon.0) 600 : cluster 3 Health check update: Degraded data redundancy: 24705/217539 objects degraded (11.357%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:04.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:04 smithi016 bash[77892]: audit 2023-10-09T08:53:04.210603+0000 mon.a (mon.0) 601 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:04.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:04 smithi016 bash[77892]: audit 2023-10-09T08:53:04.425411+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.16:0/3171722431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:04.581 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:53:04.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:04.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:04.619 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:53:04.646 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:53:04.690 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:53:04.695 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:53:04.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:04.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:04 smithi094 bash[59677]: cluster 2023-10-09T08:53:03.937529+0000 mon.a (mon.0) 600 : cluster 3 Health check update: Degraded data redundancy: 24705/217539 objects degraded (11.357%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:04.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:04 smithi094 bash[59677]: audit 2023-10-09T08:53:04.210603+0000 mon.a (mon.0) 601 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:04.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:04 smithi094 bash[59677]: audit 2023-10-09T08:53:04.425411+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.16:0/3171722431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:04.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:04.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:04.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:04.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:04.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:04.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:04.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:05.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:05.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:05.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:05.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:05.152 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:05.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:05.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:05.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:05.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:05.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:05.406 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:05.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:05.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:05.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:05.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:05.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:05.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:05.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:05.718 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:05 smithi094 bash[59677]: cluster 2023-10-09T08:53:05.377563+0000 mgr.y (mgr.25421) 463 : cluster 0 pgmap v409: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 24705/212130 objects degraded (11.646%); 2499/212130 objects misplaced (1.178%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:53:05.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:05 smithi016 bash[77892]: cluster 2023-10-09T08:53:05.377563+0000 mgr.y (mgr.25421) 463 : cluster 0 pgmap v409: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 24705/212130 objects degraded (11.646%); 2499/212130 objects misplaced (1.178%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:53:05.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:05 smithi016 bash[75179]: cluster 2023-10-09T08:53:05.377563+0000 mgr.y (mgr.25421) 463 : cluster 0 pgmap v409: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 24705/212130 objects degraded (11.646%); 2499/212130 objects misplaced (1.178%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:53:05.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:05.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:05.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:05.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:05.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:05.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:06.038 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:06.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:06.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:06.050 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:53:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:53:05] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.33.4" 2023-10-09T08:53:06.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:06.179 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:06.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:06.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:06.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:06.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:06.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:06.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:06.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:06.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:06.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:06.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:06.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:06.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:06.709 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:06.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:06.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:06.822 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:06 smithi094 bash[59677]: audit 2023-10-09T08:53:05.819868+0000 mon.a (mon.0) 602 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:06.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:06.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:06.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:06.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:06.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:07.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:07.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:07.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:07.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:07.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[75179]: audit 2023-10-09T08:53:05.819868+0000 mon.a (mon.0) 602 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[75179]: audit 2023-10-09T08:53:05.827031+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[75179]: audit 2023-10-09T08:53:06.535184+0000 mon.a (mon.0) 604 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[75179]: audit 2023-10-09T08:53:06.542671+0000 mon.a (mon.0) 605 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[77892]: audit 2023-10-09T08:53:05.819868+0000 mon.a (mon.0) 602 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[77892]: audit 2023-10-09T08:53:05.827031+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[77892]: audit 2023-10-09T08:53:06.535184+0000 mon.a (mon.0) 604 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:06 smithi016 bash[77892]: audit 2023-10-09T08:53:06.542671+0000 mon.a (mon.0) 605 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:07.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:07.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:07.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:06 smithi094 bash[59677]: audit 2023-10-09T08:53:05.827031+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:06 smithi094 bash[59677]: audit 2023-10-09T08:53:06.535184+0000 mon.a (mon.0) 604 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:06 smithi094 bash[59677]: audit 2023-10-09T08:53:06.542671+0000 mon.a (mon.0) 605 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:07.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:07.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:07.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:07.395 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:07.431 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:07.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:07.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:07.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:07.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:07.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:07.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:07.693 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:07.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:07.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:07.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:07.818 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16765 6.f 6:f3423698:::benchmark_data_smithi016_105291_object16764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.642674+0000", 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 135.16684989199999, 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.145442559, 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.819 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16765 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642674+0000", 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642674+0000", 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642673+0000", 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642697+0000", 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4227999999999999e-05 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642708+0000", 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1432e-05 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642714+0000", 2023-10-09T08:53:07.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8459999999999996e-06 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642864+0000", 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014968100000000001 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642948+0000", 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3857999999999996e-05 2023-10-09T08:53:07.823 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.643038+0000", 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0731999999999998e-05 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.736365+0000", 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.093326699999999999 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.736410+0000", 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5077999999999998e-05 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.788011+0000", 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.051600421 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.788067+0000", 2023-10-09T08:53:07.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6033999999999999e-05 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.788117+0000", 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9891000000000001e-05 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16768 6.f 6:fd815f2c:::benchmark_data_smithi016_105291_object16767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.826 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.642721+0000", 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 135.166802491, 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.145548029, 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16768 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.827 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642721+0000", 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642721+0000", 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642721+0000", 2023-10-09T08:53:07.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642740+0000", 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8666000000000001e-05 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642745+0000", 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1760000000000001e-06 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642748+0000", 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3289999999999998e-06 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.643749+0000", 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001000707 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.643833+0000", 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4393000000000003e-05 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.643922+0000", 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9036999999999995e-05 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.736420+0000", 2023-10-09T08:53:07.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.092497446999999997 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.737101+0000", 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00068109400000000004 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.788181+0000", 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.051080223000000001 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.832 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.788231+0000", 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9427e-05 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.788269+0000", 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8915000000000003e-05 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.833 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:27340 6.f 6:f8f061c7:::benchmark_data_smithi016_105291_object27339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.366929+0000", 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 129.44259532500001, 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13832528799999999, 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.834 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27340 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366929+0000", 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366929+0000", 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366927+0000", 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366958+0000", 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0465999999999999e-05 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366971+0000", 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3067e-05 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366981+0000", 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8050000000000001e-06 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.367051+0000", 2023-10-09T08:53:07.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9832999999999994e-05 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.367171+0000", 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000120387 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.367286+0000", 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000114781 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.372742+0000", 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0054567169999999998 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.373477+0000", 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00073408700000000002 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505020+0000", 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.1315434 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505195+0000", 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017518000000000001 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505254+0000", 2023-10-09T08:53:07.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8727999999999997e-05 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:31831 6.f 6:fe19beb1:::benchmark_data_smithi016_105291_object31830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:01.142201+0000", 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 126.66732323399999, 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.118926513, 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.841 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 31831 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142201+0000", 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142201+0000", 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142199+0000", 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142223+0000", 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4057999999999999e-05 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142237+0000", 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3563e-05 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142245+0000", 2023-10-09T08:53:07.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3319999999999992e-06 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142484+0000", 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000239029 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142557+0000", 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2645999999999995e-05 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.142638+0000", 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0688999999999999e-05 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.143432+0000", 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00079399099999999997 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.147364+0000", 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039322319999999999 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.260875+0000", 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.113510973 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.261085+0000", 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000210027 2023-10-09T08:53:07.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:01.261127+0000", 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2425000000000001e-05 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33222 6.f 6:ff9a883f:::benchmark_data_smithi016_105291_object33221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.075544+0000", 2023-10-09T08:53:07.848 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.733979526, 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.166447241, 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33222 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075544+0000", 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075544+0000", 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075543+0000", 2023-10-09T08:53:07.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075573+0000", 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0372e-05 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075584+0000", 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1331e-05 2023-10-09T08:53:07.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075591+0000", 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2870000000000001e-06 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075652+0000", 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0951e-05 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.075820+0000", 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016879899999999999 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076026+0000", 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020591899999999999 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236187+0000", 2023-10-09T08:53:07.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16016065700000001 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236230+0000", 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2800999999999999e-05 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.241894+0000", 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0056639710000000003 2023-10-09T08:53:07.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.241949+0000", 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5470000000000003e-05 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.241992+0000", 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2364000000000003e-05 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.855 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33226 6.f 6:f609f93e:::benchmark_data_smithi016_105291_object33225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.076430+0000", 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.73309351, 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16564711500000001, 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.856 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33226 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076430+0000", 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076430+0000", 2023-10-09T08:53:07.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076429+0000", 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076454+0000", 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4858000000000002e-05 2023-10-09T08:53:07.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076464+0000", 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0192e-05 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.076470+0000", 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0129999999999999e-06 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081117+0000", 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0046462630000000003 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081249+0000", 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000132037 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081381+0000", 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013209800000000001 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236272+0000", 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15489145400000001 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236320+0000", 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7599999999999998e-05 2023-10-09T08:53:07.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242002+0000", 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0056821500000000004 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242044+0000", 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2363000000000001e-05 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242078+0000", 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3277e-05 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.44690.0:33222 6.f e101/95 ondisk, result = 0)", 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.080203+0000", 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.72932072099999, 2023-10-09T08:53:07.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.155988399, 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080203+0000", 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.864 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080203+0000", 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080202+0000", 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080208+0000", 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6339999999999998e-06 2023-10-09T08:53:07.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080233+0000", 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5052999999999999e-05 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080242+0000", 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7879999999999995e-06 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.866 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236165+0000", 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15592344299999999 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236178+0000", 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2977e-05 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236192+0000", 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3379e-05 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:33221 6.9 e101/95 6:99c54d79:::benchmark_data_smithi016_105291_object33220:head v 101'2088, mlcod=101'2084)", 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.080327+0000", 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.729196425, 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15584847299999999, 2023-10-09T08:53:07.868 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080327+0000", 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080327+0000", 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.869 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080327+0000", 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080352+0000", 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5117e-05 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.870 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080360+0000", 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5769999999999996e-06 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.080365+0000", 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6559999999999997e-06 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081639+0000", 2023-10-09T08:53:07.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001273955 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081661+0000", 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1466000000000001e-05 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236140+0000", 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15447887799999999 2023-10-09T08:53:07.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236176+0000", 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6177999999999998e-05 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.44690.0:33222 6.f e101/95 ondisk, result = 0)", 2023-10-09T08:53:07.873 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.081277+0000", 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.72824680399999, 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.154955762, 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081277+0000", 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081277+0000", 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081276+0000", 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.875 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081283+0000", 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2099999999999998e-06 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081294+0000", 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1832e-05 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.081302+0000", 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5809999999999999e-06 2023-10-09T08:53:07.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236217+0000", 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15491479399999999 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236224+0000", 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4100000000000002e-06 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236233+0000", 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7099999999999996e-06 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:33230 6.2 e101/95 6:4fdcdb2a:::benchmark_data_smithi016_105291_object33229:head v 101'2105, mlcod=101'2103)", 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.083175+0000", 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.726348952, 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15293101200000001, 2023-10-09T08:53:07.878 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083175+0000", 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083175+0000", 2023-10-09T08:53:07.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083174+0000", 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083235+0000", 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1394000000000002e-05 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083248+0000", 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2578000000000001e-05 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083256+0000", 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1640000000000001e-06 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083300+0000", 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3611e-05 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083320+0000", 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0137e-05 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236049+0000", 2023-10-09T08:53:07.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15272914400000001 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236106+0000", 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6904000000000003e-05 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.44690.0:33226 6.f e101/95 ondisk, result = 0)", 2023-10-09T08:53:07.883 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.083273+0000", 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.726251382, 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15300248599999999, 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083273+0000", 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083273+0000", 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083272+0000", 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083276+0000", 2023-10-09T08:53:07.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7079999999999999e-06 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083283+0000", 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1300000000000003e-06 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083288+0000", 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.344e-06 2023-10-09T08:53:07.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236259+0000", 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15297111099999999 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236267+0000", 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1919999999999999e-06 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236275+0000", 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4460000000000001e-06 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:33233 6.e e101/95 6:78f96106:::benchmark_data_smithi016_105291_object33232:head v 101'2039, mlcod=101'2037)", 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.083293+0000", 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.72623042799999, 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.156148965, 2023-10-09T08:53:07.888 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083293+0000", 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083293+0000", 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083293+0000", 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083338+0000", 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4514000000000001e-05 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083347+0000", 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9649999999999997e-06 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.083353+0000", 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.032e-06 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.891 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236143+0000", 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15278999500000001 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236205+0000", 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2673000000000001e-05 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.239335+0000", 2023-10-09T08:53:07.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0031291069999999999 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.239442+0000", 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000107928 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:33237 6.0 e101/95 6:090cd5be:::benchmark_data_smithi016_105291_object33236:head v 101'2129, mlcod=101'2127)", 2023-10-09T08:53:07.893 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.085272+0000", 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.72425202399999, 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.154040752, 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.085272+0000", 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.085272+0000", 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.085271+0000", 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.085306+0000", 2023-10-09T08:53:07.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4891999999999997e-05 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236043+0000", 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.150737186 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236086+0000", 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3262e-05 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236245+0000", 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015834099999999999 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236270+0000", 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4859999999999999e-05 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.239291+0000", 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0030212329999999999 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.239313+0000", 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1841e-05 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.44690.0:33226 6.f e101/95 ondisk, result = 0)", 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.086912+0000", 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.722611855, 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14941180000000001, 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.086912+0000", 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.086912+0000", 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.086911+0000", 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.086918+0000", 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0539999999999997e-06 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236046+0000", 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.149127656 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236078+0000", 2023-10-09T08:53:07.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2027999999999999e-05 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236303+0000", 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000225362 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236310+0000", 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0029999999999997e-06 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236324+0000", 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3811999999999999e-05 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33246 6.f 6:fc00ab0f:::benchmark_data_smithi016_105291_object33245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.236228+0000", 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.573295507, 2023-10-09T08:53:07.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.11614495699999999, 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33246 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236228+0000", 2023-10-09T08:53:07.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236228+0000", 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236228+0000", 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236262+0000", 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4437000000000002e-05 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236270+0000", 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1790000000000008e-06 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236277+0000", 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9319999999999999e-06 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.238211+0000", 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001933911 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.238378+0000", 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000167178 2023-10-09T08:53:07.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.238543+0000", 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016509700000000001 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242090+0000", 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035467910000000001 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.245347+0000", 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.003256976 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.250731+0000", 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.005383451 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352316+0000", 2023-10-09T08:53:07.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.101585301 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352373+0000", 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7433999999999999e-05 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33252 6.f 6:f1ffcf64:::benchmark_data_smithi016_105291_object33251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.910 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.236305+0000", 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.573219329, 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.116232795, 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33252 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236305+0000", 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236305+0000", 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236304+0000", 2023-10-09T08:53:07.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236341+0000", 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7528999999999999e-05 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236351+0000", 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2099999999999999e-06 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.236357+0000", 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1020000000000002e-06 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.239850+0000", 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00349329 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.240008+0000", 2023-10-09T08:53:07.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015847499999999999 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.240464+0000", 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00045581700000000002 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242330+0000", 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001865954 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.245382+0000", 2023-10-09T08:53:07.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0030516749999999998 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352434+0000", 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10705235 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352496+0000", 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1838e-05 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352537+0000", 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1282000000000002e-05 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:33262 6.f 6:f380a246:::benchmark_data_smithi016_105291_object33261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.242297+0000", 2023-10-09T08:53:07.919 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.567227281, 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.110472943, 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 33262 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.920 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242297+0000", 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242297+0000", 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242295+0000", 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:53:07.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242324+0000", 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8615e-05 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242339+0000", 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5048000000000001e-05 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242348+0000", 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0550000000000005e-06 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242406+0000", 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8029000000000003e-05 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242547+0000", 2023-10-09T08:53:07.923 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014124100000000001 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.242694+0000", 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014702000000000001 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.246615+0000", 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039208300000000001 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.924 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.246633+0000", 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8009e-05 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352600+0000", 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10596718400000001 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.925 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352656+0000", 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6218000000000003e-05 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.352770+0000", 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000113139 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.926 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:33261 6.9 e101/95 6:9a5f0e1d:::benchmark_data_smithi016_105291_object33260:head v 101'2089, mlcod=101'2087)", 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.243728+0000", 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.565795885, 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.114164854, 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.243728+0000", 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.927 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.243728+0000", 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.243727+0000", 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.928 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.245450+0000", 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0017221929999999999 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.245479+0000", 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9467000000000001e-05 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.245488+0000", 2023-10-09T08:53:07.929 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8570000000000001e-06 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.246225+0000", 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00073711200000000005 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.246246+0000", 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1277999999999999e-05 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.930 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.357809+0000", 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.111563173 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.357893+0000", 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3443999999999996e-05 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.931 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:33264 6.8 e101/95 6:1156e47a:::benchmark_data_smithi016_105291_object33263:head v 101'2120, mlcod=101'2117)", 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.247836+0000", 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.561687507, 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.106529332, 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247836+0000", 2023-10-09T08:53:07.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247836+0000", 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247835+0000", 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247876+0000", 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0327999999999997e-05 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247890+0000", 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4738e-05 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247900+0000", 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6309999999999998e-06 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247960+0000", 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0420000000000001e-05 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.247982+0000", 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1212999999999998e-05 2023-10-09T08:53:07.935 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.354301+0000", 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10631937599999999 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.354366+0000", 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4764999999999994e-05 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.936 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:33269 6.9 e101/95 6:928db727:::benchmark_data_smithi016_105291_object33268:head v 101'2090, mlcod=101'2087)", 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:02.248376+0000", 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 125.56114749299999, 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.109557904, 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:07.937 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248376+0000", 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248376+0000", 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248376+0000", 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:07.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248437+0000", 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1043999999999994e-05 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248448+0000", 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1861e-05 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248455+0000", 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9260000000000003e-06 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248561+0000", 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000106066 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.248583+0000", 2023-10-09T08:53:07.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1270000000000001e-05 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.357899+0000", 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.109315934 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:02.357934+0000", 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5645000000000002e-05 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.941 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.942 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.942 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:07.942 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:07.942 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:53:07.942 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[77892]: audit 2023-10-09T08:53:07.114378+0000 mon.a (mon.0) 606 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:53:07.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[77892]: audit 2023-10-09T08:53:07.115673+0000 mon.a (mon.0) 607 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:53:07.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[77892]: audit 2023-10-09T08:53:07.124396+0000 mon.a (mon.0) 608 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[77892]: cluster 2023-10-09T08:53:07.378348+0000 mgr.y (mgr.25421) 464 : cluster 0 pgmap v410: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24705/209472 objects degraded (11.794%); 2353/209472 objects misplaced (1.123%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T08:53:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:07.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:07.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:07.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:07 smithi094 bash[59677]: audit 2023-10-09T08:53:07.114378+0000 mon.a (mon.0) 606 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:53:07.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:07 smithi094 bash[59677]: audit 2023-10-09T08:53:07.115673+0000 mon.a (mon.0) 607 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:53:07.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:07 smithi094 bash[59677]: audit 2023-10-09T08:53:07.124396+0000 mon.a (mon.0) 608 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:07.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:07 smithi094 bash[59677]: cluster 2023-10-09T08:53:07.378348+0000 mgr.y (mgr.25421) 464 : cluster 0 pgmap v410: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24705/209472 objects degraded (11.794%); 2353/209472 objects misplaced (1.123%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T08:53:08.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:08.079 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:08.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:08.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[75179]: audit 2023-10-09T08:53:07.114378+0000 mon.a (mon.0) 606 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:53:08.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[75179]: audit 2023-10-09T08:53:07.115673+0000 mon.a (mon.0) 607 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:53:08.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[75179]: audit 2023-10-09T08:53:07.124396+0000 mon.a (mon.0) 608 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:08.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:07 smithi016 bash[75179]: cluster 2023-10-09T08:53:07.378348+0000 mgr.y (mgr.25421) 464 : cluster 0 pgmap v410: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24705/209472 objects degraded (11.794%); 2353/209472 objects misplaced (1.123%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T08:53:08.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:08.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:08.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:08.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:08.325 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:08.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:08.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:08.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:08.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:08.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:08.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:08.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:08.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:08.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:08.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:08.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:08.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:08.876 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:08.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:08.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:08.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:09.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:09.037 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:09.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:09.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:09.145 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T08:53:09.173 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:09.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:09.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:09.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:09.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:09.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:09.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:09.422 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:09 smithi094 bash[59677]: audit 2023-10-09T08:53:08.309245+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.16:0/878573133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:09.422 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:09 smithi094 bash[59677]: audit 2023-10-09T08:53:08.310002+0000 mon.a (mon.0) 609 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:09.422 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:09 smithi094 bash[59677]: cluster 2023-10-09T08:53:08.940586+0000 mon.a (mon.0) 610 : cluster 3 Health check update: Degraded data redundancy: 24705/209472 objects degraded (11.794%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:09.452 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:09.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:09 smithi016 bash[75179]: audit 2023-10-09T08:53:08.309245+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.16:0/878573133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:09.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:09 smithi016 bash[75179]: audit 2023-10-09T08:53:08.310002+0000 mon.a (mon.0) 609 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:09.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:09 smithi016 bash[75179]: cluster 2023-10-09T08:53:08.940586+0000 mon.a (mon.0) 610 : cluster 3 Health check update: Degraded data redundancy: 24705/209472 objects degraded (11.794%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:09 smithi016 bash[77892]: audit 2023-10-09T08:53:08.309245+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.16:0/878573133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:09 smithi016 bash[77892]: audit 2023-10-09T08:53:08.310002+0000 mon.a (mon.0) 609 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:09 smithi016 bash[77892]: cluster 2023-10-09T08:53:08.940586+0000 mon.a (mon.0) 610 : cluster 3 Health check update: Degraded data redundancy: 24705/209472 objects degraded (11.794%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:09.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:09.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:09.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:09.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:09.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:53:09.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:09.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:09.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:09.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:09.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:09.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:09.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:09.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:10.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:10.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:10.132 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: cluster 2023-10-09T08:53:09.126431+0000 mon.a (mon.0) 611 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:53:10.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:10.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:10.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:10.278 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: cluster 2023-10-09T08:53:09.126431+0000 mon.a (mon.0) 611 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: cluster 2023-10-09T08:53:09.133902+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: cluster 2023-10-09T08:53:09.137133+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: audit 2023-10-09T08:53:09.137177+0000 mon.a (mon.0) 612 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: cluster 2023-10-09T08:53:09.137394+0000 mon.a (mon.0) 613 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: cluster 2023-10-09T08:53:09.143285+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: cluster 2023-10-09T08:53:09.147274+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[77892]: cluster 2023-10-09T08:53:09.378852+0000 mgr.y (mgr.25421) 465 : cluster 0 pgmap v412: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 24705/208992 objects degraded (11.821%); 2353/208992 objects misplaced (1.126%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: cluster 2023-10-09T08:53:09.126431+0000 mon.a (mon.0) 611 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:53:10.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: cluster 2023-10-09T08:53:09.133902+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: cluster 2023-10-09T08:53:09.137133+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: audit 2023-10-09T08:53:09.137177+0000 mon.a (mon.0) 612 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:53:10.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: cluster 2023-10-09T08:53:09.137394+0000 mon.a (mon.0) 613 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T08:53:10.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: cluster 2023-10-09T08:53:09.143285+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: cluster 2023-10-09T08:53:09.147274+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:10 smithi016 bash[75179]: cluster 2023-10-09T08:53:09.378852+0000 mgr.y (mgr.25421) 465 : cluster 0 pgmap v412: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 24705/208992 objects degraded (11.821%); 2353/208992 objects misplaced (1.126%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:10.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:10.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:10.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: cluster 2023-10-09T08:53:09.133902+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: cluster 2023-10-09T08:53:09.137133+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: audit 2023-10-09T08:53:09.137177+0000 mon.a (mon.0) 612 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:53:10.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: cluster 2023-10-09T08:53:09.137394+0000 mon.a (mon.0) 613 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T08:53:10.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: cluster 2023-10-09T08:53:09.143285+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: cluster 2023-10-09T08:53:09.147274+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T08:53:10.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:10 smithi094 bash[59677]: cluster 2023-10-09T08:53:09.378852+0000 mgr.y (mgr.25421) 465 : cluster 0 pgmap v412: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 24705/208992 objects degraded (11.821%); 2353/208992 objects misplaced (1.126%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:10.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:10.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:10.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:10.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T08:53:10.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:10.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:10.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:10.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:10.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:10.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:10.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:10.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:11.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:11.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:11.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:11.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:11.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:11.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:11.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:11.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:11.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:11.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:11.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:11.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:11.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:11.576 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:11 smithi016 bash[75179]: cluster 2023-10-09T08:53:11.379538+0000 mgr.y (mgr.25421) 466 : cluster 0 pgmap v413: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 24705/202905 objects degraded (12.176%); 2353/202905 objects misplaced (1.160%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:11.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:11 smithi016 bash[77892]: cluster 2023-10-09T08:53:11.379538+0000 mgr.y (mgr.25421) 466 : cluster 0 pgmap v413: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 24705/202905 objects degraded (12.176%); 2353/202905 objects misplaced (1.160%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:11.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:11.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:11.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:11.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:11 smithi094 bash[59677]: cluster 2023-10-09T08:53:11.379538+0000 mgr.y (mgr.25421) 466 : cluster 0 pgmap v413: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 24705/202905 objects degraded (12.176%); 2353/202905 objects misplaced (1.160%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:11.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:11.794 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:11.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:11.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:11.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:11.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:11.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:12.027 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:12.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:12.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:12.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:12.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:12.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:12.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:12.302 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:12.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:12.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:12.445 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:12.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:12.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:12.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:12.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:12.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:12.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:12.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:12.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:12.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:12.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:12.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:12.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:12.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:12.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:13.009 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:13.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:13.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:13.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:13.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:13.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:13.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:13.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:13.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:13.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:13.685 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:53:13.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:13.700 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:53:13.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:13.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:13.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:13.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:13.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:13.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:13.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:14.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:14.106 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:14.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:14.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:14.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T08:53:14.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:14.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:14.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:14.367 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:14.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:14.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:14.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:14.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:14.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:14.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:14 smithi094 bash[59677]: cluster 2023-10-09T08:53:13.380392+0000 mgr.y (mgr.25421) 467 : cluster 0 pgmap v414: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 24705/200031 objects degraded (12.351%); 2210/200031 objects misplaced (1.105%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:14.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:14 smithi094 bash[59677]: audit 2023-10-09T08:53:13.666420+0000 mgr.y (mgr.25421) 468 : audit 0 from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:53:14.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:14 smithi094 bash[59677]: cluster 2023-10-09T08:53:13.941959+0000 mon.a (mon.0) 614 : cluster 3 Health check update: Degraded data redundancy: 24705/200031 objects degraded (12.351%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:14.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:14 smithi016 bash[77892]: cluster 2023-10-09T08:53:13.380392+0000 mgr.y (mgr.25421) 467 : cluster 0 pgmap v414: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 24705/200031 objects degraded (12.351%); 2210/200031 objects misplaced (1.105%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:14.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:14 smithi016 bash[77892]: audit 2023-10-09T08:53:13.666420+0000 mgr.y (mgr.25421) 468 : audit 0 from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:53:14.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:14 smithi016 bash[77892]: cluster 2023-10-09T08:53:13.941959+0000 mon.a (mon.0) 614 : cluster 3 Health check update: Degraded data redundancy: 24705/200031 objects degraded (12.351%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:14.634 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:14.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:14.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:14.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:14.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:14.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:14.877 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:14 smithi016 bash[75179]: cluster 2023-10-09T08:53:13.380392+0000 mgr.y (mgr.25421) 467 : cluster 0 pgmap v414: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 24705/200031 objects degraded (12.351%); 2210/200031 objects misplaced (1.105%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:14.877 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:14 smithi016 bash[75179]: audit 2023-10-09T08:53:13.666420+0000 mgr.y (mgr.25421) 468 : audit 0 from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:53:14.877 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:14 smithi016 bash[75179]: cluster 2023-10-09T08:53:13.941959+0000 mon.a (mon.0) 614 : cluster 3 Health check update: Degraded data redundancy: 24705/200031 objects degraded (12.351%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:14.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:14.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:14.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:14.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:15.017 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":414,"stamp":"2023-10-09T08:53:13.379771+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4368701368,"num_objects":66677,"num_object_clones":0,"num_object_copies":200031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24705,"num_objects_misplaced":2210,"num_objects_unfound":0,"num_objects_dirty":66677,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":135857,"num_write_kb":6474382,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2839,"num_bytes_recovered":185401898,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128591,"ondisk_log_size":128591,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15239168,"kb_used_data":13674304,"kb_used_omap":126,"kb_used_meta":1564481,"kb_avail":734722048,"statfs":{"total":767960285184,"available":752355377152,"internally_reserved":0,"allocated":14002487296,"data_stored":13998568933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1602029179},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,5,2,8,7,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-382468096,"num_objects":-5836,"num_object_clones":0,"num_object_copies":-17508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-289,"num_objects_unfound":0,"num_objects_dirty":-5836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5836,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003429"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":96,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.458555+0000","last_change":"2023-10-09T08:46:49.233483+0000","last_active":"2023-10-09T08:52:48.458555+0000","last_peered":"2023-10-09T08:52:48.458555+0000","last_clean":"2023-10-09T08:52:48.458555+0000","last_became_active":"2023-10-09T08:46:49.233033+0000","last_became_peered":"2023-10-09T08:46:49.233033+0000","last_unstale":"2023-10-09T08:52:48.458555+0000","last_undegraded":"2023-10-09T08:52:48.458555+0000","last_fullsized":"2023-10-09T08:52:48.458555+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.468263+0000","last_change":"2023-10-09T08:47:16.965051+0000","last_active":"2023-10-09T08:52:48.468263+0000","last_peered":"2023-10-09T08:52:48.468263+0000","last_clean":"2023-10-09T08:52:48.468263+0000","last_became_active":"2023-10-09T08:47:16.964664+0000","last_became_peered":"2023-10-09T08:47:16.964664+0000","last_unstale":"2023-10-09T08:52:48.468263+0000","last_undegraded":"2023-10-09T08:52:48.468263+0000","last_fullsized":"2023-10-09T08:52:48.468263+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.377202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.441193+0000","last_change":"2023-10-09T08:47:16.968800+0000","last_active":"2023-10-09T08:52:48.441193+0000","last_peered":"2023-10-09T08:52:48.441193+0000","last_clean":"2023-10-09T08:52:48.441193+0000","last_became_active":"2023-10-09T08:47:16.968430+0000","last_became_peered":"2023-10-09T08:47:16.968430+0000","last_unstale":"2023-10-09T08:52:48.441193+0000","last_undegraded":"2023-10-09T08:52:48.441193+0000","last_fullsized":"2023-10-09T08:52:48.441193+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:08.011498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":82,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.443120+0000","last_change":"2023-10-09T08:51:33.436343+0000","last_active":"2023-10-09T08:52:48.443120+0000","last_peered":"2023-10-09T08:52:48.443120+0000","last_clean":"2023-10-09T08:52:48.443120+0000","last_became_active":"2023-10-09T08:51:33.435777+0000","last_became_peered":"2023-10-09T08:51:33.435777+0000","last_unstale":"2023-10-09T08:52:48.443120+0000","last_undegraded":"2023-10-09T08:52:48.443120+0000","last_fullsized":"2023-10-09T08:52:48.443120+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:42.634787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":99,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.464217+0000","last_change":"2023-10-09T08:47:16.965460+0000","last_active":"2023-10-09T08:52:48.464217+0000","last_peered":"2023-10-09T08:52:48.464217+0000","last_clean":"2023-10-09T08:52:48.464217+0000","last_became_active":"2023-10-09T08:47:16.965040+0000","last_became_peered":"2023-10-09T08:47:16.965040+0000","last_unstale":"2023-10-09T08:52:48.464217+0000","last_undegraded":"2023-10-09T08:52:48.464217+0000","last_fullsized":"2023-10-09T08:52:48.464217+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.025735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.461476+0000","last_change":"2023-10-09T08:46:18.680365+0000","last_active":"2023-10-09T08:52:48.461476+0000","last_peered":"2023-10-09T08:52:48.461476+0000","last_clean":"2023-10-09T08:52:48.461476+0000","last_became_active":"2023-10-09T08:46:18.679875+0000","last_became_peered":"2023-10-09T08:46:18.679875+0000","last_unstale":"2023-10-09T08:52:48.461476+0000","last_undegraded":"2023-10-09T08:52:48.461476+0000","last_fullsized":"2023-10-09T08:52:48.461476+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":73,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:53:02.288097+0000","last_change":"2023-10-09T08:47:16.971965+0000","last_active":"2023-10-09T08:53:02.288097+0000","last_peered":"2023-10-09T08:53:02.288097+0000","last_clean":"2023-10-09T08:53:02.288097+0000","last_became_active":"2023-10-09T08:47:16.971651+0000","last_became_peered":"2023-10-09T08:47:16.971651+0000","last_unstale":"2023-10-09T08:53:02.288097+0000","last_undegraded":"2023-10-09T08:53:02.288097+0000","last_fullsized":"2023-10-09T08:53:02.288097+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:04.259603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.464069+0000","last_change":"2023-10-09T08:47:16.969948+0000","last_active":"2023-10-09T08:52:48.464069+0000","last_peered":"2023-10-09T08:52:48.464069+0000","last_clean":"2023-10-09T08:52:48.464069+0000","last_became_active":"2023-10-09T08:47:16.969666+0000","last_became_peered":"2023-10-09T08:47:16.969666+0000","last_unstale":"2023-10-09T08:52:48.464069+0000","last_undegraded":"2023-10-09T08:52:48.464069+0000","last_fullsized":"2023-10-09T08:52:48.464069+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:25.089633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.469706+0000","last_change":"2023-10-09T08:46:18.675538+0000","last_active":"2023-10-09T08:52:48.469706+0000","last_peered":"2023-10-09T08:52:48.469706+0000","last_clean":"2023-10-09T08:52:48.469706+0000","last_became_active":"2023-10-09T08:46:18.675149+0000","last_became_peered":"2023-10-09T08:46:18.675149+0000","last_unstale":"2023-10-09T08:52:48.469706+0000","last_undegraded":"2023-10-09T08:52:48.469706+0000","last_fullsized":"2023-10-09T08:52:48.469706+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":44,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.464201+0000","last_change":"2023-10-09T08:47:16.972498+0000","last_active":"2023-10-09T08:52:48.464201+0000","last_peered":"2023-10-09T08:52:48.464201+0000","last_clean":"2023-10-09T08:52:48.464201+0000","last_became_active":"2023-10-09T08:47:16.972184+0000","last_became_peered":"2023-10-09T08:47:16.972184+0000","last_unstale":"2023-10-09T08:52:48.464201+0000","last_undegraded":"2023-10-09T08:52:48.464201+0000","last_fullsized":"2023-10-09T08:52:48.464201+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:03.559858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":82,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.468590+0000","last_change":"2023-10-09T08:47:16.973891+0000","last_active":"2023-10-09T08:52:48.468590+0000","last_peered":"2023-10-09T08:52:48.468590+0000","last_clean":"2023-10-09T08:52:48.468590+0000","last_became_active":"2023-10-09T08:47:16.973405+0000","last_became_peered":"2023-10-09T08:47:16.973405+0000","last_unstale":"2023-10-09T08:52:48.468590+0000","last_undegraded":"2023-10-09T08:52:48.468590+0000","last_fullsized":"2023-10-09T08:52:48.468590+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.442688+0000","last_change":"2023-10-09T08:46:49.231880+0000","last_active":"2023-10-09T08:52:48.442688+0000","last_peered":"2023-10-09T08:52:48.442688+0000","last_clean":"2023-10-09T08:52:48.442688+0000","last_became_active":"2023-10-09T08:46:49.231471+0000","last_became_peered":"2023-10-09T08:46:49.231471+0000","last_unstale":"2023-10-09T08:52:48.442688+0000","last_undegraded":"2023-10-09T08:52:48.442688+0000","last_fullsized":"2023-10-09T08:52:48.442688+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:31.742745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":87,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.457803+0000","last_change":"2023-10-09T08:34:27.730961+0000","last_active":"2023-10-09T08:52:48.457803+0000","last_peered":"2023-10-09T08:52:48.457803+0000","last_clean":"2023-10-09T08:52:48.457803+0000","last_became_active":"2023-10-09T08:34:27.730198+0000","last_became_peered":"2023-10-09T08:34:27.730198+0000","last_unstale":"2023-10-09T08:52:48.457803+0000","last_undegraded":"2023-10-09T08:52:48.457803+0000","last_fullsized":"2023-10-09T08:52:48.457803+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.458244+0000","last_change":"2023-10-09T08:46:18.676902+0000","last_active":"2023-10-09T08:52:48.458244+0000","last_peered":"2023-10-09T08:52:48.458244+0000","last_clean":"2023-10-09T08:52:48.458244+0000","last_became_active":"2023-10-09T08:46:18.676560+0000","last_became_peered":"2023-10-09T08:46:18.676560+0000","last_unstale":"2023-10-09T08:52:48.458244+0000","last_undegraded":"2023-10-09T08:52:48.458244+0000","last_fullsized":"2023-10-09T08:52:48.458244+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":81,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.463487+0000","last_change":"2023-10-09T08:46:49.233182+0000","last_active":"2023-10-09T08:52:48.463487+0000","last_peered":"2023-10-09T08:52:48.463487+0000","last_clean":"2023-10-09T08:52:48.463487+0000","last_became_active":"2023-10-09T08:46:49.232746+0000","last_became_peered":"2023-10-09T08:46:49.232746+0000","last_unstale":"2023-10-09T08:52:48.463487+0000","last_undegraded":"2023-10-09T08:52:48.463487+0000","last_fullsized":"2023-10-09T08:52:48.463487+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":83,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.464618+0000","last_change":"2023-10-09T08:47:16.974125+0000","last_active":"2023-10-09T08:52:48.464618+0000","last_peered":"2023-10-09T08:52:48.464618+0000","last_clean":"2023-10-09T08:52:48.464618+0000","last_became_active":"2023-10-09T08:47:16.973763+0000","last_became_peered":"2023-10-09T08:47:16.973763+0000","last_unstale":"2023-10-09T08:52:48.464618+0000","last_undegraded":"2023-10-09T08:52:48.464618+0000","last_fullsized":"2023-10-09T08:52:48.464618+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:31.548547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":82,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.442928+0000","last_change":"2023-10-09T08:51:33.436881+0000","last_active":"2023-10-09T08:52:48.442928+0000","last_peered":"2023-10-09T08:52:48.442928+0000","last_clean":"2023-10-09T08:52:48.442928+0000","last_became_active":"2023-10-09T08:51:33.436164+0000","last_became_peered":"2023-10-09T08:51:33.436164+0000","last_unstale":"2023-10-09T08:52:48.442928+0000","last_undegraded":"2023-10-09T08:52:48.442928+0000","last_fullsized":"2023-10-09T08:52:48.442928+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:34.919142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.469916+0000","last_change":"2023-10-09T08:47:16.975645+0000","last_active":"2023-10-09T08:52:48.469916+0000","last_peered":"2023-10-09T08:52:48.469916+0000","last_clean":"2023-10-09T08:52:48.469916+0000","last_became_active":"2023-10-09T08:47:16.975277+0000","last_became_peered":"2023-10-09T08:47:16.975277+0000","last_unstale":"2023-10-09T08:52:48.469916+0000","last_undegraded":"2023-10-09T08:52:48.469916+0000","last_fullsized":"2023-10-09T08:52:48.469916+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":64,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.466888+0000","last_change":"2023-10-09T08:34:27.726666+0000","last_active":"2023-10-09T08:52:48.466888+0000","last_peered":"2023-10-09T08:52:48.466888+0000","last_clean":"2023-10-09T08:52:48.466888+0000","last_became_active":"2023-10-09T08:34:27.726189+0000","last_became_peered":"2023-10-09T08:34:27.726189+0000","last_unstale":"2023-10-09T08:52:48.466888+0000","last_undegraded":"2023-10-09T08:52:48.466888+0000","last_fullsized":"2023-10-09T08:52:48.466888+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":73,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:53:02.288000+0000","last_change":"2023-10-09T08:47:16.971847+0000","last_active":"2023-10-09T08:53:02.288000+0000","last_peered":"2023-10-09T08:53:02.288000+0000","last_clean":"2023-10-09T08:53:02.288000+0000","last_became_active":"2023-10-09T08:47:16.970986+0000","last_became_peered":"2023-10-09T08:47:16.970986+0000","last_unstale":"2023-10-09T08:53:02.288000+0000","last_undegraded":"2023-10-09T08:53:02.288000+0000","last_fullsized":"2023-10-09T08:53:02.288000+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:06.084956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":126,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.460999+0000","last_change":"2023-10-09T08:51:53.462296+0000","last_active":"2023-10-09T08:52:48.460999+0000","last_peered":"2023-10-09T08:52:48.460999+0000","last_clean":"2023-10-09T08:52:48.460999+0000","last_became_active":"2023-10-09T08:51:53.454854+0000","last_became_peered":"2023-10-09T08:51:53.454854+0000","last_unstale":"2023-10-09T08:52:48.460999+0000","last_undegraded":"2023-10-09T08:52:48.460999+0000","last_fullsized":"2023-10-09T08:52:48.460999+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.465749+0000","last_change":"2023-10-09T08:51:33.443270+0000","last_active":"2023-10-09T08:52:48.465749+0000","last_peered":"2023-10-09T08:52:48.465749+0000","last_clean":"2023-10-09T08:52:48.465749+0000","last_became_active":"2023-10-09T08:51:33.442857+0000","last_became_peered":"2023-10-09T08:51:33.442857+0000","last_unstale":"2023-10-09T08:52:48.465749+0000","last_undegraded":"2023-10-09T08:52:48.465749+0000","last_fullsized":"2023-10-09T08:52:48.465749+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"113'8250","reported_seq":8276,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:10.043291+0000","last_change":"2023-10-09T08:50:25.389409+0000","last_active":"2023-10-09T08:53:10.043291+0000","last_peered":"2023-10-09T08:53:10.043291+0000","last_clean":"2023-10-09T08:53:10.043291+0000","last_became_active":"2023-10-09T08:50:25.388956+0000","last_became_peered":"2023-10-09T08:50:25.388956+0000","last_unstale":"2023-10-09T08:53:10.043291+0000","last_undegraded":"2023-10-09T08:53:10.043291+0000","last_fullsized":"2023-10-09T08:53:10.043291+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8250,"log_dups_size":0,"ondisk_log_size":8250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268697600,"num_objects":4100,"num_object_clones":0,"num_object_copies":12300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8250,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.465829+0000","last_change":"2023-10-09T08:45:49.182558+0000","last_active":"2023-10-09T08:52:48.465829+0000","last_peered":"2023-10-09T08:52:48.465829+0000","last_clean":"2023-10-09T08:52:48.465829+0000","last_became_active":"2023-10-09T08:45:49.182145+0000","last_became_peered":"2023-10-09T08:45:49.182145+0000","last_unstale":"2023-10-09T08:52:48.465829+0000","last_undegraded":"2023-10-09T08:52:48.465829+0000","last_fullsized":"2023-10-09T08:52:48.465829+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":64,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.469304+0000","last_change":"2023-10-09T08:34:29.490932+0000","last_active":"2023-10-09T08:52:48.469304+0000","last_peered":"2023-10-09T08:52:48.469304+0000","last_clean":"2023-10-09T08:52:48.469304+0000","last_became_active":"2023-10-09T08:34:29.489841+0000","last_became_peered":"2023-10-09T08:34:29.489841+0000","last_unstale":"2023-10-09T08:52:48.469304+0000","last_undegraded":"2023-10-09T08:52:48.469304+0000","last_fullsized":"2023-10-09T08:52:48.469304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"113'8468","reported_seq":8496,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:10.177165+0000","last_change":"2023-10-09T08:50:24.942723+0000","last_active":"2023-10-09T08:53:10.177165+0000","last_peered":"2023-10-09T08:53:10.177165+0000","last_clean":"2023-10-09T08:53:10.177165+0000","last_became_active":"2023-10-09T08:50:24.942192+0000","last_became_peered":"2023-10-09T08:50:24.942192+0000","last_unstale":"2023-10-09T08:53:10.177165+0000","last_undegraded":"2023-10-09T08:53:10.177165+0000","last_fullsized":"2023-10-09T08:53:10.177165+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8468,"log_dups_size":0,"ondisk_log_size":8468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:21.109317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277479424,"num_objects":4234,"num_object_clones":0,"num_object_copies":12702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8468,"num_write_kb":406464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":81,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.465377+0000","last_change":"2023-10-09T08:46:18.674774+0000","last_active":"2023-10-09T08:52:48.465377+0000","last_peered":"2023-10-09T08:52:48.465377+0000","last_clean":"2023-10-09T08:52:48.465377+0000","last_became_active":"2023-10-09T08:46:18.674334+0000","last_became_peered":"2023-10-09T08:46:18.674334+0000","last_unstale":"2023-10-09T08:52:48.465377+0000","last_undegraded":"2023-10-09T08:52:48.465377+0000","last_fullsized":"2023-10-09T08:52:48.465377+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":112,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.457667+0000","last_change":"2023-10-09T08:51:33.439973+0000","last_active":"2023-10-09T08:52:48.457667+0000","last_peered":"2023-10-09T08:52:48.457667+0000","last_clean":"2023-10-09T08:52:48.457667+0000","last_became_active":"2023-10-09T08:51:33.439461+0000","last_became_peered":"2023-10-09T08:51:33.439461+0000","last_unstale":"2023-10-09T08:52:48.457667+0000","last_undegraded":"2023-10-09T08:52:48.457667+0000","last_fullsized":"2023-10-09T08:52:48.457667+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"113'8658","reported_seq":8684,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:10.376690+0000","last_change":"2023-10-09T08:50:24.948333+0000","last_active":"2023-10-09T08:53:10.376690+0000","last_peered":"2023-10-09T08:53:10.376690+0000","last_clean":"2023-10-09T08:53:10.376690+0000","last_became_active":"2023-10-09T08:50:24.947969+0000","last_became_peered":"2023-10-09T08:50:24.947969+0000","last_unstale":"2023-10-09T08:53:10.376690+0000","last_undegraded":"2023-10-09T08:53:10.376690+0000","last_fullsized":"2023-10-09T08:53:10.376690+0000","mapping_epoch":95,"log_start":"101'500","ondisk_log_start":"101'500","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8158,"log_dups_size":0,"ondisk_log_size":8158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282722304,"num_objects":4314,"num_object_clones":0,"num_object_copies":12942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8658,"num_write_kb":415104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":65,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.472999+0000","last_change":"2023-10-09T08:34:27.733891+0000","last_active":"2023-10-09T08:52:48.472999+0000","last_peered":"2023-10-09T08:52:48.472999+0000","last_clean":"2023-10-09T08:52:48.472999+0000","last_became_active":"2023-10-09T08:34:27.733379+0000","last_became_peered":"2023-10-09T08:34:27.733379+0000","last_unstale":"2023-10-09T08:52:48.472999+0000","last_undegraded":"2023-10-09T08:52:48.472999+0000","last_fullsized":"2023-10-09T08:52:48.472999+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":88,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.465938+0000","last_change":"2023-10-09T08:46:49.231597+0000","last_active":"2023-10-09T08:52:48.465938+0000","last_peered":"2023-10-09T08:52:48.465938+0000","last_clean":"2023-10-09T08:52:48.465938+0000","last_became_active":"2023-10-09T08:46:49.231198+0000","last_became_peered":"2023-10-09T08:46:49.231198+0000","last_unstale":"2023-10-09T08:52:48.465938+0000","last_undegraded":"2023-10-09T08:52:48.465938+0000","last_fullsized":"2023-10-09T08:52:48.465938+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"113'8469","reported_seq":8497,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:12.168548+0000","last_change":"2023-10-09T08:50:24.942781+0000","last_active":"2023-10-09T08:53:12.168548+0000","last_peered":"2023-10-09T08:53:12.168548+0000","last_clean":"2023-10-09T08:53:12.168548+0000","last_became_active":"2023-10-09T08:50:24.942354+0000","last_became_peered":"2023-10-09T08:50:24.942354+0000","last_unstale":"2023-10-09T08:53:12.168548+0000","last_undegraded":"2023-10-09T08:53:12.168548+0000","last_fullsized":"2023-10-09T08:53:12.168548+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8469,"log_dups_size":0,"ondisk_log_size":8469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:17.475760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264306688,"num_objects":4033,"num_object_clones":0,"num_object_copies":12099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8469,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.473432+0000","last_change":"2023-10-09T08:47:16.975514+0000","last_active":"2023-10-09T08:52:48.473432+0000","last_peered":"2023-10-09T08:52:48.473432+0000","last_clean":"2023-10-09T08:52:48.473432+0000","last_became_active":"2023-10-09T08:47:16.975184+0000","last_became_peered":"2023-10-09T08:47:16.975184+0000","last_unstale":"2023-10-09T08:52:48.473432+0000","last_undegraded":"2023-10-09T08:52:48.473432+0000","last_fullsized":"2023-10-09T08:52:48.473432+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.443899+0000","last_change":"2023-10-09T08:46:18.683057+0000","last_active":"2023-10-09T08:52:48.443899+0000","last_peered":"2023-10-09T08:52:48.443899+0000","last_clean":"2023-10-09T08:52:48.443899+0000","last_became_active":"2023-10-09T08:46:18.682672+0000","last_became_peered":"2023-10-09T08:46:18.682672+0000","last_unstale":"2023-10-09T08:52:48.443899+0000","last_undegraded":"2023-10-09T08:52:48.443899+0000","last_fullsized":"2023-10-09T08:52:48.443899+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:42.975534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"113'8397","reported_seq":9534,"reported_epoch":113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T08:53:10.288120+0000","last_change":"2023-10-09T08:51:53.464262+0000","last_active":"2023-10-09T08:53:10.288120+0000","last_peered":"2023-10-09T08:53:10.288120+0000","last_clean":"2023-10-09T08:51:32.426090+0000","last_became_active":"2023-10-09T08:51:34.579869+0000","last_became_peered":"2023-10-09T08:51:34.579869+0000","last_unstale":"2023-10-09T08:53:10.288120+0000","last_undegraded":"2023-10-09T08:51:53.462339+0000","last_fullsized":"2023-10-09T08:51:34.434299+0000","mapping_epoch":105,"log_start":"101'1000","ondisk_log_start":"101'1000","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274923520,"num_objects":4195,"num_object_clones":0,"num_object_copies":12585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3887,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8397,"num_write_kb":402944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":550,"num_bytes_recovered":36044800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":2336},{"shards":"1,4","objects":308}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":82,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.464730+0000","last_change":"2023-10-09T08:47:16.965045+0000","last_active":"2023-10-09T08:52:48.464730+0000","last_peered":"2023-10-09T08:52:48.464730+0000","last_clean":"2023-10-09T08:52:48.464730+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:52:48.464730+0000","last_undegraded":"2023-10-09T08:52:48.464730+0000","last_fullsized":"2023-10-09T08:52:48.464730+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:49.126847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.457887+0000","last_change":"2023-10-09T08:47:16.972109+0000","last_active":"2023-10-09T08:52:48.457887+0000","last_peered":"2023-10-09T08:52:48.457887+0000","last_clean":"2023-10-09T08:52:48.457887+0000","last_became_active":"2023-10-09T08:47:16.971716+0000","last_became_peered":"2023-10-09T08:47:16.971716+0000","last_unstale":"2023-10-09T08:52:48.457887+0000","last_undegraded":"2023-10-09T08:52:48.457887+0000","last_fullsized":"2023-10-09T08:52:48.457887+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"113'8498","reported_seq":8526,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:12.374952+0000","last_change":"2023-10-09T08:50:24.945100+0000","last_active":"2023-10-09T08:53:12.374952+0000","last_peered":"2023-10-09T08:53:12.374952+0000","last_clean":"2023-10-09T08:53:12.374952+0000","last_became_active":"2023-10-09T08:50:24.944761+0000","last_became_peered":"2023-10-09T08:50:24.944761+0000","last_unstale":"2023-10-09T08:53:12.374952+0000","last_undegraded":"2023-10-09T08:53:12.374952+0000","last_fullsized":"2023-10-09T08:53:12.374952+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8498,"log_dups_size":0,"ondisk_log_size":8498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:11.053008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268435456,"num_objects":4096,"num_object_clones":0,"num_object_copies":12288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8498,"num_write_kb":403008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.467204+0000","last_change":"2023-10-09T08:46:18.681222+0000","last_active":"2023-10-09T08:52:48.467204+0000","last_peered":"2023-10-09T08:52:48.467204+0000","last_clean":"2023-10-09T08:52:48.467204+0000","last_became_active":"2023-10-09T08:46:18.680710+0000","last_became_peered":"2023-10-09T08:46:18.680710+0000","last_unstale":"2023-10-09T08:52:48.467204+0000","last_undegraded":"2023-10-09T08:52:48.467204+0000","last_fullsized":"2023-10-09T08:52:48.467204+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":94,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.461099+0000","last_change":"2023-10-09T08:46:49.234714+0000","last_active":"2023-10-09T08:52:48.461099+0000","last_peered":"2023-10-09T08:52:48.461099+0000","last_clean":"2023-10-09T08:52:48.461099+0000","last_became_active":"2023-10-09T08:46:49.234307+0000","last_became_peered":"2023-10-09T08:46:49.234307+0000","last_unstale":"2023-10-09T08:52:48.461099+0000","last_undegraded":"2023-10-09T08:52:48.461099+0000","last_fullsized":"2023-10-09T08:52:48.461099+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"113'8314","reported_seq":8345,"reported_epoch":113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T08:53:10.366440+0000","last_change":"2023-10-09T08:51:54.698630+0000","last_active":"2023-10-09T08:53:10.366440+0000","last_peered":"2023-10-09T08:53:10.366440+0000","last_clean":"2023-10-09T08:51:52.369065+0000","last_became_active":"2023-10-09T08:51:54.680135+0000","last_became_peered":"2023-10-09T08:51:54.680135+0000","last_unstale":"2023-10-09T08:53:10.366440+0000","last_undegraded":"2023-10-09T08:51:54.488166+0000","last_fullsized":"2023-10-09T08:51:54.454793+0000","mapping_epoch":108,"log_start":"101'400","ondisk_log_start":"101'400","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270401536,"num_objects":4126,"num_object_clones":0,"num_object_copies":12378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5347,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8314,"num_write_kb":398080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":3253},{"shards":"5,6","objects":-1221}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.457488+0000","last_change":"2023-10-09T08:46:49.235099+0000","last_active":"2023-10-09T08:52:48.457488+0000","last_peered":"2023-10-09T08:52:48.457488+0000","last_clean":"2023-10-09T08:52:48.457488+0000","last_became_active":"2023-10-09T08:46:49.234136+0000","last_became_peered":"2023-10-09T08:46:49.234136+0000","last_unstale":"2023-10-09T08:52:48.457488+0000","last_undegraded":"2023-10-09T08:52:48.457488+0000","last_fullsized":"2023-10-09T08:52:48.457488+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.468657+0000","last_change":"2023-10-09T08:46:18.676843+0000","last_active":"2023-10-09T08:52:48.468657+0000","last_peered":"2023-10-09T08:52:48.468657+0000","last_clean":"2023-10-09T08:52:48.468657+0000","last_became_active":"2023-10-09T08:46:18.676468+0000","last_became_peered":"2023-10-09T08:46:18.676468+0000","last_unstale":"2023-10-09T08:52:48.468657+0000","last_undegraded":"2023-10-09T08:52:48.468657+0000","last_fullsized":"2023-10-09T08:52:48.468657+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"113'8311","reported_seq":8338,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:10.160320+0000","last_change":"2023-10-09T08:50:24.948807+0000","last_active":"2023-10-09T08:53:10.160320+0000","last_peered":"2023-10-09T08:53:10.160320+0000","last_clean":"2023-10-09T08:53:10.160320+0000","last_became_active":"2023-10-09T08:50:24.948396+0000","last_became_peered":"2023-10-09T08:50:24.948396+0000","last_unstale":"2023-10-09T08:53:10.160320+0000","last_undegraded":"2023-10-09T08:53:10.160320+0000","last_fullsized":"2023-10-09T08:53:10.160320+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8311,"log_dups_size":0,"ondisk_log_size":8311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266600472,"num_objects":4069,"num_object_clones":0,"num_object_copies":12207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4069,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8311,"num_write_kb":396097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.441204+0000","last_change":"2023-10-09T08:51:33.439568+0000","last_active":"2023-10-09T08:52:48.441204+0000","last_peered":"2023-10-09T08:52:48.441204+0000","last_clean":"2023-10-09T08:52:48.441204+0000","last_became_active":"2023-10-09T08:51:33.439106+0000","last_became_peered":"2023-10-09T08:51:33.439106+0000","last_unstale":"2023-10-09T08:52:48.441204+0000","last_undegraded":"2023-10-09T08:52:48.441204+0000","last_fullsized":"2023-10-09T08:52:48.441204+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:57.602485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":69,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:53:01.908032+0000","last_change":"2023-10-09T08:47:16.965479+0000","last_active":"2023-10-09T08:53:01.908032+0000","last_peered":"2023-10-09T08:53:01.908032+0000","last_clean":"2023-10-09T08:53:01.908032+0000","last_became_active":"2023-10-09T08:47:16.965141+0000","last_became_peered":"2023-10-09T08:47:16.965141+0000","last_unstale":"2023-10-09T08:53:01.908032+0000","last_undegraded":"2023-10-09T08:53:01.908032+0000","last_fullsized":"2023-10-09T08:53:01.908032+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:50.145649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":94,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.446096+0000","last_change":"2023-10-09T08:46:18.680373+0000","last_active":"2023-10-09T08:52:48.446096+0000","last_peered":"2023-10-09T08:52:48.446096+0000","last_clean":"2023-10-09T08:52:48.446096+0000","last_became_active":"2023-10-09T08:46:18.679999+0000","last_became_peered":"2023-10-09T08:46:18.679999+0000","last_unstale":"2023-10-09T08:52:48.446096+0000","last_undegraded":"2023-10-09T08:52:48.446096+0000","last_fullsized":"2023-10-09T08:52:48.446096+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:32.374511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":95,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.469984+0000","last_change":"2023-10-09T08:47:16.976238+0000","last_active":"2023-10-09T08:52:48.469984+0000","last_peered":"2023-10-09T08:52:48.469984+0000","last_clean":"2023-10-09T08:52:48.469984+0000","last_became_active":"2023-10-09T08:47:16.975840+0000","last_became_peered":"2023-10-09T08:47:16.975840+0000","last_unstale":"2023-10-09T08:52:48.469984+0000","last_undegraded":"2023-10-09T08:52:48.469984+0000","last_fullsized":"2023-10-09T08:52:48.469984+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.442606+0000","last_change":"2023-10-09T08:46:18.684501+0000","last_active":"2023-10-09T08:52:48.442606+0000","last_peered":"2023-10-09T08:52:48.442606+0000","last_clean":"2023-10-09T08:52:48.442606+0000","last_became_active":"2023-10-09T08:46:18.684159+0000","last_became_peered":"2023-10-09T08:46:18.684159+0000","last_unstale":"2023-10-09T08:52:48.442606+0000","last_undegraded":"2023-10-09T08:52:48.442606+0000","last_fullsized":"2023-10-09T08:52:48.442606+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:21.270259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"113'8775","reported_seq":8803,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:12.286475+0000","last_change":"2023-10-09T08:50:24.945548+0000","last_active":"2023-10-09T08:53:12.286475+0000","last_peered":"2023-10-09T08:53:12.286475+0000","last_clean":"2023-10-09T08:53:12.286475+0000","last_became_active":"2023-10-09T08:50:24.945187+0000","last_became_peered":"2023-10-09T08:50:24.945187+0000","last_unstale":"2023-10-09T08:53:12.286475+0000","last_undegraded":"2023-10-09T08:53:12.286475+0000","last_fullsized":"2023-10-09T08:53:12.286475+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8775,"log_dups_size":0,"ondisk_log_size":8775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:10:48.344698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278331392,"num_objects":4247,"num_object_clones":0,"num_object_copies":12741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8775,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"113'8581","reported_seq":8616,"reported_epoch":113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T08:53:10.306547+0000","last_change":"2023-10-09T08:51:54.732974+0000","last_active":"2023-10-09T08:53:10.306547+0000","last_peered":"2023-10-09T08:53:10.306547+0000","last_clean":"2023-10-09T08:51:52.408850+0000","last_became_active":"2023-10-09T08:51:54.721534+0000","last_became_peered":"2023-10-09T08:51:54.721534+0000","last_unstale":"2023-10-09T08:53:10.306547+0000","last_undegraded":"2023-10-09T08:51:54.645910+0000","last_fullsized":"2023-10-09T08:51:54.453457+0000","mapping_epoch":108,"log_start":"101'1200","ondisk_log_start":"101'1200","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7381,"log_dups_size":0,"ondisk_log_size":7381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279248896,"num_objects":4261,"num_object_clones":0,"num_object_copies":12783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5486,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8581,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3326},{"shards":"4,7","objects":-1225}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":84,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.470373+0000","last_change":"2023-10-09T08:46:18.680020+0000","last_active":"2023-10-09T08:52:48.470373+0000","last_peered":"2023-10-09T08:52:48.470373+0000","last_clean":"2023-10-09T08:52:48.470373+0000","last_became_active":"2023-10-09T08:46:18.679652+0000","last_became_peered":"2023-10-09T08:46:18.679652+0000","last_unstale":"2023-10-09T08:52:48.470373+0000","last_undegraded":"2023-10-09T08:52:48.470373+0000","last_fullsized":"2023-10-09T08:52:48.470373+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":80,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.458037+0000","last_change":"2023-10-09T08:45:49.184088+0000","last_active":"2023-10-09T08:52:48.458037+0000","last_peered":"2023-10-09T08:52:48.458037+0000","last_clean":"2023-10-09T08:52:48.458037+0000","last_became_active":"2023-10-09T08:45:49.183674+0000","last_became_peered":"2023-10-09T08:45:49.183674+0000","last_unstale":"2023-10-09T08:52:48.458037+0000","last_undegraded":"2023-10-09T08:52:48.458037+0000","last_fullsized":"2023-10-09T08:52:48.458037+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.473291+0000","last_change":"2023-10-09T08:51:33.441403+0000","last_active":"2023-10-09T08:52:48.473291+0000","last_peered":"2023-10-09T08:52:48.473291+0000","last_clean":"2023-10-09T08:52:48.473291+0000","last_became_active":"2023-10-09T08:51:33.440813+0000","last_became_peered":"2023-10-09T08:51:33.440813+0000","last_unstale":"2023-10-09T08:52:48.473291+0000","last_undegraded":"2023-10-09T08:52:48.473291+0000","last_fullsized":"2023-10-09T08:52:48.473291+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"113'8435","reported_seq":8461,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:10.317192+0000","last_change":"2023-10-09T08:50:25.388446+0000","last_active":"2023-10-09T08:53:10.317192+0000","last_peered":"2023-10-09T08:53:10.317192+0000","last_clean":"2023-10-09T08:53:10.317192+0000","last_became_active":"2023-10-09T08:50:25.387879+0000","last_became_peered":"2023-10-09T08:50:25.387879+0000","last_unstale":"2023-10-09T08:53:10.317192+0000","last_undegraded":"2023-10-09T08:53:10.317192+0000","last_fullsized":"2023-10-09T08:53:10.317192+0000","mapping_epoch":95,"log_start":"101'300","ondisk_log_start":"101'300","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8135,"log_dups_size":0,"ondisk_log_size":8135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267452416,"num_objects":4081,"num_object_clones":0,"num_object_copies":12243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8435,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":114,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:53:02.392291+0000","last_change":"2023-10-09T08:51:33.431789+0000","last_active":"2023-10-09T08:53:02.392291+0000","last_peered":"2023-10-09T08:53:02.392291+0000","last_clean":"2023-10-09T08:53:02.392291+0000","last_became_active":"2023-10-09T08:51:33.431448+0000","last_became_peered":"2023-10-09T08:51:33.431448+0000","last_unstale":"2023-10-09T08:53:02.392291+0000","last_undegraded":"2023-10-09T08:53:02.392291+0000","last_fullsized":"2023-10-09T08:53:02.392291+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:42.314482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":69,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.441995+0000","last_change":"2023-10-09T08:46:49.231894+0000","last_active":"2023-10-09T08:52:48.441995+0000","last_peered":"2023-10-09T08:52:48.441995+0000","last_clean":"2023-10-09T08:52:48.441995+0000","last_became_active":"2023-10-09T08:46:49.231531+0000","last_became_peered":"2023-10-09T08:46:49.231531+0000","last_unstale":"2023-10-09T08:52:48.441995+0000","last_undegraded":"2023-10-09T08:52:48.441995+0000","last_fullsized":"2023-10-09T08:52:48.441995+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:50.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.443218+0000","last_change":"2023-10-09T08:46:18.681375+0000","last_active":"2023-10-09T08:52:48.443218+0000","last_peered":"2023-10-09T08:52:48.443218+0000","last_clean":"2023-10-09T08:52:48.443218+0000","last_became_active":"2023-10-09T08:46:18.680713+0000","last_became_peered":"2023-10-09T08:46:18.680713+0000","last_unstale":"2023-10-09T08:52:48.443218+0000","last_undegraded":"2023-10-09T08:52:48.443218+0000","last_fullsized":"2023-10-09T08:52:48.443218+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:59.671965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"113'8369","reported_seq":8395,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:10.164352+0000","last_change":"2023-10-09T08:50:24.947882+0000","last_active":"2023-10-09T08:53:10.164352+0000","last_peered":"2023-10-09T08:53:10.164352+0000","last_clean":"2023-10-09T08:53:10.164352+0000","last_became_active":"2023-10-09T08:50:24.947467+0000","last_became_peered":"2023-10-09T08:50:24.947467+0000","last_unstale":"2023-10-09T08:53:10.164352+0000","last_undegraded":"2023-10-09T08:53:10.164352+0000","last_fullsized":"2023-10-09T08:53:10.164352+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8369,"log_dups_size":0,"ondisk_log_size":8369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270860288,"num_objects":4133,"num_object_clones":0,"num_object_copies":12399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8369,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":84,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.469596+0000","last_change":"2023-10-09T08:46:49.234616+0000","last_active":"2023-10-09T08:52:48.469596+0000","last_peered":"2023-10-09T08:52:48.469596+0000","last_clean":"2023-10-09T08:52:48.469596+0000","last_became_active":"2023-10-09T08:46:49.234217+0000","last_became_peered":"2023-10-09T08:46:49.234217+0000","last_unstale":"2023-10-09T08:52:48.469596+0000","last_undegraded":"2023-10-09T08:52:48.469596+0000","last_fullsized":"2023-10-09T08:52:48.469596+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":315,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.467124+0000","last_change":"2023-10-09T08:45:49.183161+0000","last_active":"2023-10-09T08:52:48.467124+0000","last_peered":"2023-10-09T08:52:48.467124+0000","last_clean":"2023-10-09T08:52:48.467124+0000","last_became_active":"2023-10-09T08:45:49.182560+0000","last_became_peered":"2023-10-09T08:45:49.182560+0000","last_unstale":"2023-10-09T08:52:48.467124+0000","last_undegraded":"2023-10-09T08:52:48.467124+0000","last_fullsized":"2023-10-09T08:52:48.467124+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":33,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.469374+0000","last_change":"2023-10-09T08:51:33.440228+0000","last_active":"2023-10-09T08:52:48.469374+0000","last_peered":"2023-10-09T08:52:48.469374+0000","last_clean":"2023-10-09T08:52:48.469374+0000","last_became_active":"2023-10-09T08:51:33.439714+0000","last_became_peered":"2023-10-09T08:51:33.439714+0000","last_unstale":"2023-10-09T08:52:48.469374+0000","last_undegraded":"2023-10-09T08:52:48.469374+0000","last_fullsized":"2023-10-09T08:52:48.469374+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":64,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.461463+0000","last_change":"2023-10-09T08:34:27.860287+0000","last_active":"2023-10-09T08:52:48.461463+0000","last_peered":"2023-10-09T08:52:48.461463+0000","last_clean":"2023-10-09T08:52:48.461463+0000","last_became_active":"2023-10-09T08:34:27.859851+0000","last_became_peered":"2023-10-09T08:34:27.859851+0000","last_unstale":"2023-10-09T08:52:48.461463+0000","last_undegraded":"2023-10-09T08:52:48.461463+0000","last_fullsized":"2023-10-09T08:52:48.461463+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":98,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.467198+0000","last_change":"2023-10-09T08:46:18.680004+0000","last_active":"2023-10-09T08:52:48.467198+0000","last_peered":"2023-10-09T08:52:48.467198+0000","last_clean":"2023-10-09T08:52:48.467198+0000","last_became_active":"2023-10-09T08:46:18.679617+0000","last_became_peered":"2023-10-09T08:46:18.679617+0000","last_unstale":"2023-10-09T08:52:48.467198+0000","last_undegraded":"2023-10-09T08:52:48.467198+0000","last_fullsized":"2023-10-09T08:52:48.467198+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.442567+0000","last_change":"2023-10-09T08:51:33.441477+0000","last_active":"2023-10-09T08:52:48.442567+0000","last_peered":"2023-10-09T08:52:48.442567+0000","last_clean":"2023-10-09T08:52:48.442567+0000","last_became_active":"2023-10-09T08:51:33.441144+0000","last_became_peered":"2023-10-09T08:51:33.441144+0000","last_unstale":"2023-10-09T08:52:48.442567+0000","last_undegraded":"2023-10-09T08:52:48.442567+0000","last_fullsized":"2023-10-09T08:52:48.442567+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:54.567772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.443306+0000","last_change":"2023-10-09T08:46:49.232279+0000","last_active":"2023-10-09T08:52:48.443306+0000","last_peered":"2023-10-09T08:52:48.443306+0000","last_clean":"2023-10-09T08:52:48.443306+0000","last_became_active":"2023-10-09T08:46:49.231911+0000","last_became_peered":"2023-10-09T08:46:49.231911+0000","last_unstale":"2023-10-09T08:52:48.443306+0000","last_undegraded":"2023-10-09T08:52:48.443306+0000","last_fullsized":"2023-10-09T08:52:48.443306+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:38.497218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"113'8538","reported_seq":13130,"reported_epoch":113,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T08:53:11.893472+0000","last_change":"2023-10-09T08:51:53.477978+0000","last_active":"2023-10-09T08:53:11.893472+0000","last_peered":"2023-10-09T08:53:11.893472+0000","last_clean":"2023-10-09T08:51:32.412115+0000","last_became_active":"2023-10-09T08:51:34.766231+0000","last_became_peered":"2023-10-09T08:51:34.766231+0000","last_unstale":"2023-10-09T08:53:11.893472+0000","last_undegraded":"2023-10-09T08:53:11.893472+0000","last_fullsized":"2023-10-09T08:51:34.426974+0000","mapping_epoch":105,"log_start":"101'1200","ondisk_log_start":"101'1200","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7338,"log_dups_size":0,"ondisk_log_size":7338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:09.433080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272105472,"num_objects":4152,"num_object_clones":0,"num_object_copies":12456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2210,"num_objects_unfound":0,"num_objects_dirty":4152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8538,"num_write_kb":406080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2278,"num_bytes_recovered":149291008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":2210},{"shards":"1,5","objects":1942}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":83,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.468678+0000","last_change":"2023-10-09T08:45:49.181115+0000","last_active":"2023-10-09T08:52:48.468678+0000","last_peered":"2023-10-09T08:52:48.468678+0000","last_clean":"2023-10-09T08:52:48.468678+0000","last_became_active":"2023-10-09T08:45:49.180736+0000","last_became_peered":"2023-10-09T08:45:49.180736+0000","last_unstale":"2023-10-09T08:52:48.468678+0000","last_undegraded":"2023-10-09T08:52:48.468678+0000","last_fullsized":"2023-10-09T08:52:48.468678+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":69,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.444789+0000","last_change":"2023-10-09T08:46:49.233112+0000","last_active":"2023-10-09T08:52:48.444789+0000","last_peered":"2023-10-09T08:52:48.444789+0000","last_clean":"2023-10-09T08:52:48.444789+0000","last_became_active":"2023-10-09T08:46:49.232724+0000","last_became_peered":"2023-10-09T08:46:49.232724+0000","last_unstale":"2023-10-09T08:52:48.444789+0000","last_undegraded":"2023-10-09T08:52:48.444789+0000","last_fullsized":"2023-10-09T08:52:48.444789+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.285159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.442631+0000","last_change":"2023-10-09T08:45:49.182610+0000","last_active":"2023-10-09T08:52:48.442631+0000","last_peered":"2023-10-09T08:52:48.442631+0000","last_clean":"2023-10-09T08:52:48.442631+0000","last_became_active":"2023-10-09T08:45:49.182013+0000","last_became_peered":"2023-10-09T08:45:49.182013+0000","last_unstale":"2023-10-09T08:52:48.442631+0000","last_undegraded":"2023-10-09T08:52:48.442631+0000","last_fullsized":"2023-10-09T08:52:48.442631+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:44.316894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"113'8511","reported_seq":8547,"reported_epoch":113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T08:53:11.750829+0000","last_change":"2023-10-09T08:51:54.741864+0000","last_active":"2023-10-09T08:53:11.750829+0000","last_peered":"2023-10-09T08:53:11.750829+0000","last_clean":"2023-10-09T08:51:52.421304+0000","last_became_active":"2023-10-09T08:51:54.727888+0000","last_became_peered":"2023-10-09T08:51:54.727888+0000","last_unstale":"2023-10-09T08:53:11.750829+0000","last_undegraded":"2023-10-09T08:51:54.592940+0000","last_fullsized":"2023-10-09T08:51:54.450912+0000","mapping_epoch":108,"log_start":"101'1100","ondisk_log_start":"101'1100","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7411,"log_dups_size":0,"ondisk_log_size":7411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:54.177829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270860288,"num_objects":4133,"num_object_clones":0,"num_object_copies":12399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8511,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":3220},{"shards":"1,6","objects":-1276}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":82,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.466798+0000","last_change":"2023-10-09T08:47:16.974508+0000","last_active":"2023-10-09T08:52:48.466798+0000","last_peered":"2023-10-09T08:52:48.466798+0000","last_clean":"2023-10-09T08:52:48.466798+0000","last_became_active":"2023-10-09T08:47:16.974255+0000","last_became_peered":"2023-10-09T08:47:16.974255+0000","last_unstale":"2023-10-09T08:52:48.466798+0000","last_undegraded":"2023-10-09T08:52:48.466798+0000","last_fullsized":"2023-10-09T08:52:48.466798+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:40.049955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":54,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.445452+0000","last_change":"2023-10-09T08:46:18.678094+0000","last_active":"2023-10-09T08:52:48.445452+0000","last_peered":"2023-10-09T08:52:48.445452+0000","last_clean":"2023-10-09T08:52:48.445452+0000","last_became_active":"2023-10-09T08:46:18.677705+0000","last_became_peered":"2023-10-09T08:46:18.677705+0000","last_unstale":"2023-10-09T08:52:48.445452+0000","last_undegraded":"2023-10-09T08:52:48.445452+0000","last_fullsized":"2023-10-09T08:52:48.445452+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:31.647828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.466984+0000","last_change":"2023-10-09T08:51:33.441276+0000","last_active":"2023-10-09T08:52:48.466984+0000","last_peered":"2023-10-09T08:52:48.466984+0000","last_clean":"2023-10-09T08:52:48.466984+0000","last_became_active":"2023-10-09T08:51:33.439762+0000","last_became_peered":"2023-10-09T08:51:33.439762+0000","last_unstale":"2023-10-09T08:52:48.466984+0000","last_undegraded":"2023-10-09T08:52:48.466984+0000","last_fullsized":"2023-10-09T08:52:48.466984+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"113'8626","reported_seq":8657,"reported_epoch":113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T08:53:10.321433+0000","last_change":"2023-10-09T08:51:34.790943+0000","last_active":"2023-10-09T08:53:10.321433+0000","last_peered":"2023-10-09T08:53:10.321433+0000","last_clean":"2023-10-09T08:51:32.405237+0000","last_became_active":"2023-10-09T08:51:34.760613+0000","last_became_peered":"2023-10-09T08:51:34.760613+0000","last_unstale":"2023-10-09T08:53:10.321433+0000","last_undegraded":"2023-10-09T08:51:34.471265+0000","last_fullsized":"2023-10-09T08:51:34.434298+0000","mapping_epoch":105,"log_start":"101'1300","ondisk_log_start":"101'1300","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281018368,"num_objects":4288,"num_object_clones":0,"num_object_copies":12864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8626,"num_write_kb":413248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":2407},{"shards":"2,4","objects":-288}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":93,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.461357+0000","last_change":"2023-10-09T08:50:37.369204+0000","last_active":"2023-10-09T08:52:48.461357+0000","last_peered":"2023-10-09T08:52:48.461357+0000","last_clean":"2023-10-09T08:52:48.461357+0000","last_became_active":"2023-10-09T08:50:37.368197+0000","last_became_peered":"2023-10-09T08:50:37.368197+0000","last_unstale":"2023-10-09T08:52:48.461357+0000","last_undegraded":"2023-10-09T08:52:48.461357+0000","last_fullsized":"2023-10-09T08:52:48.461357+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":77,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.443218+0000","last_change":"2023-10-09T08:46:18.683082+0000","last_active":"2023-10-09T08:52:48.443218+0000","last_peered":"2023-10-09T08:52:48.443218+0000","last_clean":"2023-10-09T08:52:48.443218+0000","last_became_active":"2023-10-09T08:46:18.682657+0000","last_became_peered":"2023-10-09T08:46:18.682657+0000","last_unstale":"2023-10-09T08:52:48.443218+0000","last_undegraded":"2023-10-09T08:52:48.443218+0000","last_fullsized":"2023-10-09T08:52:48.443218+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:45.531585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":64,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.458378+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:52:48.458378+0000","last_peered":"2023-10-09T08:52:48.458378+0000","last_clean":"2023-10-09T08:52:48.458378+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:52:48.458378+0000","last_undegraded":"2023-10-09T08:52:48.458378+0000","last_fullsized":"2023-10-09T08:52:48.458378+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"113'8397","reported_seq":8423,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-10-09T08:53:10.364652+0000","last_change":"2023-10-09T08:50:25.388961+0000","last_active":"2023-10-09T08:53:10.364652+0000","last_peered":"2023-10-09T08:53:10.364652+0000","last_clean":"2023-10-09T08:53:10.364652+0000","last_became_active":"2023-10-09T08:50:25.387512+0000","last_became_peered":"2023-10-09T08:50:25.387512+0000","last_unstale":"2023-10-09T08:53:10.364652+0000","last_undegraded":"2023-10-09T08:53:10.364652+0000","last_fullsized":"2023-10-09T08:53:10.364652+0000","mapping_epoch":95,"log_start":"101'200","ondisk_log_start":"101'200","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8197,"log_dups_size":0,"ondisk_log_size":8197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274661376,"num_objects":4191,"num_object_clones":0,"num_object_copies":12573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8397,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":61,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.444009+0000","last_change":"2023-10-09T08:45:49.184830+0000","last_active":"2023-10-09T08:52:48.444009+0000","last_peered":"2023-10-09T08:52:48.444009+0000","last_clean":"2023-10-09T08:52:48.444009+0000","last_became_active":"2023-10-09T08:45:49.184434+0000","last_became_peered":"2023-10-09T08:45:49.184434+0000","last_unstale":"2023-10-09T08:52:48.444009+0000","last_undegraded":"2023-10-09T08:52:48.444009+0000","last_fullsized":"2023-10-09T08:52:48.444009+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:41.880170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.444886+0000","last_change":"2023-10-09T08:51:53.456605+0000","last_active":"2023-10-09T08:52:48.444886+0000","last_peered":"2023-10-09T08:52:48.444886+0000","last_clean":"2023-10-09T08:52:48.444886+0000","last_became_active":"2023-10-09T08:51:53.456254+0000","last_became_peered":"2023-10-09T08:51:53.456254+0000","last_unstale":"2023-10-09T08:52:48.444886+0000","last_undegraded":"2023-10-09T08:52:48.444886+0000","last_fullsized":"2023-10-09T08:52:48.444886+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:22.564027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":82,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.473728+0000","last_change":"2023-10-09T08:46:18.677157+0000","last_active":"2023-10-09T08:52:48.473728+0000","last_peered":"2023-10-09T08:52:48.473728+0000","last_clean":"2023-10-09T08:52:48.473728+0000","last_became_active":"2023-10-09T08:46:18.676750+0000","last_became_peered":"2023-10-09T08:46:18.676750+0000","last_unstale":"2023-10-09T08:52:48.473728+0000","last_undegraded":"2023-10-09T08:52:48.473728+0000","last_fullsized":"2023-10-09T08:52:48.473728+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":94,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.458170+0000","last_change":"2023-10-09T08:47:16.968742+0000","last_active":"2023-10-09T08:52:48.458170+0000","last_peered":"2023-10-09T08:52:48.458170+0000","last_clean":"2023-10-09T08:52:48.458170+0000","last_became_active":"2023-10-09T08:47:16.968317+0000","last_became_peered":"2023-10-09T08:47:16.968317+0000","last_unstale":"2023-10-09T08:52:48.458170+0000","last_undegraded":"2023-10-09T08:52:48.458170+0000","last_fullsized":"2023-10-09T08:52:48.458170+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":67,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.470060+0000","last_change":"2023-10-09T08:34:27.739113+0000","last_active":"2023-10-09T08:52:48.470060+0000","last_peered":"2023-10-09T08:52:48.470060+0000","last_clean":"2023-10-09T08:52:48.470060+0000","last_became_active":"2023-10-09T08:34:27.738645+0000","last_became_peered":"2023-10-09T08:34:27.738645+0000","last_unstale":"2023-10-09T08:52:48.470060+0000","last_undegraded":"2023-10-09T08:52:48.470060+0000","last_fullsized":"2023-10-09T08:52:48.470060+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.458627+0000","last_change":"2023-10-09T08:51:33.443239+0000","last_active":"2023-10-09T08:52:48.458627+0000","last_peered":"2023-10-09T08:52:48.458627+0000","last_clean":"2023-10-09T08:52:48.458627+0000","last_became_active":"2023-10-09T08:51:33.442063+0000","last_became_peered":"2023-10-09T08:51:33.442063+0000","last_unstale":"2023-10-09T08:52:48.458627+0000","last_undegraded":"2023-10-09T08:52:48.458627+0000","last_fullsized":"2023-10-09T08:52:48.458627+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":89,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.464641+0000","last_change":"2023-10-09T08:47:16.970132+0000","last_active":"2023-10-09T08:52:48.464641+0000","last_peered":"2023-10-09T08:52:48.464641+0000","last_clean":"2023-10-09T08:52:48.464641+0000","last_became_active":"2023-10-09T08:47:16.969760+0000","last_became_peered":"2023-10-09T08:47:16.969760+0000","last_unstale":"2023-10-09T08:52:48.464641+0000","last_undegraded":"2023-10-09T08:52:48.464641+0000","last_fullsized":"2023-10-09T08:52:48.464641+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.801830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":89,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.458734+0000","last_change":"2023-10-09T08:51:53.455827+0000","last_active":"2023-10-09T08:52:48.458734+0000","last_peered":"2023-10-09T08:52:48.458734+0000","last_clean":"2023-10-09T08:52:48.458734+0000","last_became_active":"2023-10-09T08:51:53.455328+0000","last_became_peered":"2023-10-09T08:51:53.455328+0000","last_unstale":"2023-10-09T08:52:48.458734+0000","last_undegraded":"2023-10-09T08:52:48.458734+0000","last_fullsized":"2023-10-09T08:52:48.458734+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:46.784739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":92,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.443500+0000","last_change":"2023-10-09T08:51:33.445775+0000","last_active":"2023-10-09T08:52:48.443500+0000","last_peered":"2023-10-09T08:52:48.443500+0000","last_clean":"2023-10-09T08:52:48.443500+0000","last_became_active":"2023-10-09T08:51:33.440135+0000","last_became_peered":"2023-10-09T08:51:33.440135+0000","last_unstale":"2023-10-09T08:52:48.443500+0000","last_undegraded":"2023-10-09T08:52:48.443500+0000","last_fullsized":"2023-10-09T08:52:48.443500+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:02.419868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T08:52:48.457380+0000","last_change":"2023-10-09T08:46:18.676602+0000","last_active":"2023-10-09T08:52:48.457380+0000","last_peered":"2023-10-09T08:52:48.457380+0000","last_clean":"2023-10-09T08:52:48.457380+0000","last_became_active":"2023-10-09T08:46:18.676122+0000","last_became_peered":"2023-10-09T08:46:18.676122+0000","last_unstale":"2023-10-09T08:52:48.457380+0000","last_undegraded":"2023-10-09T08:52:48.457380+0000","last_fullsized":"2023-10-09T08:52:48.457380+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4368105496,"num_objects":66653,"num_object_clones":0,"num_object_copies":199959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24705,"num_objects_misplaced":2210,"num_objects_unfound":0,"num_objects_dirty":66653,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135597,"num_write_kb":6471937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2829,"num_bytes_recovered":185401344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13949743104,"data_stored":13949730888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128397,"ondisk_log_size":128397,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430472,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2029136,"kb_used_data":1731912,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91716016,"statfs":{"total":95995035648,"available":93917200384,"internally_reserved":0,"allocated":1773477888,"data_stored":1773017174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,2,5,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:52:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5369999999999999,"5min":2.1299999999999999,"15min":1.3260000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":9.9719999999999995,"5min":64.768000000000001,"15min":64.768000000000001},"last":9.718},{"interface":"front","average":{"1min":2.5710000000000002,"5min":2.1440000000000001,"15min":1.347},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.205,"5min":64.742999999999995,"15min":64.742999999999995},"last":0.61299999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":1.2,"15min":1.0580000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.038,"5min":10.038,"15min":10.038},"last":9.5909999999999993},{"interface":"front","average":{"1min":2.5390000000000001,"5min":1.292,"15min":1.079},"min":{"1min":0.39000000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.657,"5min":10.657,"15min":10.657},"last":0.57899999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:52:57 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":1.2829999999999999,"15min":1.0469999999999999},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.407,"5min":14.407,"15min":14.407},"last":0.78600000000000003},{"interface":"front","average":{"1min":2.681,"5min":1.409,"15min":1.091},"min":{"1min":0.42799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.327,"5min":10.327,"15min":10.327},"last":9.6140000000000008}]},{"osd":3,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":2.2160000000000002,"15min":1.3520000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":10.694000000000001,"5min":67.224000000000004,"15min":67.224000000000004},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.802,"5min":1.9910000000000001,"15min":1.339},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":10.356999999999999,"5min":66.471000000000004,"15min":66.471000000000004},"last":9.6890000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:52:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.93200000000000005,"15min":0.88500000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.847,"5min":3.847,"15min":3.847},"last":0.71499999999999997},{"interface":"front","average":{"1min":2.226,"5min":1.3049999999999999,"15min":1.048},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.2300000000000004,"5min":9.2300000000000004,"15min":9.2300000000000004},"last":9.516}]},{"osd":5,"last update":"Mon Oct 9 08:52:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.232,"15min":1.0589999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":4.8150000000000004,"5min":4.8150000000000004,"15min":4.8150000000000004},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.29,"5min":1.1539999999999999,"15min":1.042},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.7949999999999999,"5min":3.8100000000000001,"15min":3.8100000000000001},"last":0.753}]},{"osd":6,"last update":"Mon Oct 9 08:52:18 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.2490000000000001,"15min":1.091},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.3810000000000002,"5min":9.3810000000000002,"15min":9.3810000000000002},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.3280000000000001,"15min":1.1479999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":5.7130000000000001,"5min":5.7130000000000001,"15min":5.7130000000000001},"last":9.5630000000000006}]}]},{"osd":6,"up_from":42,"seq":180388626705,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1932552,"kb_used_data":1635328,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91812600,"statfs":{"total":95995035648,"available":94016102400,"internally_reserved":0,"allocated":1674575872,"data_stored":1674123708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1,4,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:53:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9690000000000001,"5min":1.2470000000000001,"15min":1.034},"min":{"1min":0.30399999999999999,"5min":0.249,"15min":0.249},"max":{"1min":5.3780000000000001,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.251},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.1080000000000001,"15min":1.0309999999999999},"min":{"1min":0.46300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.5459999999999998,"5min":10.776,"15min":10.776},"last":1.1839999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.121,"15min":0.97099999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.0949999999999998,"5min":8.6880000000000006,"15min":8.6880000000000006},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.7189999999999999,"5min":1.3120000000000001,"15min":1.0780000000000001},"min":{"1min":0.379,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":15.23,"5min":15.23,"15min":15.23},"last":1.2509999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:53:01 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.3340000000000001,"15min":1.056},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.4470000000000001,"5min":7.4470000000000001,"15min":7.4470000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.671,"5min":1.1910000000000001,"15min":1.0209999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.747,"5min":10.747,"15min":10.747},"last":1.2210000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:52:26 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.9399999999999999,"15min":1.2529999999999999},"min":{"1min":0.374,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.4730000000000008,"5min":57.636000000000003,"15min":57.636000000000003},"last":0.434},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.677,"15min":1.1910000000000001},"min":{"1min":0.39000000000000001,"5min":0.374,"15min":0.374},"max":{"1min":5.3659999999999997,"5min":58.393000000000001,"15min":58.393000000000001},"last":0.495}]},{"osd":4,"last update":"Mon Oct 9 08:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":1.22,"15min":1.0049999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.379},"max":{"1min":10.625,"5min":10.625,"15min":10.625},"last":1.3260000000000001},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.169,"15min":1.016},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.5739999999999998,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":0.85299999999999998}]},{"osd":5,"last update":"Mon Oct 9 08:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.25,"15min":1.0600000000000001},"min":{"1min":0.438,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":12.404999999999999,"5min":12.404999999999999,"15min":12.404999999999999},"last":1.3009999999999999},{"interface":"front","average":{"1min":1.669,"5min":1.1899999999999999,"15min":1.0860000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.734,"5min":6.7910000000000004,"15min":6.7910000000000004},"last":0.67500000000000004}]},{"osd":7,"last update":"Mon Oct 9 08:53:04 2023","interfaces":[{"interface":"back","average":{"1min":2.794,"5min":1.522,"15min":1.198},"min":{"1min":0.45600000000000002,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.1630000000000003,"5min":10.678000000000001,"15min":10.678000000000001},"last":1.278},{"interface":"front","average":{"1min":2.5910000000000002,"5min":1.448,"15min":1.1950000000000001},"min":{"1min":0.501,"5min":0.377,"15min":0.377},"max":{"1min":6.173,"5min":6.173,"15min":6.173},"last":0.53800000000000003}]}]},{"osd":1,"up_from":83,"seq":356482285653,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1405276,"kb_used_data":1329428,"kb_used_omap":9,"kb_used_meta":75830,"kb_avail":92339876,"statfs":{"total":95995035648,"available":94556033024,"internally_reserved":0,"allocated":1361334272,"data_stored":1360805178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":77650921},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":0.89400000000000002,"15min":0.71399999999999997},"min":{"1min":0.218,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.3979999999999997,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.85699999999999998,"15min":0.73299999999999998},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.4420000000000002,"5min":6.4420000000000002,"15min":6.4420000000000002},"last":0.53500000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.008,"15min":0.83399999999999996},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.6539999999999999,"5min":7.468,"15min":7.468},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.1299999999999999,"5min":0.997,"15min":0.80300000000000005},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.0289999999999999,"5min":6.2800000000000002,"15min":6.2800000000000002},"last":1.788}]},{"osd":3,"last update":"Mon Oct 9 08:52:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":1.3100000000000001,"15min":1.0089999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.01,"5min":13.01,"15min":13.01},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.028,"5min":1.2250000000000001,"15min":0.97799999999999998},"min":{"1min":0.40899999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5800000000000001,"5min":8.5800000000000001,"15min":8.5800000000000001},"last":0.68400000000000005}]},{"osd":4,"last update":"Mon Oct 9 08:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7150000000000001,"5min":1.1240000000000001,"15min":1.0169999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.3970000000000002,"5min":7.3970000000000002,"15min":7.3970000000000002},"last":0.76100000000000001},{"interface":"front","average":{"1min":2.121,"5min":1.26,"15min":1.0660000000000001},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.25,"5min":7.25,"15min":7.25},"last":0.747}]},{"osd":5,"last update":"Mon Oct 9 08:52:30 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.4099999999999999,"15min":1.1819999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":7.4610000000000003,"5min":7.4610000000000003,"15min":7.4610000000000003},"last":1.8440000000000001},{"interface":"front","average":{"1min":2.5899999999999999,"5min":1.353,"15min":1.165},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.2759999999999998,"5min":7.2759999999999998,"15min":7.2759999999999998},"last":1.8340000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:52:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6619999999999999,"5min":1.4399999999999999,"15min":1.202},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":8.0579999999999998,"5min":8.0579999999999998,"15min":8.0579999999999998},"last":0.91500000000000004},{"interface":"front","average":{"1min":3.2170000000000001,"5min":1.5820000000000001,"15min":1.266},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":8.077,"5min":8.077,"15min":8.077},"last":0.93300000000000005}]},{"osd":7,"last update":"Mon Oct 9 08:52:30 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.302,"15min":1.1240000000000001},"min":{"1min":0.55000000000000004,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":7.2080000000000002,"5min":7.2080000000000002,"15min":7.2080000000000002},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.7410000000000001,"5min":1.2869999999999999,"15min":1.167},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":7.1710000000000003,"5min":7.1710000000000003,"15min":7.1710000000000003},"last":0.78600000000000003}]}]},{"osd":0,"up_from":78,"seq":335007449179,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1108464,"kb_used_data":1044840,"kb_used_omap":28,"kb_used_meta":63587,"kb_avail":92636688,"statfs":{"total":95995035648,"available":94859968512,"internally_reserved":0,"allocated":1069916160,"data_stored":1069369146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":65113936},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:52:32 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.194,"15min":0.877},"min":{"1min":0.26300000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.7709999999999999,"5min":37.380000000000003,"15min":37.380000000000003},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.1910000000000001,"15min":0.92200000000000004},"min":{"1min":0.31,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.2510000000000003,"5min":38.012999999999998,"15min":38.012999999999998},"last":1.0149999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:53:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.93999999999999995,"15min":0.88400000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.032,"5min":6.032,"15min":6.032},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0109999999999999,"15min":0.97899999999999998},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.8230000000000004,"5min":5.4089999999999998,"15min":5.4089999999999998},"last":0.81899999999999995}]},{"osd":3,"last update":"Mon Oct 9 08:52:22 2023","interfaces":[{"interface":"back","average":{"1min":6.8289999999999997,"5min":3.0049999999999999,"15min":1.5349999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":110.988,"5min":110.988,"15min":110.988},"last":1.8819999999999999},{"interface":"front","average":{"1min":6.7009999999999996,"5min":2.9910000000000001,"15min":1.5509999999999999},"min":{"1min":0.371,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":107.66500000000001,"5min":107.66500000000001,"15min":107.66500000000001},"last":0.74399999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.093,"15min":1.0620000000000001},"min":{"1min":0.38400000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.1299999999999999,"5min":6.1299999999999999,"15min":6.1299999999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.145,"15min":1.026},"min":{"1min":0.45200000000000001,"5min":0.371,"15min":0.371},"max":{"1min":5.6660000000000004,"5min":6.5700000000000003,"15min":6.5700000000000003},"last":6.2539999999999996}]},{"osd":5,"last update":"Mon Oct 9 08:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.109,"15min":1.05},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.9710000000000001,"5min":7.9710000000000001,"15min":7.9710000000000001},"last":0.497},{"interface":"front","average":{"1min":1.903,"5min":1.278,"15min":1.1739999999999999},"min":{"1min":0.442,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":4.6029999999999998,"5min":5.4980000000000002,"15min":5.4980000000000002},"last":6.1550000000000002}]},{"osd":6,"last update":"Mon Oct 9 08:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.3080000000000001,"15min":1.2130000000000001},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":5.9420000000000002,"5min":5.9420000000000002,"15min":5.9420000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.0430000000000001,"5min":1.329,"15min":1.26},"min":{"1min":0.45300000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":5.7569999999999997,"5min":5.7569999999999997,"15min":5.7569999999999997},"last":0.79500000000000004}]},{"osd":7,"last update":"Mon Oct 9 08:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.3100000000000001,"15min":1.2629999999999999},"min":{"1min":0.46700000000000003,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":7.1900000000000004,"5min":7.1900000000000004,"15min":7.1900000000000004},"last":6.1079999999999997},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.4510000000000001,"15min":1.2529999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":8.2919999999999998,"5min":8.2919999999999998,"15min":8.2919999999999998},"last":2.2480000000000002}]}]},{"osd":2,"up_from":88,"seq":377957122127,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2335632,"kb_used_data":2227592,"kb_used_omap":81,"kb_used_meta":107886,"kb_avail":91409520,"statfs":{"total":95995035648,"available":93603348480,"internally_reserved":0,"allocated":2281054208,"data_stored":2280535755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":110475482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:53:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":0.95899999999999996,"15min":0.78500000000000003},"min":{"1min":0.223,"5min":0.215,"15min":0.215},"max":{"1min":4.452,"5min":4.452,"15min":4.452},"last":3.4049999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.85199999999999998,"15min":0.82999999999999996},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.0229999999999999,"5min":7.2450000000000001,"15min":7.2450000000000001},"last":0.48899999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:53:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.022,"15min":0.86799999999999999},"min":{"1min":0.25700000000000001,"5min":0.249,"15min":0.249},"max":{"1min":3.9780000000000002,"5min":11.728,"15min":11.728},"last":3.4609999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.1080000000000001,"15min":0.93400000000000005},"min":{"1min":0.40000000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":3.536,"5min":8.266,"15min":8.266},"last":1.0069999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.0640000000000001,"15min":0.90400000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.3019999999999996,"5min":7.3019999999999996,"15min":7.3019999999999996},"last":0.70999999999999996},{"interface":"front","average":{"1min":3.7469999999999999,"5min":1.635,"15min":1.085},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":45.512999999999998,"5min":45.512999999999998,"15min":45.512999999999998},"last":0.30599999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:53:00 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.526,"15min":1.1579999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.5640000000000001,"5min":9.5640000000000001,"15min":9.5640000000000001},"last":0.379},{"interface":"front","average":{"1min":1.764,"5min":1.244,"15min":1.1240000000000001},"min":{"1min":0.377,"5min":0.314,"15min":0.314},"max":{"1min":4.0279999999999996,"5min":4.0279999999999996,"15min":4.0279999999999996},"last":3.5059999999999998}]},{"osd":5,"last update":"Mon Oct 9 08:53:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.3029999999999999,"15min":1.2150000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.556,"5min":7.3419999999999996,"15min":7.3419999999999996},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.5609999999999999,"15min":1.2410000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":9.6080000000000005,"5min":9.6080000000000005,"15min":9.6080000000000005},"last":0.41499999999999998}]},{"osd":6,"last update":"Mon Oct 9 08:53:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.3620000000000001,"15min":1.216},"min":{"1min":0.40799999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.2549999999999999,"5min":5.2549999999999999,"15min":5.2549999999999999},"last":4.617},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.401,"15min":1.2749999999999999},"min":{"1min":0.46700000000000003,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":4.6630000000000003,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":7.9660000000000002}]},{"osd":7,"last update":"Mon Oct 9 08:53:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.4770000000000001,"15min":1.268},"min":{"1min":0.44,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":7.6440000000000001,"5min":7.6440000000000001,"15min":7.6440000000000001},"last":4.2460000000000004},{"interface":"front","average":{"1min":1.7649999999999999,"5min":1.3939999999999999,"15min":1.2949999999999999},"min":{"1min":0.55100000000000005,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":4.3490000000000002,"5min":7.415,"15min":7.415},"last":0.68300000000000005}]}]},{"osd":3,"up_from":93,"seq":399431958601,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3002368,"kb_used_data":2898616,"kb_used_omap":7,"kb_used_meta":103736,"kb_avail":90742784,"statfs":{"total":95995035648,"available":92920610816,"internally_reserved":0,"allocated":2968182784,"data_stored":2967677738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":106226280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.2569999999999999,"15min":0.90800000000000003},"min":{"1min":0.25800000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.916,"5min":13.916,"15min":13.916},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.071,"5min":1.0580000000000001,"15min":0.88200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.6099999999999999,"5min":7.5919999999999996,"15min":7.5919999999999996},"last":0.439}]},{"osd":1,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.2,"15min":0.92600000000000005},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.3010000000000002,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.173,"5min":1.323,"15min":0.98199999999999998},"min":{"1min":0.35499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":8.8209999999999997,"5min":12.564,"15min":12.564},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.5509999999999999,"15min":1.032},"min":{"1min":0.28899999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.1059999999999999,"5min":28.655000000000001,"15min":28.655000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.3820000000000001,"5min":1.7330000000000001,"15min":1.121},"min":{"1min":0.41899999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.3399999999999999,"5min":30.785,"15min":30.785},"last":0.312}]},{"osd":4,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4700000000000002,"5min":1.544,"15min":1.194},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":8.8699999999999992,"5min":12.877000000000001,"15min":12.877000000000001},"last":4.4989999999999997},{"interface":"front","average":{"1min":2.0979999999999999,"5min":1.429,"15min":1.169},"min":{"1min":0.376,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.4659999999999993,"5min":9.4659999999999993,"15min":9.4659999999999993},"last":4.4139999999999997}]},{"osd":5,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.319,"15min":1.129},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":7.3390000000000004,"5min":8.0050000000000008,"15min":8.0050000000000008},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.04,"5min":1.472,"15min":1.2090000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":8.6419999999999995,"5min":12.204000000000001,"15min":12.204000000000001},"last":1.9370000000000001}]},{"osd":6,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4219999999999999,"15min":1.2270000000000001},"min":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":4.2800000000000002,"5min":10.048,"15min":10.048},"last":1.4430000000000001},{"interface":"front","average":{"1min":2.0430000000000001,"5min":1.528,"15min":1.3120000000000001},"min":{"1min":0.40500000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":6.3369999999999997,"5min":12.275,"15min":12.275},"last":1.452}]},{"osd":7,"last update":"Mon Oct 9 08:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":1.55,"15min":1.292},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":10.284000000000001,"5min":12.244,"15min":12.244},"last":0.79200000000000004},{"interface":"front","average":{"1min":2.9350000000000001,"5min":1.74,"15min":1.3580000000000001},"min":{"1min":0.51400000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":13.339,"5min":13.339,"15min":13.339},"last":0.70699999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019171,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1664332,"kb_used_data":1354756,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92080820,"statfs":{"total":95995035648,"available":94290759680,"internally_reserved":0,"allocated":1387270144,"data_stored":1386813177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:53:05 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":1.49,"15min":1.079},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.8700000000000001,"5min":22.815000000000001,"15min":22.815000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.2350000000000001,"15min":1.01},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6029999999999998,"5min":22.843,"15min":22.843},"last":3.754}]},{"osd":1,"last update":"Mon Oct 9 08:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":1.0700000000000001,"15min":0.92300000000000004},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.9169999999999998,"5min":8.9169999999999998,"15min":8.9169999999999998},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.97,"5min":1.147,"15min":1.0209999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":6.2169999999999996,"5min":6.2169999999999996,"15min":6.2169999999999996},"last":0.54100000000000004}]},{"osd":2,"last update":"Mon Oct 9 08:53:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1739999999999999,"5min":1.6299999999999999,"15min":1.167},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.9109999999999996,"5min":15.163,"15min":15.163},"last":0.60099999999999998},{"interface":"front","average":{"1min":2.323,"5min":1.2929999999999999,"15min":1.0649999999999999},"min":{"1min":0.36099999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.2129999999999992,"5min":8.2129999999999992,"15min":8.2129999999999992},"last":3.6819999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.214,"5min":1.292,"15min":1.0920000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.5969999999999995,"5min":9.5969999999999995,"15min":9.5969999999999995},"last":1.012},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.2270000000000001,"15min":1.097},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.5110000000000001,"5min":5.5110000000000001,"15min":5.5110000000000001},"last":3.8010000000000002}]},{"osd":5,"last update":"Mon Oct 9 08:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":1.143,"15min":0.97299999999999998},"min":{"1min":0.377,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.9130000000000003,"5min":5.9130000000000003,"15min":5.9130000000000003},"last":0.623},{"interface":"front","average":{"1min":1.47,"5min":1.111,"15min":1.016},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.218,"5min":4.218,"15min":4.218},"last":3.778}]},{"osd":6,"last update":"Mon Oct 9 08:52:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":1.29,"15min":1.0509999999999999},"min":{"1min":0.40000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.0060000000000002,"5min":9.0060000000000002,"15min":9.0060000000000002},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.202,"15min":1.089},"min":{"1min":0.39200000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":0.64700000000000002}]},{"osd":7,"last update":"Mon Oct 9 08:53:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4359999999999999,"5min":1.395,"15min":1.1679999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.8369999999999997,"5min":6.8369999999999997,"15min":6.8369999999999997},"last":3.7269999999999999},{"interface":"front","average":{"1min":3.6880000000000002,"5min":1.871,"15min":1.335},"min":{"1min":0.51300000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":13.321999999999999,"5min":13.911,"15min":13.911},"last":0.99099999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822939,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761408,"kb_used_data":1451832,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91983744,"statfs":{"total":95995035648,"available":94191353856,"internally_reserved":0,"allocated":1486675968,"data_stored":1486227057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:53:01 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":1.2749999999999999,"15min":1.01},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.5599999999999996,"5min":6.5599999999999996,"15min":6.5599999999999996},"last":6.2729999999999997},{"interface":"front","average":{"1min":1.536,"5min":1.0629999999999999,"15min":0.95999999999999996},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.7949999999999999,"5min":7.7949999999999999,"15min":7.7949999999999999},"last":0.443}]},{"osd":1,"last update":"Mon Oct 9 08:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.091,"15min":0.97099999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.226,"5min":6.4009999999999998,"15min":6.4009999999999998},"last":6.3129999999999997},{"interface":"front","average":{"1min":3.8969999999999998,"5min":1.5129999999999999,"15min":1.115},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":35.042000000000002,"5min":35.042000000000002,"15min":35.042000000000002},"last":6.048}]},{"osd":2,"last update":"Mon Oct 9 08:53:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":1.3420000000000001,"15min":1.0680000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.7949999999999999,"5min":8.7949999999999999,"15min":8.7949999999999999},"last":6.3499999999999996},{"interface":"front","average":{"1min":1.512,"5min":1.1339999999999999,"15min":1.046},"min":{"1min":0.38200000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.8719999999999999,"5min":7.8719999999999999,"15min":7.8719999999999999},"last":0.41699999999999998}]},{"osd":3,"last update":"Mon Oct 9 08:52:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.079,"15min":1.03},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":5.7539999999999996,"5min":5.7539999999999996,"15min":5.7539999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.363,"5min":1.2410000000000001,"15min":1.0489999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.9509999999999996,"5min":7.9509999999999996,"15min":7.9509999999999996},"last":6.0800000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:52:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":1.1439999999999999,"15min":0.94999999999999996},"min":{"1min":0.38200000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.4279999999999999,"5min":5.4279999999999999,"15min":5.4279999999999999},"last":6.1399999999999997},{"interface":"front","average":{"1min":2.056,"5min":1.218,"15min":1.034},"min":{"1min":0.44400000000000001,"5min":0.375,"15min":0.375},"max":{"1min":7.8339999999999996,"5min":7.8339999999999996,"15min":7.8339999999999996},"last":3.2719999999999998}]},{"osd":6,"last update":"Mon Oct 9 08:52:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1880000000000002,"5min":1.22,"15min":1.0509999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.1129999999999995,"5min":8.1129999999999995,"15min":8.1129999999999995},"last":6.1580000000000004},{"interface":"front","average":{"1min":1.4319999999999999,"5min":1.161,"15min":1.0649999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.40400000000000003},"max":{"1min":5.2910000000000004,"5min":5.2910000000000004,"15min":5.2910000000000004},"last":0.39100000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:52:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":1.4490000000000001,"15min":1.171},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":15.849,"5min":15.849,"15min":15.849},"last":0.35199999999999998},{"interface":"front","average":{"1min":3.258,"5min":1.5129999999999999,"15min":1.2250000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":12.346,"5min":12.346,"15min":12.346},"last":6.125}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1067847680,"data_stored":1067843608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1349390336,"data_stored":1349386264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2279735296,"data_stored":2279735296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2962292736,"data_stored":2962292736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1380253696,"data_stored":1380253696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1474564096,"data_stored":1474560024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1668022272,"data_stored":1668022272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1767636992,"data_stored":1767636992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:53:15.024 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T08:53:15.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-09T08:53:15.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:15.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:15.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:15.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:15.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:15.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:15.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:15.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:15.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:15.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:15.431 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:15.431 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:15.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:15.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:15.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:15.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:15.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:15.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:15.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:15.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:15.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:15.818 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:15.867 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:15.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:15.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:15.970 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:16.116 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:16.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:16.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:16.250 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:53:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:53:15] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.33.4" 2023-10-09T08:53:16.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:16.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:16.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:16.661 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:16 smithi094 bash[59677]: cluster 2023-10-09T08:53:15.381081+0000 mgr.y (mgr.25421) 469 : cluster 0 pgmap v415: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 24705/195900 objects degraded (12.611%); 2210/195900 objects misplaced (1.128%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:16.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:16.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:16.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:16 smithi016 bash[75179]: cluster 2023-10-09T08:53:15.381081+0000 mgr.y (mgr.25421) 469 : cluster 0 pgmap v415: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 24705/195900 objects degraded (12.611%); 2210/195900 objects misplaced (1.128%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:16.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:16 smithi016 bash[77892]: cluster 2023-10-09T08:53:15.381081+0000 mgr.y (mgr.25421) 469 : cluster 0 pgmap v415: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 24705/195900 objects degraded (12.611%); 2210/195900 objects misplaced (1.128%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:16.788 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:16.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:16.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:16.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:16.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:16.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:16.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:17.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:17.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:17.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:17.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:17.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:17.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:17.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:17.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:17.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:17.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:17.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:17.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:17.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:17.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:17.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:17.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:17.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:17.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:17.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:17.808 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:17.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:17.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:17.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:17.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:17.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:17.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:17.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:18.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:18.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:18.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:18.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:18.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:18.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:18.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:18.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:18.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:18.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:18.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:18.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:18.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:18.617 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:18.643 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:18.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:18 smithi094 bash[59677]: cluster 2023-10-09T08:53:17.381804+0000 mgr.y (mgr.25421) 470 : cluster 0 pgmap v416: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 24705/191085 objects degraded (12.929%); 2044/191085 objects misplaced (1.070%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:18.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:18 smithi016 bash[75179]: cluster 2023-10-09T08:53:17.381804+0000 mgr.y (mgr.25421) 470 : cluster 0 pgmap v416: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 24705/191085 objects degraded (12.929%); 2044/191085 objects misplaced (1.070%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:18 smithi016 bash[77892]: cluster 2023-10-09T08:53:17.381804+0000 mgr.y (mgr.25421) 470 : cluster 0 pgmap v416: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 24705/191085 objects degraded (12.929%); 2044/191085 objects misplaced (1.070%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:18.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:18.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:18.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:18.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:18.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:18.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:19.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:19.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:19.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:19.136 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:19.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:19.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:19.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:19.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:19.340 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:19.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:19.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:19.443 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:19.447 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:19 smithi094 bash[59677]: cluster 2023-10-09T08:53:18.943207+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 24705/191085 objects degraded (12.929%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:19.447 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:19 smithi094 bash[59677]: audit 2023-10-09T08:53:19.217379+0000 mon.a (mon.0) 616 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:19.447 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:19 smithi094 bash[59677]: audit 2023-10-09T08:53:19.218579+0000 mon.a (mon.0) 617 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:19.474 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:19.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:19.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:19.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:19.608 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:19.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:19.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:19.722 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:19 smithi016 bash[75179]: cluster 2023-10-09T08:53:18.943207+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 24705/191085 objects degraded (12.929%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:19.722 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:19 smithi016 bash[75179]: audit 2023-10-09T08:53:19.217379+0000 mon.a (mon.0) 616 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:19.722 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:19 smithi016 bash[75179]: audit 2023-10-09T08:53:19.218579+0000 mon.a (mon.0) 617 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:19.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:19 smithi016 bash[77892]: cluster 2023-10-09T08:53:18.943207+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 24705/191085 objects degraded (12.929%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:19.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:19 smithi016 bash[77892]: audit 2023-10-09T08:53:19.217379+0000 mon.a (mon.0) 616 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:19.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:19 smithi016 bash[77892]: audit 2023-10-09T08:53:19.218579+0000 mon.a (mon.0) 617 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:19.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:19.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:19.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:19.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:19.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:19.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:19.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:20.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:20.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:20.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:20.133 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:20.133 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:20.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:20.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:20.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:20.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:20.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:20.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:20.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:20.516 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:20.560 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:20.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:20.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:20.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:20.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:20.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:20.944 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:20.974 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:20.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:21.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:21.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:21.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:21.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:21.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:21.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:21.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:21.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:21.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:21.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:21.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:21.492 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:21.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:21 smithi094 bash[59677]: cluster 2023-10-09T08:53:19.385842+0000 mgr.y (mgr.25421) 471 : cluster 0 pgmap v417: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24705/190536 objects degraded (12.966%); 2044/190536 objects misplaced (1.073%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:21.526 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:21.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:21.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:21.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:21 smithi016 bash[75179]: cluster 2023-10-09T08:53:19.385842+0000 mgr.y (mgr.25421) 471 : cluster 0 pgmap v417: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24705/190536 objects degraded (12.966%); 2044/190536 objects misplaced (1.073%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:21.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:21 smithi016 bash[77892]: cluster 2023-10-09T08:53:19.385842+0000 mgr.y (mgr.25421) 471 : cluster 0 pgmap v417: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24705/190536 objects degraded (12.966%); 2044/190536 objects misplaced (1.073%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T08:53:21.633 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:21.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:21.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:21.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:21.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:21.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:21.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:21.926 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:21.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:21.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:22.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:22.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:22.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:22.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:22.166 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:22.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:22.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:22.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:22.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:22.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:22.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:22.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:22.459 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:22.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:22.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:22.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:22.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:22.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:22.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:22.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:22.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:22.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:22.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:22.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:22.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:23.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:23.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:23.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:23.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:23.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:23.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:23.253 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:23.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:23.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:23.382 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:23.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:23.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:23 smithi094 bash[59677]: cluster 2023-10-09T08:53:21.386489+0000 mgr.y (mgr.25421) 472 : cluster 0 pgmap v418: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 24705/184209 objects degraded (13.411%); 2044/184209 objects misplaced (1.110%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:53:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:23 smithi016 bash[77892]: cluster 2023-10-09T08:53:21.386489+0000 mgr.y (mgr.25421) 472 : cluster 0 pgmap v418: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 24705/184209 objects degraded (13.411%); 2044/184209 objects misplaced (1.110%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:53:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:23 smithi016 bash[75179]: cluster 2023-10-09T08:53:21.386489+0000 mgr.y (mgr.25421) 472 : cluster 0 pgmap v418: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 24705/184209 objects degraded (13.411%); 2044/184209 objects misplaced (1.110%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T08:53:23.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:23.558 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:23.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:23.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:23.663 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:23.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:23.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:23.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:23.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:23.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:23.863 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:53:23.896 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:143577 6.f e113/95 6:f32406ff:::benchmark_data_smithi016_105291_object42450:head v 113'8936, mlcod=113'8933)", 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:23.872333+0000", 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.018562426, 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018589722999999999, 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:23.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872333+0000", 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872333+0000", 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872331+0000", 2023-10-09T08:53:23.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872341+0000", 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2410000000000001e-06 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872466+0000", 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000124941 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872478+0000", 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2510999999999999e-05 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872708+0000", 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00022956300000000001 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:23.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.872768+0000", 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0359000000000003e-05 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:143576 6.7 e113/95 6:e1e05d72:::benchmark_data_smithi016_105291_object42449:head v 113'8911, mlcod=113'8909)", 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:23.875674+0000", 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.015221281999999999, 2023-10-09T08:53:23.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015311116, 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875674+0000", 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:23.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875674+0000", 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875673+0000", 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875683+0000", 2023-10-09T08:53:23.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0427e-05 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875705+0000", 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1733e-05 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875714+0000", 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4210000000000001e-06 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875874+0000", 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000159643 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.875903+0000", 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9465e-05 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:143579 6.f e113/95 6:f9025dd7:::benchmark_data_smithi016_105291_object42452:head v 113'8937, mlcod=113'8933)", 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:23.878028+0000", 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.012866918, 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012999682, 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:23.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878028+0000", 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878028+0000", 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878027+0000", 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878037+0000", 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0000000000000001e-05 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878054+0000", 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7059999999999999e-05 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:23.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878064+0000", 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0213e-05 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878850+0000", 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00078526599999999996 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.878906+0000", 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6475999999999999e-05 2023-10-09T08:53:23.909 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:143583 6.b 6:dae415a1:::benchmark_data_smithi016_105291_object42456:head [delete] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:23.883938+0000", 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0069570049999999996, 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0071276150000000003, 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:53:23.910 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143583 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.883938+0000", 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:23.911 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.883938+0000", 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.883936+0000", 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.883979+0000", 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3436000000000002e-05 2023-10-09T08:53:23.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.913 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:23.913 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884002+0000", 2023-10-09T08:53:23.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2832000000000001e-05 2023-10-09T08:53:23.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.913 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884013+0000", 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0769e-05 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884074+0000", 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0504000000000002e-05 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:23.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884235+0000", 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000161733 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884370+0000", 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134604 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.890740+0000", 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0063702450000000001 2023-10-09T08:53:23.915 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:143582 6.0 e113/95 6:0e876259:::benchmark_data_smithi016_105291_object42455:head v 113'9255, mlcod=113'9253)", 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:23.884541+0000", 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0063537790000000004, 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0065753169999999998, 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.916 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884541+0000", 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884541+0000", 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884541+0000", 2023-10-09T08:53:23.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884548+0000", 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9330000000000001e-06 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884559+0000", 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0865e-05 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884566+0000", 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9750000000000001e-06 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884872+0000", 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00030551100000000001 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.884927+0000", 2023-10-09T08:53:23.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4965e-05 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:143586 6.7 e113/95)", 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:23.889962+0000", 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.00093292600000000005, 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00118717, 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:53:23.920 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.889962+0000", 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.889962+0000", 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:53:23.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.889961+0000", 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.889973+0000", 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1861999999999999e-05 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.889995+0000", 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.179e-05 2023-10-09T08:53:23.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:23.890005+0000", 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9809999999999998e-06 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 6 2023-10-09T08:53:23.923 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:53:23.924 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:53:23.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:23.956 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:53:23.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:24.005 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:53:24.050 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:53:24.089 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:53:24.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:24.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:24.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:24.135 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:53:24.175 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:53:24.195 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.195 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.195 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.195 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:53:24.232 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:24.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:24.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:24.415 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:24.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:24.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:24 smithi094 bash[59677]: audit 2023-10-09T08:53:23.809758+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.16:0/681186435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:24.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:24 smithi094 bash[59677]: cluster 2023-10-09T08:53:23.944480+0000 mon.a (mon.0) 618 : cluster 3 Health check update: Degraded data redundancy: 24705/184209 objects degraded (13.411%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:24.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:24 smithi016 bash[75179]: audit 2023-10-09T08:53:23.809758+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.16:0/681186435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:24.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:24 smithi016 bash[75179]: cluster 2023-10-09T08:53:23.944480+0000 mon.a (mon.0) 618 : cluster 3 Health check update: Degraded data redundancy: 24705/184209 objects degraded (13.411%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:24.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:24 smithi016 bash[77892]: audit 2023-10-09T08:53:23.809758+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.16:0/681186435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:24.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:24 smithi016 bash[77892]: cluster 2023-10-09T08:53:23.944480+0000 mon.a (mon.0) 618 : cluster 3 Health check update: Degraded data redundancy: 24705/184209 objects degraded (13.411%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:24.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:24.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:24.660 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:24.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:24.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:24.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:24.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:24.968 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:24.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:25.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:25.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:25.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.235 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:25 smithi094 bash[59677]: cluster 2023-10-09T08:53:23.387088+0000 mgr.y (mgr.25421) 473 : cluster 0 pgmap v419: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 24705/181374 objects degraded (13.621%); 1873/181374 objects misplaced (1.033%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T08:53:25.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:25.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:25.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:25.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:25.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:25.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:25.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:25 smithi016 bash[75179]: cluster 2023-10-09T08:53:23.387088+0000 mgr.y (mgr.25421) 473 : cluster 0 pgmap v419: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 24705/181374 objects degraded (13.621%); 1873/181374 objects misplaced (1.033%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T08:53:25.496 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:25 smithi016 bash[77892]: cluster 2023-10-09T08:53:23.387088+0000 mgr.y (mgr.25421) 473 : cluster 0 pgmap v419: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 24705/181374 objects degraded (13.621%); 1873/181374 objects misplaced (1.033%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T08:53:25.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:25.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:25.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:25.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:25.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:53:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:53:25] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.33.4" 2023-10-09T08:53:25.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:25.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:25.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:25.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:25.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:26.019 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.059 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:26.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:26.165 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.204 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:26.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:26.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:26.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:26.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:26.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:26.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:26.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:26.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:26.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:26.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:26.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:26.874 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:26.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:26.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:26.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:27.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:27.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:27.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:27.150 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:27.188 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:27.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:27.238 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:27 smithi016 bash[77892]: cluster 2023-10-09T08:53:25.387814+0000 mgr.y (mgr.25421) 474 : cluster 0 pgmap v420: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 24705/177459 objects degraded (13.922%); 1873/177459 objects misplaced (1.055%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:27.293 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:27.324 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:27.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:27.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:27.433 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:27.441 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:27 smithi094 bash[59677]: cluster 2023-10-09T08:53:25.387814+0000 mgr.y (mgr.25421) 474 : cluster 0 pgmap v420: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 24705/177459 objects degraded (13.922%); 1873/177459 objects misplaced (1.055%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:27.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:27.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:27.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:27 smithi016 bash[75179]: cluster 2023-10-09T08:53:25.387814+0000 mgr.y (mgr.25421) 474 : cluster 0 pgmap v420: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 24705/177459 objects degraded (13.922%); 1873/177459 objects misplaced (1.055%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T08:53:27.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T08:53:27.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:27.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:27.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:27.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:27.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:27.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:27.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:27.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:27.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:27.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:27.986 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:28.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:28.133 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:28.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:28.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:28.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:28.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:28.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:28.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:28.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:28.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:28.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:28.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:28.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:28.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:28.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:28.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:28.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:28.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:29.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:29.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:29.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:29.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:29.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:29.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:29.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:29.377 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:29 smithi094 bash[59677]: cluster 2023-10-09T08:53:27.388581+0000 mgr.y (mgr.25421) 475 : cluster 0 pgmap v421: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24705/173037 objects degraded (14.277%); 1708/173037 objects misplaced (0.987%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:53:29.377 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:29 smithi094 bash[59677]: cluster 2023-10-09T08:53:28.945633+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 24705/173037 objects degraded (14.277%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:29.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:29.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:29.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:29.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:29.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:29 smithi016 bash[75179]: cluster 2023-10-09T08:53:27.388581+0000 mgr.y (mgr.25421) 475 : cluster 0 pgmap v421: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24705/173037 objects degraded (14.277%); 1708/173037 objects misplaced (0.987%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:53:29.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:29 smithi016 bash[75179]: cluster 2023-10-09T08:53:28.945633+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 24705/173037 objects degraded (14.277%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:29.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:29 smithi016 bash[77892]: cluster 2023-10-09T08:53:27.388581+0000 mgr.y (mgr.25421) 475 : cluster 0 pgmap v421: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24705/173037 objects degraded (14.277%); 1708/173037 objects misplaced (0.987%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:53:29.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:29 smithi016 bash[77892]: cluster 2023-10-09T08:53:28.945633+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 24705/173037 objects degraded (14.277%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:29.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:29.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:29.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:29.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:29.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:29.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:53:29.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:29.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:29.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:29.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:29.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:29.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:29.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:30.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T08:53:30.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:30.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:30.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:30.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:30.224 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:30.268 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T08:53:30.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:30.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:30.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:30.384 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:30 smithi094 bash[59677]: audit 2023-10-09T08:53:30.059554+0000 mon.a (mon.0) 620 : audit 1 from='client.? 172.21.15.16:0/2449936328' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T08:53:30.384 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:30 smithi094 bash[59677]: audit 2023-10-09T08:53:30.180617+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.16:0/1171133486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:53:30.384 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:30 smithi094 bash[59677]: audit 2023-10-09T08:53:30.181432+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:53:30.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:30.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:30 smithi016 bash[75179]: audit 2023-10-09T08:53:30.059554+0000 mon.a (mon.0) 620 : audit 1 from='client.? 172.21.15.16:0/2449936328' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T08:53:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:30 smithi016 bash[75179]: audit 2023-10-09T08:53:30.180617+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.16:0/1171133486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:53:30.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:30 smithi016 bash[75179]: audit 2023-10-09T08:53:30.181432+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:53:30.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:30 smithi016 bash[77892]: audit 2023-10-09T08:53:30.059554+0000 mon.a (mon.0) 620 : audit 1 from='client.? 172.21.15.16:0/2449936328' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T08:53:30.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:30 smithi016 bash[77892]: audit 2023-10-09T08:53:30.180617+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.16:0/1171133486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:53:30.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:30 smithi016 bash[77892]: audit 2023-10-09T08:53:30.181432+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:53:30.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:30.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:30.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:30.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:30.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:30.678 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:30.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:30.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:30.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:30.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:30.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:30.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:30.913 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:30.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:30.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:31.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:31.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:31.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:31.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:31.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:31.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:31.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:31.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:31.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:31.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:29.389211+0000 mgr.y (mgr.25421) 476 : cluster 0 pgmap v422: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 24705/172476 objects degraded (14.324%); 1708/172476 objects misplaced (0.990%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:30.240114+0000 mon.a (mon.0) 622 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:30.254422+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: audit 2023-10-09T08:53:30.254558+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:30.254653+0000 mon.a (mon.0) 624 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:30.255153+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:30.256386+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:30.262066+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[75179]: cluster 2023-10-09T08:53:30.272793+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:29.389211+0000 mgr.y (mgr.25421) 476 : cluster 0 pgmap v422: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 24705/172476 objects degraded (14.324%); 1708/172476 objects misplaced (0.990%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:30.240114+0000 mon.a (mon.0) 622 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:30.254422+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: audit 2023-10-09T08:53:30.254558+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:30.254653+0000 mon.a (mon.0) 624 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:30.255153+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:30.256386+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:30.262066+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:31 smithi016 bash[77892]: cluster 2023-10-09T08:53:30.272793+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:31.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:31.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:31.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:29.389211+0000 mgr.y (mgr.25421) 476 : cluster 0 pgmap v422: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 24705/172476 objects degraded (14.324%); 1708/172476 objects misplaced (0.990%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:53:31.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:30.240114+0000 mon.a (mon.0) 622 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:53:31.496 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:30.254422+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: audit 2023-10-09T08:53:30.254558+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:53:31.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:30.254653+0000 mon.a (mon.0) 624 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-10-09T08:53:31.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:30.255153+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:30.256386+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:30.262066+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.497 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:31 smithi094 bash[59677]: cluster 2023-10-09T08:53:30.272793+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T08:53:31.577 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:31.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:31.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:31.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:31.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:31.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:31.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:31.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:31.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:31.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:32.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:32.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:32.107 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.139 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:32.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:32.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T08:53:32.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:32.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:32.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:32.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:32.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:32.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:32.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:32.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:32.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:32.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:32.812 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:32.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:32.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:32.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:32.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:32.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:33.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:33.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:33.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:33.221 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:33.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:33.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:33.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:33.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:33.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:33 smithi094 bash[59677]: cluster 2023-10-09T08:53:31.390051+0000 mgr.y (mgr.25421) 477 : cluster 0 pgmap v424: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 24705/166503 objects degraded (14.838%); 1708/166503 objects misplaced (1.026%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:53:33.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:33.510 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:33 smithi016 bash[75179]: cluster 2023-10-09T08:53:31.390051+0000 mgr.y (mgr.25421) 477 : cluster 0 pgmap v424: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 24705/166503 objects degraded (14.838%); 1708/166503 objects misplaced (1.026%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:53:33.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:33 smithi016 bash[77892]: cluster 2023-10-09T08:53:31.390051+0000 mgr.y (mgr.25421) 477 : cluster 0 pgmap v424: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 24705/166503 objects degraded (14.838%); 1708/166503 objects misplaced (1.026%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T08:53:33.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:33.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:33.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:33.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:33.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:33.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:33.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:33.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:33.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:33.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:33.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:33.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:34.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:34.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:34.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:34.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:34.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:34.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:34.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:34.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:34.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:34 smithi016 bash[75179]: cluster 2023-10-09T08:53:33.946962+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 24705/166503 objects degraded (14.838%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:34.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:34 smithi016 bash[75179]: audit 2023-10-09T08:53:34.218398+0000 mon.a (mon.0) 626 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:34.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:34 smithi016 bash[75179]: audit 2023-10-09T08:53:34.219852+0000 mon.a (mon.0) 627 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:34.327 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:34 smithi016 bash[77892]: cluster 2023-10-09T08:53:33.946962+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 24705/166503 objects degraded (14.838%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:34.327 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:34 smithi016 bash[77892]: audit 2023-10-09T08:53:34.218398+0000 mon.a (mon.0) 626 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:34.327 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:34 smithi016 bash[77892]: audit 2023-10-09T08:53:34.219852+0000 mon.a (mon.0) 627 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:34.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:34.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T08:53:34.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:34.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:34.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:34.506 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:34.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:34.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:34.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:34.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:34.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:34 smithi094 bash[59677]: cluster 2023-10-09T08:53:33.946962+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 24705/166503 objects degraded (14.838%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:34.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:34 smithi094 bash[59677]: audit 2023-10-09T08:53:34.218398+0000 mon.a (mon.0) 626 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:53:34.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:34 smithi094 bash[59677]: audit 2023-10-09T08:53:34.219852+0000 mon.a (mon.0) 627 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:34.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:34.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:34.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:34.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:34.873 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:34.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:34.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:34.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:34.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:35.036 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:35.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:35.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:35.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:35.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:35.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:35.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:35.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:35.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:35.456 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:35.465 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:35 smithi094 bash[59677]: cluster 2023-10-09T08:53:33.390830+0000 mgr.y (mgr.25421) 478 : cluster 0 pgmap v425: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24705/163530 objects degraded (15.107%); 1514/163530 objects misplaced (0.926%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:35.498 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:35 smithi016 bash[75179]: cluster 2023-10-09T08:53:33.390830+0000 mgr.y (mgr.25421) 478 : cluster 0 pgmap v425: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24705/163530 objects degraded (15.107%); 1514/163530 objects misplaced (0.926%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:35 smithi016 bash[77892]: cluster 2023-10-09T08:53:33.390830+0000 mgr.y (mgr.25421) 478 : cluster 0 pgmap v425: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24705/163530 objects degraded (15.107%); 1514/163530 objects misplaced (0.926%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:35.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:35.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:35.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:35.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:35.746 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:35.777 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:35.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:35.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:35.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:35.887 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:35.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:35.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:35.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:36.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:53:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:53:35] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.33.4" 2023-10-09T08:53:36.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:36.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:36.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:36.188 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:36.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:36.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:36.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:36.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:36.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:36.554 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:36.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:36.684 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:36.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:36.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:36.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:36.859 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:36.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:36.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:36.993 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:36.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:37.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:37.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:37.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:37.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:37.229 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:37.268 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:53:37.269 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:53:37.269 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T08:53:37.269 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-10-09T08:53:37.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-10-09T08:53:37.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:37.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:37.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:37.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:37.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:37.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:37.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:37 smithi094 bash[59677]: cluster 2023-10-09T08:53:35.391463+0000 mgr.y (mgr.25421) 479 : cluster 0 pgmap v426: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 24705/161238 objects degraded (15.322%); 1514/161238 objects misplaced (0.939%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:37.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:37.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:37 smithi016 bash[75179]: cluster 2023-10-09T08:53:35.391463+0000 mgr.y (mgr.25421) 479 : cluster 0 pgmap v426: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 24705/161238 objects degraded (15.322%); 1514/161238 objects misplaced (0.939%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:37.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:37 smithi016 bash[77892]: cluster 2023-10-09T08:53:35.391463+0000 mgr.y (mgr.25421) 479 : cluster 0 pgmap v426: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 24705/161238 objects degraded (15.322%); 1514/161238 objects misplaced (0.939%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:37.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:37.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:37.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:37.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:37.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:37.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:37.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:37.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:37.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:37.963 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:37.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:37.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:38.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:38.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:38.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:38.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:38.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:38.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:38.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:38.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:38.345 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:38.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:38.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:38.495 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:38.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:38.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:38.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:38.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:38.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:38.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:38.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:38.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:38.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:38.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:38.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:38.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:38.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:39.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:39.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:39.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:39.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:39.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:39.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:39.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:39.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:39.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:39.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:39.448 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:39.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:39 smithi094 bash[59677]: cluster 2023-10-09T08:53:37.393499+0000 mgr.y (mgr.25421) 480 : cluster 0 pgmap v427: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 24729/154992 objects degraded (15.955%); 1317/154992 objects misplaced (0.850%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T08:53:39.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:39 smithi094 bash[59677]: cluster 2023-10-09T08:53:38.947880+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 24729/154992 objects degraded (15.955%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:39.478 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:39.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:39.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:39.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:39.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:39.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:39.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:39.724 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:39.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:39 smithi016 bash[77892]: cluster 2023-10-09T08:53:37.393499+0000 mgr.y (mgr.25421) 480 : cluster 0 pgmap v427: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 24729/154992 objects degraded (15.955%); 1317/154992 objects misplaced (0.850%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T08:53:39.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:39 smithi016 bash[77892]: cluster 2023-10-09T08:53:38.947880+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 24729/154992 objects degraded (15.955%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:39.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:39 smithi016 bash[75179]: cluster 2023-10-09T08:53:37.393499+0000 mgr.y (mgr.25421) 480 : cluster 0 pgmap v427: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 24729/154992 objects degraded (15.955%); 1317/154992 objects misplaced (0.850%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T08:53:39.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:39 smithi016 bash[75179]: cluster 2023-10-09T08:53:38.947880+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 24729/154992 objects degraded (15.955%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:39.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:39.763 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:39.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:39.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:39.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:39.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:39.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:40.009 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:40.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:40.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:40.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:40.203 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:40.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:40.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:40.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:40.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:40.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:40.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.613 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:40.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:40.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:40.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:40.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:40.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:40.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:40.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:40.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:40.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:41.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:41.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:41.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:41.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:41.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:41.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:41.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:41.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:41.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:41.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:41.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:41.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:41.455 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:41.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:41.560 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:41.569 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:41 smithi094 bash[59677]: cluster 2023-10-09T08:53:39.393983+0000 mgr.y (mgr.25421) 481 : cluster 0 pgmap v428: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24729/154458 objects degraded (16.010%); 1317/154458 objects misplaced (0.853%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T08:53:41.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:41.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:41.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:41.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:41.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:41.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:41.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:41 smithi016 bash[75179]: cluster 2023-10-09T08:53:39.393983+0000 mgr.y (mgr.25421) 481 : cluster 0 pgmap v428: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24729/154458 objects degraded (16.010%); 1317/154458 objects misplaced (0.853%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T08:53:41.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:41 smithi016 bash[77892]: cluster 2023-10-09T08:53:39.393983+0000 mgr.y (mgr.25421) 481 : cluster 0 pgmap v428: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24729/154458 objects degraded (16.010%); 1317/154458 objects misplaced (0.853%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T08:53:41.849 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:41.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:41.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:41.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:41.986 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:42.019 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:42.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:42.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:42.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:42.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:42.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:42.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:42.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:42.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:42.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:42.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:42.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:42.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:42.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:42.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:42.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:42.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:42.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:42.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:43.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:43.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:43.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:43.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:43.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:43.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:43.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:43.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:43.377 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:43.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:43.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:43.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:43 smithi016 bash[75179]: cluster 2023-10-09T08:53:41.394746+0000 mgr.y (mgr.25421) 482 : cluster 0 pgmap v429: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 24729/148662 objects degraded (16.634%); 1317/148662 objects misplaced (0.886%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T08:53:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:43 smithi016 bash[77892]: cluster 2023-10-09T08:53:41.394746+0000 mgr.y (mgr.25421) 482 : cluster 0 pgmap v429: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 24729/148662 objects degraded (16.634%); 1317/148662 objects misplaced (0.886%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T08:53:43.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:43.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:43.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:43.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:43.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:43.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:43.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:43 smithi094 bash[59677]: cluster 2023-10-09T08:53:41.394746+0000 mgr.y (mgr.25421) 482 : cluster 0 pgmap v429: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 24729/148662 objects degraded (16.634%); 1317/148662 objects misplaced (0.886%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T08:53:43.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:43.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:43.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:43.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:43.824 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:53:43.855 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:53:43.882 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:53:43.888 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:43.912 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:53:43.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:43.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:43.948 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:53:43.990 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:53:44.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:44.033 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:53:44.055 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:44.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:44.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:44.068 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:53:44.077 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.077 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.077 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.078 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.078 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.078 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.078 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.078 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:53:44.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:44.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:44.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:44.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:44.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:44.314 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:44 smithi094 bash[59677]: audit 2023-10-09T08:53:43.789127+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.16:0/2889959159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:44.314 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:44 smithi094 bash[59677]: cluster 2023-10-09T08:53:43.948772+0000 mon.a (mon.0) 629 : cluster 3 Health check update: Degraded data redundancy: 24729/148662 objects degraded (16.634%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:44.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:44.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:44.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:44.437 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:44 smithi016 bash[75179]: audit 2023-10-09T08:53:43.789127+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.16:0/2889959159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:44.438 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:44 smithi016 bash[75179]: cluster 2023-10-09T08:53:43.948772+0000 mon.a (mon.0) 629 : cluster 3 Health check update: Degraded data redundancy: 24729/148662 objects degraded (16.634%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:44.438 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:44 smithi016 bash[77892]: audit 2023-10-09T08:53:43.789127+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.16:0/2889959159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:53:44.438 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:44 smithi016 bash[77892]: cluster 2023-10-09T08:53:43.948772+0000 mon.a (mon.0) 629 : cluster 3 Health check update: Degraded data redundancy: 24729/148662 objects degraded (16.634%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:44.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:44.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:44.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:44.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:44.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:44.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:44.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:44.713 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:53:44.713 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T08:53:44.713 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T08:53:44.713 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T08:53:44.714 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:53:44.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:44.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:44.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:44.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:44.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:44.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:44.971 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:45.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:45.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:45.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:45.248 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:45.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:45.318 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:45 smithi016 bash[77892]: cluster 2023-10-09T08:53:43.395338+0000 mgr.y (mgr.25421) 483 : cluster 0 pgmap v430: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 24733/145860 objects degraded (16.957%); 1096/145860 objects misplaced (0.751%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:53:45.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:45.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:45.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:45.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:45.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:45.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:45 smithi016 bash[75179]: cluster 2023-10-09T08:53:43.395338+0000 mgr.y (mgr.25421) 483 : cluster 0 pgmap v430: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 24733/145860 objects degraded (16.957%); 1096/145860 objects misplaced (0.751%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:53:45.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:45 smithi094 bash[59677]: cluster 2023-10-09T08:53:43.395338+0000 mgr.y (mgr.25421) 483 : cluster 0 pgmap v430: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 24733/145860 objects degraded (16.957%); 1096/145860 objects misplaced (0.751%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:53:45.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:45.802 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:45.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:45.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:45.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:45.997 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:53:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:53:45] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.33.4" 2023-10-09T08:53:46.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:46.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:46.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:46.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:46.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:46.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:46.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:46.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:46.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:46.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:46.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:46.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:46.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:46.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:46.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:46.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:46.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:46.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:46.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:46.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:46.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:46.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:46.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:46.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:47.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:47.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:47.130 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:47.160 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:47.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:47.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:47.405 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:47.437 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:47.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:47.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:47.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:47.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:47.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:47.679 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:47 smithi016 bash[75179]: cluster 2023-10-09T08:53:45.395900+0000 mgr.y (mgr.25421) 484 : cluster 0 pgmap v431: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 24733/143649 objects degraded (17.218%); 1096/143649 objects misplaced (0.763%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T08:53:47.679 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:47 smithi016 bash[77892]: cluster 2023-10-09T08:53:45.395900+0000 mgr.y (mgr.25421) 484 : cluster 0 pgmap v431: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 24733/143649 objects degraded (17.218%); 1096/143649 objects misplaced (0.763%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T08:53:47.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:47 smithi094 bash[59677]: cluster 2023-10-09T08:53:45.395900+0000 mgr.y (mgr.25421) 484 : cluster 0 pgmap v431: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 24733/143649 objects degraded (17.218%); 1096/143649 objects misplaced (0.763%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T08:53:47.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:47.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:47.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:47.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:47.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:47.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:48.068 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:48.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:48.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:48.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:48.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:48.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:48.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T08:53:48.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:48.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:48.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:48.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:48.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:48.499 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:48.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:48.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:48.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:48.643 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:48.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:48.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:48.771 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:48.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:48.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:48.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:48.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:53:48.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:48.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:48.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:49.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:49.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:49.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:49.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:49.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:49.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:49.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:49.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:49.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:49.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:49.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:49.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:49.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:49.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:49 smithi094 bash[59677]: cluster 2023-10-09T08:53:47.398295+0000 mgr.y (mgr.25421) 485 : cluster 0 pgmap v432: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 24733/137433 objects degraded (17.996%); 891/137433 objects misplaced (0.648%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T08:53:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:49 smithi094 bash[59677]: cluster 2023-10-09T08:53:48.950075+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 24733/137433 objects degraded (17.996%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:49.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:49 smithi094 bash[59677]: audit 2023-10-09T08:53:49.211778+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:49.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:49.566 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:49 smithi016 bash[75179]: cluster 2023-10-09T08:53:47.398295+0000 mgr.y (mgr.25421) 485 : cluster 0 pgmap v432: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 24733/137433 objects degraded (17.996%); 891/137433 objects misplaced (0.648%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T08:53:49.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:49 smithi016 bash[75179]: cluster 2023-10-09T08:53:48.950075+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 24733/137433 objects degraded (17.996%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:49.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:49 smithi016 bash[75179]: audit 2023-10-09T08:53:49.211778+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:49.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:49 smithi016 bash[77892]: cluster 2023-10-09T08:53:47.398295+0000 mgr.y (mgr.25421) 485 : cluster 0 pgmap v432: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 24733/137433 objects degraded (17.996%); 891/137433 objects misplaced (0.648%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T08:53:49.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:49 smithi016 bash[77892]: cluster 2023-10-09T08:53:48.950075+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 24733/137433 objects degraded (17.996%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:49.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:49 smithi016 bash[77892]: audit 2023-10-09T08:53:49.211778+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:53:49.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:49.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:49.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:49.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:49.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:49.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:49.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:49.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:49.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:50.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:50.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:50.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:50.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:50.286 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:50.396 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:50.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:50.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:50.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:50.552 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:50.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:50.657 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:50.707 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:50.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:50.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:50.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:50.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:50.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:51.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:51.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:51.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:51.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:51.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:51.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:51.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:51.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:51.372 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:51.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:51.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:51.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:51.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:51.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:51.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:51.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:51 smithi016 bash[75179]: cluster 2023-10-09T08:53:49.398822+0000 mgr.y (mgr.25421) 486 : cluster 0 pgmap v433: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 24733/136929 objects degraded (18.063%); 891/136929 objects misplaced (0.651%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:51.625 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:51 smithi016 bash[77892]: cluster 2023-10-09T08:53:49.398822+0000 mgr.y (mgr.25421) 486 : cluster 0 pgmap v433: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 24733/136929 objects degraded (18.063%); 891/136929 objects misplaced (0.651%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:51.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:51.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:51.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:51 smithi094 bash[59677]: cluster 2023-10-09T08:53:49.398822+0000 mgr.y (mgr.25421) 486 : cluster 0 pgmap v433: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 24733/136929 objects degraded (18.063%); 891/136929 objects misplaced (0.651%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:51.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:51.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:51.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:51.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:51.888 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:51.921 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:51.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:52.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:52.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:52.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:52.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:52.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:52.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:52.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:52.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:52.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:52.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:52.371 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.5 primary-affinity to 0 (802) 2023-10-09T08:53:52.372 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T08:53:52.440 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:52.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:52.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:52.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:52.607 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:52.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:52 smithi094 bash[59677]: audit 2023-10-09T08:53:51.953597+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.16:0/2502403981' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:52.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:52 smithi094 bash[59677]: audit 2023-10-09T08:53:51.954011+0000 mon.a (mon.0) 632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:52.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:52 smithi094 bash[59677]: audit 2023-10-09T08:53:52.273453+0000 mon.a (mon.0) 633 : audit 1 from='client.? 172.21.15.16:0/4234302148' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T08:53:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:52.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:52.697 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:52 smithi016 bash[75179]: audit 2023-10-09T08:53:51.953597+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.16:0/2502403981' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:52.697 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:52 smithi016 bash[75179]: audit 2023-10-09T08:53:51.954011+0000 mon.a (mon.0) 632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:52.697 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:52 smithi016 bash[75179]: audit 2023-10-09T08:53:52.273453+0000 mon.a (mon.0) 633 : audit 1 from='client.? 172.21.15.16:0/4234302148' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T08:53:52.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:52 smithi016 bash[77892]: audit 2023-10-09T08:53:51.953597+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.16:0/2502403981' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:52.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:52 smithi016 bash[77892]: audit 2023-10-09T08:53:51.954011+0000 mon.a (mon.0) 632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:53:52.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:52 smithi016 bash[77892]: audit 2023-10-09T08:53:52.273453+0000 mon.a (mon.0) 633 : audit 1 from='client.? 172.21.15.16:0/4234302148' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T08:53:52.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:52.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:52.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:52.845 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:52.875 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:52.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:52.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:53.108 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:53.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:53.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:53.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:53.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:53.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:53.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:53.399 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: cluster 2023-10-09T08:53:51.399615+0000 mgr.y (mgr.25421) 487 : cluster 0 pgmap v434: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 24733/130707 objects degraded (18.922%); 891/130707 objects misplaced (0.682%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:53.399 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: cluster 2023-10-09T08:53:52.344542+0000 mon.a (mon.0) 634 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:53:53.399 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: cluster 2023-10-09T08:53:52.359912+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.399 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: audit 2023-10-09T08:53:52.363362+0000 mon.a (mon.0) 635 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:53:53.400 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: audit 2023-10-09T08:53:52.363465+0000 mon.a (mon.0) 636 : audit 1 from='client.? 172.21.15.16:0/4234302148' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T08:53:53.400 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: cluster 2023-10-09T08:53:52.363530+0000 mon.a (mon.0) 637 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-10-09T08:53:53.400 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: cluster 2023-10-09T08:53:52.368882+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.400 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:53 smithi094 bash[59677]: cluster 2023-10-09T08:53:52.371999+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:53.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:53.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: cluster 2023-10-09T08:53:51.399615+0000 mgr.y (mgr.25421) 487 : cluster 0 pgmap v434: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 24733/130707 objects degraded (18.922%); 891/130707 objects misplaced (0.682%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:53.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: cluster 2023-10-09T08:53:52.344542+0000 mon.a (mon.0) 634 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:53:53.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: cluster 2023-10-09T08:53:52.359912+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: audit 2023-10-09T08:53:52.363362+0000 mon.a (mon.0) 635 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:53:53.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: audit 2023-10-09T08:53:52.363465+0000 mon.a (mon.0) 636 : audit 1 from='client.? 172.21.15.16:0/4234302148' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T08:53:53.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: cluster 2023-10-09T08:53:52.363530+0000 mon.a (mon.0) 637 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-10-09T08:53:53.512 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: cluster 2023-10-09T08:53:52.368882+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[75179]: cluster 2023-10-09T08:53:52.371999+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: cluster 2023-10-09T08:53:51.399615+0000 mgr.y (mgr.25421) 487 : cluster 0 pgmap v434: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 24733/130707 objects degraded (18.922%); 891/130707 objects misplaced (0.682%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: cluster 2023-10-09T08:53:52.344542+0000 mon.a (mon.0) 634 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: cluster 2023-10-09T08:53:52.359912+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: audit 2023-10-09T08:53:52.363362+0000 mon.a (mon.0) 635 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: audit 2023-10-09T08:53:52.363465+0000 mon.a (mon.0) 636 : audit 1 from='client.? 172.21.15.16:0/4234302148' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: cluster 2023-10-09T08:53:52.363530+0000 mon.a (mon.0) 637 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-10-09T08:53:53.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: cluster 2023-10-09T08:53:52.368882+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:53 smithi016 bash[77892]: cluster 2023-10-09T08:53:52.371999+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T08:53:53.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:53.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:53.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:53.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:53.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:53.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:54.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:54.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:54.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:54.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:54.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:54.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:54.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:54.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:54.337 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:54.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:54.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:54.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.361502+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.361609+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.363925+0000 mon.a (mon.0) 638 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.364198+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.364219+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.364549+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.367044+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.369446+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.951356+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 24733/130707 objects degraded (18.922%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:54.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.361502+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.361609+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.363925+0000 mon.a (mon.0) 638 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.364198+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.364219+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.364549+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.367044+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.369446+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:54 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.951356+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 24733/130707 objects degraded (18.922%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:54.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:54.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:54.578 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:54.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.361502+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.361609+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.363925+0000 mon.a (mon.0) 638 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-09T08:53:54.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.364198+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.588 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.364219+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.588 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.364549+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.588 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.367044+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.588 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.369446+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T08:53:54.588 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:54 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.951356+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 24733/130707 objects degraded (18.922%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:54.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:54.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:54.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:54.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:54.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:54.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:54.858 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:54.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:54.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:54.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:55.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:55.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:55.033 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:55.033 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:53:55.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:55.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:55.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:55.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:55.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:55.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:55.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:55.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:55 smithi016 bash[75179]: cluster 2023-10-09T08:53:53.400320+0000 mgr.y (mgr.25421) 488 : cluster 0 pgmap v437: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 24733/127866 objects degraded (19.343%); 682/127866 objects misplaced (0.533%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T08:53:55.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:55 smithi016 bash[75179]: cluster 2023-10-09T08:53:54.360771+0000 mon.a (mon.0) 640 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T08:53:55.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:55 smithi016 bash[77892]: cluster 2023-10-09T08:53:53.400320+0000 mgr.y (mgr.25421) 488 : cluster 0 pgmap v437: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 24733/127866 objects degraded (19.343%); 682/127866 objects misplaced (0.533%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T08:53:55.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:55 smithi016 bash[77892]: cluster 2023-10-09T08:53:54.360771+0000 mon.a (mon.0) 640 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T08:53:55.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:55.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:55.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:55.549 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:55 smithi094 bash[59677]: cluster 2023-10-09T08:53:53.400320+0000 mgr.y (mgr.25421) 488 : cluster 0 pgmap v437: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 24733/127866 objects degraded (19.343%); 682/127866 objects misplaced (0.533%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T08:53:55.549 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:55 smithi094 bash[59677]: cluster 2023-10-09T08:53:54.360771+0000 mon.a (mon.0) 640 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T08:53:55.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:55.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:55.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:55.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:55.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:55.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:55.962 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:53:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:53:55] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T08:53:55.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:55.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:56.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:56.111 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:56.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:56.218 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:56.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:53:56.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:56.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:53:56.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:56.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:56.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:56.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:56.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:56.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:56.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:56.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:56.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T08:53:56.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:56.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:56.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:56.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:56.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:56.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:53:56.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:53:56.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:56.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:56.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:56.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:57.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:57.076 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:57.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:57.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:57.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:57.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:57.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:57.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:57.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:57.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:57.441 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:57.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:57.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:57.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:57.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:57 smithi016 bash[75179]: cluster 2023-10-09T08:53:55.401097+0000 mgr.y (mgr.25421) 489 : cluster 0 pgmap v438: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 24733/127005 objects degraded (19.474%); 682/127005 objects misplaced (0.537%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T08:53:57.497 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:57 smithi016 bash[77892]: cluster 2023-10-09T08:53:55.401097+0000 mgr.y (mgr.25421) 489 : cluster 0 pgmap v438: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 24733/127005 objects degraded (19.474%); 682/127005 objects misplaced (0.537%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T08:53:57.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:57.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:57.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:57.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:57.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:57.721 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:57 smithi094 bash[59677]: cluster 2023-10-09T08:53:55.401097+0000 mgr.y (mgr.25421) 489 : cluster 0 pgmap v438: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 24733/127005 objects degraded (19.474%); 682/127005 objects misplaced (0.537%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T08:53:57.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:57.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:57.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:57.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:57.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:57.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:57.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:57.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:58.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:58.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:58.118 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:58.152 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:53:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:53:58.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:58.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:58.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:58.420 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:58.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:58.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:58.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:58.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:53:58.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:53:58.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:58.829 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:58.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:58.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:58.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:58.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:59.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:59.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:59.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:59.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:59.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:59.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:59.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:59.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:59.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:59.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:59.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:53:59.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:59.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:59 smithi016 bash[75179]: cluster 2023-10-09T08:53:57.402102+0000 mgr.y (mgr.25421) 490 : cluster 0 pgmap v439: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 24733/120687 objects degraded (20.494%); 341/120687 objects misplaced (0.283%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T08:53:59.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:53:59 smithi016 bash[75179]: cluster 2023-10-09T08:53:58.953388+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 24733/120687 objects degraded (20.494%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:59.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:59 smithi016 bash[77892]: cluster 2023-10-09T08:53:57.402102+0000 mgr.y (mgr.25421) 490 : cluster 0 pgmap v439: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 24733/120687 objects degraded (20.494%); 341/120687 objects misplaced (0.283%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T08:53:59.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:53:59 smithi016 bash[77892]: cluster 2023-10-09T08:53:58.953388+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 24733/120687 objects degraded (20.494%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:59.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:53:59.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:59.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:53:59.533 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:53:59.533 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:53:59.534 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:53:59.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T08:53:59.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:59.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:53:59.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:59.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:53:59.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:59.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:59 smithi094 bash[59677]: cluster 2023-10-09T08:53:57.402102+0000 mgr.y (mgr.25421) 490 : cluster 0 pgmap v439: 89 pgs: 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 24733/120687 objects degraded (20.494%); 341/120687 objects misplaced (0.283%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T08:53:59.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:53:59 smithi094 bash[59677]: cluster 2023-10-09T08:53:58.953388+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 24733/120687 objects degraded (20.494%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:53:59.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:53:59.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:59.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:53:59.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:59.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:53:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:53:59.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:00.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:00.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:00.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:00.150 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:00.182 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:00.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:00.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:00.322 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:00.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:00.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:00.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:00 smithi016 bash[75179]: cluster 2023-10-09T08:53:59.402691+0000 mgr.y (mgr.25421) 491 : cluster 0 pgmap v440: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 436 op/s; 24733/120237 objects degraded (20.570%); 341/120237 objects misplaced (0.284%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T08:54:00.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:00 smithi016 bash[75179]: cluster 2023-10-09T08:54:00.379105+0000 mon.a (mon.0) 642 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T08:54:00.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:00 smithi094 bash[59677]: cluster 2023-10-09T08:53:59.402691+0000 mgr.y (mgr.25421) 491 : cluster 0 pgmap v440: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 436 op/s; 24733/120237 objects degraded (20.570%); 341/120237 objects misplaced (0.284%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T08:54:00.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:00 smithi094 bash[59677]: cluster 2023-10-09T08:54:00.379105+0000 mon.a (mon.0) 642 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T08:54:00.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:00.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:00.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:00.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:00.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:00.692 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:00.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:00.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:00.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:00 smithi016 bash[77892]: cluster 2023-10-09T08:53:59.402691+0000 mgr.y (mgr.25421) 491 : cluster 0 pgmap v440: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 436 op/s; 24733/120237 objects degraded (20.570%); 341/120237 objects misplaced (0.284%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T08:54:00.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:00 smithi016 bash[77892]: cluster 2023-10-09T08:54:00.379105+0000 mon.a (mon.0) 642 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T08:54:00.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:00.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:00.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:00.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:00.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:01.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:01.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:01.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:01.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:01.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:01.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:01.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:01.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:01.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:01.509 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:01.557 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:01.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:01.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:01.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:01.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:01.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:01.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:01.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:01.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:01.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:01.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:01.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:02.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:02.094 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:02.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:02.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:02.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:02.225 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:02.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:02.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:02.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:02.348 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16749 6.b 6:d2a2ba62:::benchmark_data_smithi016_105291_object16748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.634291+0000", 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.70628508600001, 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14878321999999999, 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.349 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16749 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.634291+0000", 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.634291+0000", 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.634290+0000", 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.634315+0000", 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4765000000000001e-05 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.634328+0000", 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2809000000000001e-05 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.634336+0000", 2023-10-09T08:54:02.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3590000000000008e-06 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.636347+0000", 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002010431 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.636478+0000", 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000130975 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.636627+0000", 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014994300000000001 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.637331+0000", 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00070315000000000004 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.638997+0000", 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001666533 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642238+0000", 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.003240382 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642303+0000", 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5254999999999998e-05 2023-10-09T08:54:02.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783074+0000", 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14077161699999999 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.356 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16760 6.b 6:d9f2af5c:::benchmark_data_smithi016_105291_object16759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:54:02.357 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.640960+0000", 2023-10-09T08:54:02.357 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.69961598099999, 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14987315100000001, 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16760 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.640960+0000", 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.640960+0000", 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.640959+0000", 2023-10-09T08:54:02.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.640984+0000", 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5452e-05 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.640999+0000", 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4116e-05 2023-10-09T08:54:02.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641007+0000", 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7590000000000003e-06 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641049+0000", 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1699000000000002e-05 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641153+0000", 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000103879 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641326+0000", 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000173487 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.784509+0000", 2023-10-09T08:54:02.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14318288500000001 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.784558+0000", 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8837e-05 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790723+0000", 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0061653089999999999 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790771+0000", 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7367000000000002e-05 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790833+0000", 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2691e-05 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16761 6.c 6:3d4bb449:::benchmark_data_smithi016_105291_object16760:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.641028+0000", 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.699548585, 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14962309500000001, 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16761 2023-10-09T08:54:02.365 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641028+0000", 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641028+0000", 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641027+0000", 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641051+0000", 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3842e-05 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641059+0000", 2023-10-09T08:54:02.367 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2029999999999992e-06 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641066+0000", 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9959999999999996e-06 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641182+0000", 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000116477 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641316+0000", 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013325000000000001 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641479+0000", 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000163456 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.782999+0000", 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14151960599999999 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.789038+0000", 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0060395529999999996 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790417+0000", 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0013788559999999999 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790465+0000", 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7513999999999999e-05 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790651+0000", 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000186109 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.44690.0:16749 6.b e101/95 ondisk, result = 0)", 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.641155+0000", 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.699421261, 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14193582399999999, 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641155+0000", 2023-10-09T08:54:02.372 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641155+0000", 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641155+0000", 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641159+0000", 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2479999999999998e-06 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641169+0000", 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0597999999999999e-05 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.374 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641177+0000", 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3710000000000002e-06 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642220+0000", 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00104357 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642231+0000", 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0618999999999999e-05 2023-10-09T08:54:02.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783091+0000", 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14085992999999999 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16766 6.6 6:61564e9b:::benchmark_data_smithi016_105291_object16765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.641611+0000", 2023-10-09T08:54:02.376 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.6989652, 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14887904399999999, 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16766 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641611+0000", 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641611+0000", 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641610+0000", 2023-10-09T08:54:02.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641663+0000", 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2509999999999997e-05 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641675+0000", 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2378000000000001e-05 2023-10-09T08:54:02.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641695+0000", 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9649000000000001e-05 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.641750+0000", 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5614e-05 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.380 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642024+0000", 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000273715 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,7", 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642172+0000", 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014758600000000001 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783041+0000", 2023-10-09T08:54:02.381 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14086891800000001 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.789225+0000", 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0061848889999999998 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790413+0000", 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011871340000000001 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790456+0000", 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3751999999999997e-05 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790490+0000", 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3834000000000002e-05 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.383 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:16763 6.7 e101/95 6:e9d8ca39:::benchmark_data_smithi016_105291_object16762:head v 101'1033, mlcod=101'1031)", 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.642320+0000", 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.69825676799999, 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14548259099999999, 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642320+0000", 2023-10-09T08:54:02.384 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642320+0000", 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642319+0000", 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.385 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642357+0000", 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8810000000000003e-05 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.782881+0000", 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14052313999999999 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783007+0000", 2023-10-09T08:54:02.386 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012599099999999999 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783106+0000", 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9333000000000006e-05 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783131+0000", 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4851999999999999e-05 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.787754+0000", 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004622925 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.787802+0000", 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8372999999999999e-05 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.388 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:16759 6.5 e101/95 6:ac876bf5:::benchmark_data_smithi016_105291_object16758:head v 101'1056, mlcod=101'1054)", 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.642395+0000", 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.69818112300001, 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14259982600000001, 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642395+0000", 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.389 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642395+0000", 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642394+0000", 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.390 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642433+0000", 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8172e-05 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.782929+0000", 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14049684000000001 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.782963+0000", 2023-10-09T08:54:02.391 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3714999999999998e-05 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783058+0000", 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5296000000000004e-05 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783095+0000", 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6684999999999999e-05 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.392 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.784956+0000", 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018603649999999999 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.784995+0000", 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9434000000000002e-05 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.393 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:16767 6.b 6:debee5cb:::benchmark_data_smithi016_105291_object16766:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:52.642900+0000", 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 189.697676632, 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.148166622, 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 16767 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.394 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642900+0000", 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642900+0000", 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.395 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642899+0000", 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.642936+0000", 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.765e-05 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783074+0000", 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14013748700000001 2023-10-09T08:54:02.396 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783087+0000", 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3049e-05 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783122+0000", 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5456000000000001e-05 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.397 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783319+0000", 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019644199999999999 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.783539+0000", 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00022040199999999999 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.398 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.786997+0000", 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0034577470000000002 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.787006+0000", 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8279999999999992e-06 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.790972+0000", 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039665170000000001 2023-10-09T08:54:02.399 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.791024+0000", 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1878e-05 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:52.791066+0000", 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2144000000000002e-05 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.400 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:27326 6.5 e101/95 6:ac6f3e07:::benchmark_data_smithi016_105291_object27325:head v 101'1754, mlcod=101'1752)", 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.361107+0000", 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 183.97946930500001, 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13887260600000001, 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.401 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361107+0000", 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361107+0000", 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361106+0000", 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:54:02.402 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361140+0000", 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4443000000000002e-05 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361168+0000", 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7146999999999999e-05 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.403 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361175+0000", 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2649999999999999e-06 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361224+0000", 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9129999999999999e-05 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.361245+0000", 2023-10-09T08:54:02.404 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0594e-05 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.499879+0000", 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.138634534 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.499980+0000", 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010045200000000001 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.405 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:27336 6.b 6:d926e81a:::benchmark_data_smithi016_105291_object27335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.364831+0000", 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 183.97574486100001, 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13608040099999999, 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.406 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27336 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364831+0000", 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.407 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364831+0000", 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364831+0000", 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364856+0000", 2023-10-09T08:54:02.408 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5040000000000001e-05 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364867+0000", 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0871e-05 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364874+0000", 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2749999999999998e-06 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.409 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.364921+0000", 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6703000000000003e-05 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365126+0000", 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000205751 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.410 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365262+0000", 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000135475 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368722+0000", 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0034599710000000001 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.411 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500172+0000", 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.131450654 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500301+0000", 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012856799999999999 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500863+0000", 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00056157500000000003 2023-10-09T08:54:02.412 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500912+0000", 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9298999999999997e-05 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:27329 6.7 e101/95 6:e1c6a617:::benchmark_data_smithi016_105291_object27328:head v 101'1680, mlcod=101'1677)", 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.365309+0000", 2023-10-09T08:54:02.413 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 183.97526746599999, 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.137265899, 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365309+0000", 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.414 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365309+0000", 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365308+0000", 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365350+0000", 2023-10-09T08:54:02.415 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1613000000000003e-05 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365361+0000", 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0896e-05 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365368+0000", 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0169999999999999e-06 2023-10-09T08:54:02.416 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366131+0000", 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00076302300000000004 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.366169+0000", 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8605000000000001e-05 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.417 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.502528+0000", 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13635909600000001 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.502575+0000", 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6248000000000002e-05 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.418 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:27332 6.2 e101/95 6:4a3ded73:::benchmark_data_smithi016_105291_object27331:head v 101'1720, mlcod=101'1718)", 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.365391+0000", 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 183.975184912, 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13759229000000001, 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365391+0000", 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.419 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365391+0000", 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.365391+0000", 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.420 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368233+0000", 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028417400000000002 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368419+0000", 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018634399999999999 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.421 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368427+0000", 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3860000000000007e-06 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368486+0000", 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8347000000000002e-05 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368513+0000", 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7254e-05 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.422 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.502948+0000", 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13443471800000001 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.502984+0000", 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5882e-05 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.423 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.424 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:27338 6.2 e101/95 6:486832a1:::benchmark_data_smithi016_105291_object27337:head v 101'1721, mlcod=101'1718)", 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:50:58.368461+0000", 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 183.97211483800001, 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13676787900000001, 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.426 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368461+0000", 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368461+0000", 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.427 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368461+0000", 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368591+0000", 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000129882 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368601+0000", 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0122000000000001e-05 2023-10-09T08:54:02.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.368607+0000", 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1979999999999998e-06 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500230+0000", 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13162312400000001 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.500253+0000", 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2578e-05 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505177+0000", 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0049240430000000003 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:50:58.505229+0000", 2023-10-09T08:54:02.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2543000000000002e-05 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:160460 6.7 6:e6f82eea:::benchmark_data_smithi016_105291_object59333:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e115)", 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:52.377194+0000", 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 9.9633824240000006, 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0369090299999999, 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.431 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 160460 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377194+0000", 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.432 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377194+0000", 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377192+0000", 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377202+0000", 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0210999999999999e-05 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377228+0000", 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6313999999999999e-05 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377241+0000", 2023-10-09T08:54:02.434 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2678e-05 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.378665+0000", 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0014242059999999999 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op must wait for map", 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.378689+0000", 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4119999999999999e-05 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.435 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.385787+0000", 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0070978500000000002 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.385798+0000", 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0801999999999999e-05 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.411339+0000", 2023-10-09T08:54:02.436 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.025541077 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.414103+0000", 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002763929 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.437 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:160469 6.7 6:e99a2193:::benchmark_data_smithi016_105291_object59342:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e115)", 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:52.377260+0000", 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 9.9633164569999995, 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.037353438, 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 160469 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.438 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377260+0000", 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377260+0000", 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.439 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377259+0000", 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377264+0000", 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.425e-06 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377286+0000", 2023-10-09T08:54:02.440 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.234e-05 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.377294+0000", 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9850000000000006e-06 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.378965+0000", 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001671184 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.441 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting_for_map not empty", 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.378993+0000", 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8245000000000001e-05 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.386036+0000", 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0070421980000000004 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:54:02.442 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.386046+0000", 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0311e-05 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.414533+0000", 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0284871520000001 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.414613+0000", 2023-10-09T08:54:02.443 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0232e-05 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:160553 6.7 6:eeff49f0:::benchmark_data_smithi016_105291_object59426:head [delete] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:52.556765+0000", 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 9.7838112539999997, 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.86696376799999997, 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.444 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 160553 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556765+0000", 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.445 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556765+0000", 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556763+0000", 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556773+0000", 2023-10-09T08:54:02.446 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0101e-05 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556793+0000", 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0024999999999999e-05 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556803+0000", 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4339999999999998e-06 2023-10-09T08:54:02.447 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556890+0000", 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6989000000000003e-05 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.556904+0000", 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4851999999999999e-05 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.448 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.414680+0000", 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.85777541400000001 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.415877+0000", 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001197519 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-10-09T08:54:02.449 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.416072+0000", 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019452799999999999 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.421669+0000", 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0055974479999999997 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.423564+0000", 2023-10-09T08:54:02.450 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018950620000000001 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.423625+0000", 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0942e-05 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.423685+0000", 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9460999999999999e-05 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.451 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.423729+0000", 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3980999999999998e-05 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163877 6.b 6:d9759120:::benchmark_data_smithi016_105291_object62750:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.903428+0000", 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 3.4371479320000002, 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28219493600000001, 2023-10-09T08:54:02.452 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163877 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903428+0000", 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.453 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903428+0000", 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903427+0000", 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.454 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903434+0000", 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7389999999999999e-06 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903450+0000", 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5257e-05 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.455 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903457+0000", 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5639999999999999e-06 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903512+0000", 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5140999999999999e-05 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.456 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903647+0000", 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134509 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903773+0000", 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012631099999999999 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.457 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909325+0000", 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0055514079999999999 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.927574+0000", 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018249431 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.185253+0000", 2023-10-09T08:54:02.458 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.257678503 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.185504+0000", 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025145199999999999 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.185623+0000", 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011924799999999999 2023-10-09T08:54:02.459 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163882 6.b 6:de953810:::benchmark_data_smithi016_105291_object62755:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.909537+0000", 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 3.4310397529999999, 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29206790199999999, 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.460 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163882 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909537+0000", 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.461 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909537+0000", 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909535+0000", 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.462 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909541+0000", 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1399999999999997e-06 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909556+0000", 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4517e-05 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909564+0000", 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0790000000000001e-06 2023-10-09T08:54:02.463 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909602+0000", 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8353999999999999e-05 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909736+0000", 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134206 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.464 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909867+0000", 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013098999999999999 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.918047+0000", 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0081791429999999998 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.932903+0000", 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.01485618 2023-10-09T08:54:02.465 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201497+0000", 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26859458899999999 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201552+0000", 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4635999999999997e-05 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.466 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201604+0000", 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2417999999999999e-05 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163883 6.b 6:db0a5a25:::benchmark_data_smithi016_105291_object62756:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.909572+0000", 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 3.431004459, 2023-10-09T08:54:02.467 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29216171800000001, 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163883 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909572+0000", 2023-10-09T08:54:02.468 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909572+0000", 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909571+0000", 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.469 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909574+0000", 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.295e-06 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909578+0000", 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3189999999999996e-06 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.470 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909582+0000", 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8639999999999998e-06 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.914341+0000", 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0047590280000000002 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.914426+0000", 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5076999999999999e-05 2023-10-09T08:54:02.471 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.914510+0000", 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4016000000000003e-05 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.918209+0000", 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00369886 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.472 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.933365+0000", 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015156549 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201656+0000", 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26829077299999998 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.473 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201697+0000", 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1208999999999998e-05 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201734+0000", 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6087000000000002e-05 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.474 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163905 6.b 6:da014ced:::benchmark_data_smithi016_105291_object62778:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.957027+0000", 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 3.3835496429999998, 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24524141099999999, 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163905 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:54:02.475 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957027+0000", 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957027+0000", 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957025+0000", 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957034+0000", 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6830000000000003e-06 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957053+0000", 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8215000000000001e-05 2023-10-09T08:54:02.477 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957065+0000", 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2475e-05 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957122+0000", 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6542000000000002e-05 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.478 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957303+0000", 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018179900000000001 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,7", 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.957490+0000", 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018694 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.479 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.963232+0000", 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0057417259999999999 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.964003+0000", 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00077116399999999996 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.202185+0000", 2023-10-09T08:54:02.480 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23818152300000001 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.202232+0000", 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6896e-05 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.202268+0000", 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6424000000000002e-05 2023-10-09T08:54:02.481 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.482 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.482 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.482 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:54:02.482 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:54:02.482 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:54:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:02.483 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:02.483 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:54:02.484 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:54:02.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:02.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:02.525 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:54:02.569 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:54:02.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:02.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:02.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:02.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:02.623 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:54:02.661 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:54:02.710 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:54:02.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:02.729 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:02 smithi094 bash[59677]: cluster 2023-10-09T08:54:01.403620+0000 mgr.y (mgr.25421) 492 : cluster 0 pgmap v441: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 24733/114822 objects degraded (21.540%); 341/114822 objects misplaced (0.297%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T08:54:02.729 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:02 smithi094 bash[59677]: audit 2023-10-09T08:54:02.408257+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.16:0/3038071059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:02.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:02 smithi016 bash[75179]: cluster 2023-10-09T08:54:01.403620+0000 mgr.y (mgr.25421) 492 : cluster 0 pgmap v441: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 24733/114822 objects degraded (21.540%); 341/114822 objects misplaced (0.297%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T08:54:02.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:02 smithi016 bash[75179]: audit 2023-10-09T08:54:02.408257+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.16:0/3038071059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:02.751 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:54:02.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:02 smithi016 bash[77892]: cluster 2023-10-09T08:54:01.403620+0000 mgr.y (mgr.25421) 492 : cluster 0 pgmap v441: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 24733/114822 objects degraded (21.540%); 341/114822 objects misplaced (0.297%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T08:54:02.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:02 smithi016 bash[77892]: audit 2023-10-09T08:54:02.408257+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.16:0/3038071059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.753 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:54:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:02.764 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:02.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:02.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:02.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:02.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:02.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:03.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:03.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:03.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:03.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:03.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:03.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:03.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:03.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:03.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:03.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:03.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:03.695 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:03.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:03.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:03.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:03.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:03.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:03.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:04.056 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:04.098 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:04.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:04.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:04.241 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:04.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:04.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:04.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:04.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:04.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:04.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:04.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:04.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:04.613 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:04.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:04.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:04.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:04 smithi094 bash[59677]: cluster 2023-10-09T08:54:03.404890+0000 mgr.y (mgr.25421) 493 : cluster 0 pgmap v442: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24733/111513 objects degraded (22.179%); 94/111513 objects misplaced (0.084%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T08:54:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:04 smithi094 bash[59677]: cluster 2023-10-09T08:54:03.954762+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 24733/111513 objects degraded (22.179%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:04 smithi094 bash[59677]: audit 2023-10-09T08:54:04.211849+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:04.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:04 smithi016 bash[75179]: cluster 2023-10-09T08:54:03.404890+0000 mgr.y (mgr.25421) 493 : cluster 0 pgmap v442: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24733/111513 objects degraded (22.179%); 94/111513 objects misplaced (0.084%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T08:54:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:04 smithi016 bash[75179]: cluster 2023-10-09T08:54:03.954762+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 24733/111513 objects degraded (22.179%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:04 smithi016 bash[75179]: audit 2023-10-09T08:54:04.211849+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:04 smithi016 bash[77892]: cluster 2023-10-09T08:54:03.404890+0000 mgr.y (mgr.25421) 493 : cluster 0 pgmap v442: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24733/111513 objects degraded (22.179%); 94/111513 objects misplaced (0.084%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T08:54:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:04 smithi016 bash[77892]: cluster 2023-10-09T08:54:03.954762+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 24733/111513 objects degraded (22.179%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:04 smithi016 bash[77892]: audit 2023-10-09T08:54:04.211849+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:04.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:04.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:04.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:04.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:04.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:04.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:05.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:05.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:05.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:05.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:05.195 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:05.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:05.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T08:54:05.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:05.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:05.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:05.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:05.483 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:05.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:05.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:05.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:05.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:05.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:05.744 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:05.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:05.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:05.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:05.900 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:05.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:05.988 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:54:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:54:05] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.33.4" 2023-10-09T08:54:06.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:06.030 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:06.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:06.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:06.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:06.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:06.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:06.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:06.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:06.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:06.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:06.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:06.683 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:06 smithi094 bash[59677]: cluster 2023-10-09T08:54:05.405473+0000 mgr.y (mgr.25421) 494 : cluster 0 pgmap v443: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 24733/110391 objects degraded (22.405%); 94/110391 objects misplaced (0.085%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T08:54:06.683 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:06 smithi094 bash[59677]: cluster 2023-10-09T08:54:05.463099+0000 mon.a (mon.0) 645 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T08:54:06.683 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:06 smithi094 bash[59677]: cluster 2023-10-09T08:54:05.463723+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.683 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:06 smithi094 bash[59677]: cluster 2023-10-09T08:54:05.464136+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.683 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:06 smithi094 bash[59677]: cluster 2023-10-09T08:54:05.470964+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.683 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:06 smithi094 bash[59677]: cluster 2023-10-09T08:54:05.476417+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:06.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[75179]: cluster 2023-10-09T08:54:05.405473+0000 mgr.y (mgr.25421) 494 : cluster 0 pgmap v443: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 24733/110391 objects degraded (22.405%); 94/110391 objects misplaced (0.085%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T08:54:06.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[75179]: cluster 2023-10-09T08:54:05.463099+0000 mon.a (mon.0) 645 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T08:54:06.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[75179]: cluster 2023-10-09T08:54:05.463723+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[75179]: cluster 2023-10-09T08:54:05.464136+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[75179]: cluster 2023-10-09T08:54:05.470964+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[75179]: cluster 2023-10-09T08:54:05.476417+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[77892]: cluster 2023-10-09T08:54:05.405473+0000 mgr.y (mgr.25421) 494 : cluster 0 pgmap v443: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 24733/110391 objects degraded (22.405%); 94/110391 objects misplaced (0.085%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T08:54:06.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[77892]: cluster 2023-10-09T08:54:05.463099+0000 mon.a (mon.0) 645 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T08:54:06.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[77892]: cluster 2023-10-09T08:54:05.463723+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[77892]: cluster 2023-10-09T08:54:05.464136+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[77892]: cluster 2023-10-09T08:54:05.470964+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.763 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:06 smithi016 bash[77892]: cluster 2023-10-09T08:54:05.476417+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T08:54:06.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:06.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:06.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:06.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:06.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:07.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:07.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:07.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:07.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:07.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:07.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:07.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:07.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:54:07.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:07.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:07.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:07.485 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[75179]: cluster 2023-10-09T08:54:06.470474+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.485 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[77892]: cluster 2023-10-09T08:54:06.470474+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.485 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[77892]: cluster 2023-10-09T08:54:06.472937+0000 mon.a (mon.0) 646 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-10-09T08:54:07.485 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[77892]: cluster 2023-10-09T08:54:06.473042+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.485 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[77892]: cluster 2023-10-09T08:54:06.473042+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.485 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[77892]: audit 2023-10-09T08:54:07.206201+0000 mon.a (mon.0) 647 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:07.557 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:07.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:07.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:07.702 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:07 smithi094 bash[59677]: cluster 2023-10-09T08:54:06.470474+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:07 smithi094 bash[59677]: cluster 2023-10-09T08:54:06.472937+0000 mon.a (mon.0) 646 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-10-09T08:54:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:07 smithi094 bash[59677]: cluster 2023-10-09T08:54:06.473042+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:07 smithi094 bash[59677]: cluster 2023-10-09T08:54:06.473042+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:07 smithi094 bash[59677]: audit 2023-10-09T08:54:07.206201+0000 mon.a (mon.0) 647 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:07.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:07.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[75179]: cluster 2023-10-09T08:54:06.472937+0000 mon.a (mon.0) 646 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-10-09T08:54:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[75179]: cluster 2023-10-09T08:54:06.473042+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[75179]: cluster 2023-10-09T08:54:06.473042+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T08:54:07.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:07 smithi016 bash[75179]: audit 2023-10-09T08:54:07.206201+0000 mon.a (mon.0) 647 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:07.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:07.901 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:07.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:08.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:08.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:08.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:08.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:08.136 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:08.179 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:08.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:08.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:08.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:08.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:08.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:08.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:08.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:08.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:08.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:08.486 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:08 smithi094 bash[59677]: cluster 2023-10-09T08:54:07.406227+0000 mgr.y (mgr.25421) 495 : cluster 0 pgmap v446: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 23088/103539 objects degraded (22.299%); 1593/103539 objects misplaced (1.539%); 2.9 MiB/s, 49 objects/s recovering 2023-10-09T08:54:08.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:08.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:08.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:08.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:08.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:08.706 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:08 smithi016 bash[75179]: cluster 2023-10-09T08:54:07.406227+0000 mgr.y (mgr.25421) 495 : cluster 0 pgmap v446: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 23088/103539 objects degraded (22.299%); 1593/103539 objects misplaced (1.539%); 2.9 MiB/s, 49 objects/s recovering 2023-10-09T08:54:08.706 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:08 smithi016 bash[77892]: cluster 2023-10-09T08:54:07.406227+0000 mgr.y (mgr.25421) 495 : cluster 0 pgmap v446: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 23088/103539 objects degraded (22.299%); 1593/103539 objects misplaced (1.539%); 2.9 MiB/s, 49 objects/s recovering 2023-10-09T08:54:08.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:08.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:08.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:08.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:08.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:08.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:09.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:09.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:09.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:09.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:09.230 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:09.281 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:09.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:09.387 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:09.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:09.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:09 smithi094 bash[59677]: cluster 2023-10-09T08:54:08.955688+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 23088/103539 objects degraded (22.299%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:09.519 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:09.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:09.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:09.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:09 smithi016 bash[75179]: cluster 2023-10-09T08:54:08.955688+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 23088/103539 objects degraded (22.299%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:09 smithi016 bash[77892]: cluster 2023-10-09T08:54:08.955688+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 23088/103539 objects degraded (22.299%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:09.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:09.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:09.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:09.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:09.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:10.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:10.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:10.147 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:54:10.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:10.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:10.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:10.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:10.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:10.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:10.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:10.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:10.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:10.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:10.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:10.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:10.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:10.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:10.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:10.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:10.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:10.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:10.994 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:10 smithi016 bash[75179]: cluster 2023-10-09T08:54:09.406801+0000 mgr.y (mgr.25421) 496 : cluster 0 pgmap v447: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 23088/103047 objects degraded (22.405%); 1593/103047 objects misplaced (1.546%); 2.9 MiB/s, 49 objects/s recovering 2023-10-09T08:54:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:10.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:10.995 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:10 smithi016 bash[77892]: cluster 2023-10-09T08:54:09.406801+0000 mgr.y (mgr.25421) 496 : cluster 0 pgmap v447: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 23088/103047 objects degraded (22.405%); 1593/103047 objects misplaced (1.546%); 2.9 MiB/s, 49 objects/s recovering 2023-10-09T08:54:11.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:11.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:11.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:11.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:11.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:11.282 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:11.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:11.302 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:10 smithi094 bash[59677]: cluster 2023-10-09T08:54:09.406801+0000 mgr.y (mgr.25421) 496 : cluster 0 pgmap v447: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 23088/103047 objects degraded (22.405%); 1593/103047 objects misplaced (1.546%); 2.9 MiB/s, 49 objects/s recovering 2023-10-09T08:54:11.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:11.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:11.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:11.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:11.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:11.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:11.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:11.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:11.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:11.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:11.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:11.838 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:11.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:11.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:11.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:11 smithi016 bash[75179]: audit 2023-10-09T08:54:11.874038+0000 mon.a (mon.0) 649 : audit 1 from='client.? 172.21.15.16:0/3925363942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:54:11.972 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:11 smithi016 bash[77892]: audit 2023-10-09T08:54:11.874038+0000 mon.a (mon.0) 649 : audit 1 from='client.? 172.21.15.16:0/3925363942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:54:11.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:11.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:11.990 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T08:54:12.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:12.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:12.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:12.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:12.222 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:11 smithi094 bash[59677]: audit 2023-10-09T08:54:11.874038+0000 mon.a (mon.0) 649 : audit 1 from='client.? 172.21.15.16:0/3925363942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:54:12.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:12.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:12.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:12.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:12.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:12.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:12.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:12.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:12.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:12.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:12.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:12.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:12.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:12.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:12.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:12.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:12.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:12.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:13.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:13.042 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:12 smithi094 bash[59677]: cluster 2023-10-09T08:54:11.407409+0000 mgr.y (mgr.25421) 497 : cluster 0 pgmap v448: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 23192/98268 objects degraded (23.601%); 1269/98268 objects misplaced (1.291%); 2.6 MiB/s, 45 objects/s recovering 2023-10-09T08:54:13.042 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:12 smithi094 bash[59677]: cluster 2023-10-09T08:54:11.960949+0000 mon.a (mon.0) 650 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:54:13.042 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:12 smithi094 bash[59677]: cluster 2023-10-09T08:54:11.971232+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T08:54:13.042 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:12 smithi094 bash[59677]: audit 2023-10-09T08:54:11.974699+0000 mon.a (mon.0) 651 : audit 1 from='client.? 172.21.15.16:0/3925363942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:54:13.042 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:12 smithi094 bash[59677]: cluster 2023-10-09T08:54:11.974780+0000 mon.a (mon.0) 652 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-10-09T08:54:13.068 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:13.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:13.174 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:13.218 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:13.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:13.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[75179]: cluster 2023-10-09T08:54:11.407409+0000 mgr.y (mgr.25421) 497 : cluster 0 pgmap v448: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 23192/98268 objects degraded (23.601%); 1269/98268 objects misplaced (1.291%); 2.6 MiB/s, 45 objects/s recovering 2023-10-09T08:54:13.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[75179]: cluster 2023-10-09T08:54:11.960949+0000 mon.a (mon.0) 650 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:54:13.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[75179]: cluster 2023-10-09T08:54:11.971232+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T08:54:13.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[75179]: audit 2023-10-09T08:54:11.974699+0000 mon.a (mon.0) 651 : audit 1 from='client.? 172.21.15.16:0/3925363942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:54:13.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[75179]: cluster 2023-10-09T08:54:11.974780+0000 mon.a (mon.0) 652 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-10-09T08:54:13.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[77892]: cluster 2023-10-09T08:54:11.407409+0000 mgr.y (mgr.25421) 497 : cluster 0 pgmap v448: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 23192/98268 objects degraded (23.601%); 1269/98268 objects misplaced (1.291%); 2.6 MiB/s, 45 objects/s recovering 2023-10-09T08:54:13.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[77892]: cluster 2023-10-09T08:54:11.960949+0000 mon.a (mon.0) 650 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:54:13.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[77892]: cluster 2023-10-09T08:54:11.971232+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T08:54:13.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[77892]: audit 2023-10-09T08:54:11.974699+0000 mon.a (mon.0) 651 : audit 1 from='client.? 172.21.15.16:0/3925363942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:54:13.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:12 smithi016 bash[77892]: cluster 2023-10-09T08:54:11.974780+0000 mon.a (mon.0) 652 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-10-09T08:54:13.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:13.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:13.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:13.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:13.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:13.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:13.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:13.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:13.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:13.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:13.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:13.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:13.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:13.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:13.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:13.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:13.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:13.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:13.997 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:13 smithi016 bash[75179]: cluster 2023-10-09T08:54:13.956847+0000 mon.a (mon.0) 653 : cluster 3 Health check update: Degraded data redundancy: 23192/98268 objects degraded (23.601%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:14.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:14.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:14.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:14.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:14.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:13 smithi094 bash[59677]: cluster 2023-10-09T08:54:13.956847+0000 mon.a (mon.0) 653 : cluster 3 Health check update: Degraded data redundancy: 23192/98268 objects degraded (23.601%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:14.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:13 smithi016 bash[77892]: cluster 2023-10-09T08:54:13.956847+0000 mon.a (mon.0) 653 : cluster 3 Health check update: Degraded data redundancy: 23192/98268 objects degraded (23.601%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:14.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:14.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:14.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:14.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:14.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:14.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:14.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:14.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:14.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:14.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.707 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:14 smithi016 bash[75179]: debug 2023-10-09T08:54:14.304+0000 7fa046a4d700 -1 mon.a@0(leader).osd e119 definitely_dead 0 2023-10-09T08:54:14.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:14.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:14.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:14.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:14.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:14.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:15.076 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.7. 2023-10-09T08:54:15.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:15.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:15.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[75179]: cluster 2023-10-09T08:54:13.408087+0000 mgr.y (mgr.25421) 498 : cluster 0 pgmap v450: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23192/95058 objects degraded (24.398%); 1269/95058 objects misplaced (1.335%); 2.7 MiB/s, 46 objects/s recovering 2023-10-09T08:54:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[75179]: audit 2023-10-09T08:54:14.050626+0000 mon.a (mon.0) 654 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[75179]: audit 2023-10-09T08:54:14.059194+0000 mon.a (mon.0) 655 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[75179]: audit 2023-10-09T08:54:14.308833+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.16:0/1743192997' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[75179]: audit 2023-10-09T08:54:14.309586+0000 mon.a (mon.0) 656 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[75179]: cluster 2023-10-09T08:54:14.309725+0000 mon.a (mon.0) 657 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[77892]: cluster 2023-10-09T08:54:13.408087+0000 mgr.y (mgr.25421) 498 : cluster 0 pgmap v450: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23192/95058 objects degraded (24.398%); 1269/95058 objects misplaced (1.335%); 2.7 MiB/s, 46 objects/s recovering 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[77892]: audit 2023-10-09T08:54:14.050626+0000 mon.a (mon.0) 654 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[77892]: audit 2023-10-09T08:54:14.059194+0000 mon.a (mon.0) 655 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[77892]: audit 2023-10-09T08:54:14.308833+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.16:0/1743192997' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[77892]: audit 2023-10-09T08:54:14.309586+0000 mon.a (mon.0) 656 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T08:54:15.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[77892]: cluster 2023-10-09T08:54:14.309725+0000 mon.a (mon.0) 657 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T08:54:15.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:15.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:15.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:15 smithi094 bash[59677]: cluster 2023-10-09T08:54:13.408087+0000 mgr.y (mgr.25421) 498 : cluster 0 pgmap v450: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23192/95058 objects degraded (24.398%); 1269/95058 objects misplaced (1.335%); 2.7 MiB/s, 46 objects/s recovering 2023-10-09T08:54:15.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:15 smithi094 bash[59677]: audit 2023-10-09T08:54:14.050626+0000 mon.a (mon.0) 654 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:15.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:15 smithi094 bash[59677]: audit 2023-10-09T08:54:14.059194+0000 mon.a (mon.0) 655 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:15.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:15 smithi094 bash[59677]: audit 2023-10-09T08:54:14.308833+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.16:0/1743192997' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T08:54:15.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:15 smithi094 bash[59677]: audit 2023-10-09T08:54:14.309586+0000 mon.a (mon.0) 656 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T08:54:15.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:15 smithi094 bash[59677]: cluster 2023-10-09T08:54:14.309725+0000 mon.a (mon.0) 657 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T08:54:15.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:15.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:15.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:15.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:15.787 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:15.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T08:54:15.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:15.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:16.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:16.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:16.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:16.165 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:16.250 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:54:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:54:15] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.33.4" 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: audit 2023-10-09T08:54:15.061285+0000 mon.a (mon.0) 658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.061384+0000 mon.a (mon.0) 659 : cluster 0 osdmap e120: 8 total, 8 up, 7 in 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.061601+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.061780+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.062365+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.067160+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.069112+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.072019+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: audit 2023-10-09T08:54:15.061285+0000 mon.a (mon.0) 658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.061384+0000 mon.a (mon.0) 659 : cluster 0 osdmap e120: 8 total, 8 up, 7 in 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.061601+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.061780+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.062365+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.067160+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.069112+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:16 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.072019+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.268 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:16.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: audit 2023-10-09T08:54:15.061285+0000 mon.a (mon.0) 658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T08:54:16.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.061384+0000 mon.a (mon.0) 659 : cluster 0 osdmap e120: 8 total, 8 up, 7 in 2023-10-09T08:54:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.061601+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.061780+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.062365+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.067160+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.069112+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:16 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.072019+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T08:54:16.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:16.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:16.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:16.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:16.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:16.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:16.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:16.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:16.689 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:16.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:16.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:16.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:16.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:16.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:16.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:16.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:17.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:15.409193+0000 mgr.y (mgr.25421) 499 : cluster 0 pgmap v452: 89 pgs: 1 unknown, 6 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 23192/93540 objects degraded (24.794%); 1269/93540 objects misplaced (1.357%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:54:17.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:16.058011+0000 mon.a (mon.0) 660 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T08:54:17.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:16.067258+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:16.067936+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:16.070274+0000 mon.a (mon.0) 661 : cluster 0 osdmap e121: 8 total, 8 up, 7 in 2023-10-09T08:54:17.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:16.070370+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:16.073869+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[77892]: cluster 2023-10-09T08:54:16.077224+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:17.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:17.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:17.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:15.409193+0000 mgr.y (mgr.25421) 499 : cluster 0 pgmap v452: 89 pgs: 1 unknown, 6 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 23192/93540 objects degraded (24.794%); 1269/93540 objects misplaced (1.357%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:54:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:16.058011+0000 mon.a (mon.0) 660 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T08:54:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:16.067258+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:16.067936+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:16.070274+0000 mon.a (mon.0) 661 : cluster 0 osdmap e121: 8 total, 8 up, 7 in 2023-10-09T08:54:17.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:16.070370+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:16.073869+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:17 smithi094 bash[59677]: cluster 2023-10-09T08:54:16.077224+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:17.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:17.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:17.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:15.409193+0000 mgr.y (mgr.25421) 499 : cluster 0 pgmap v452: 89 pgs: 1 unknown, 6 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 23192/93540 objects degraded (24.794%); 1269/93540 objects misplaced (1.357%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T08:54:17.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:16.058011+0000 mon.a (mon.0) 660 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T08:54:17.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:16.067258+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:16.067936+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:16.070274+0000 mon.a (mon.0) 661 : cluster 0 osdmap e121: 8 total, 8 up, 7 in 2023-10-09T08:54:17.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:16.070370+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:16.073869+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:17 smithi016 bash[75179]: cluster 2023-10-09T08:54:16.077224+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T08:54:17.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:17.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:17.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:17.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:17.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:17.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:17.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:17.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:17.717 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:17.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:17.822 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:17.847 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:17.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:17.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:17.980 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:17.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:17.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:18.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:18.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:18.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:18.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:18.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.074088+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.076841+0000 mon.a (mon.0) 662 : cluster 0 osdmap e122: 8 total, 8 up, 7 in 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.076924+0000 osd.3 (osd.3) 1 : cluster 0 6.8 starting backfill to osd.6 from (0'0,0'0] MAX to 119'11009 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.077216+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.077410+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.077865+0000 osd.3 (osd.3) 2 : cluster 0 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 119'10799 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.078319+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.079648+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.080720+0000 osd.4 (osd.4) 55 : cluster 0 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 119'10943 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.081720+0000 osd.0 (osd.0) 1 : cluster 0 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 119'10689 2023-10-09T08:54:18.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.082546+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.084667+0000 osd.1 (osd.1) 1 : cluster 0 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 119'10824 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.088838+0000 osd.6 (osd.6) 46 : cluster 0 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 119'10742 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.088849+0000 osd.6 (osd.6) 47 : cluster 0 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 119'10620 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.088917+0000 osd.6 (osd.6) 48 : cluster 0 6.5 starting backfill to osd.2 from (0'0,0'0] MAX to 119'10744 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.206382+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: audit 2023-10-09T08:54:18.033232+0000 mon.a (mon.0) 663 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[75179]: audit 2023-10-09T08:54:18.046975+0000 mon.a (mon.0) 664 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.074088+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.076841+0000 mon.a (mon.0) 662 : cluster 0 osdmap e122: 8 total, 8 up, 7 in 2023-10-09T08:54:18.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.076924+0000 osd.3 (osd.3) 1 : cluster 0 6.8 starting backfill to osd.6 from (0'0,0'0] MAX to 119'11009 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.077216+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.077410+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.077865+0000 osd.3 (osd.3) 2 : cluster 0 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 119'10799 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.078319+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.079648+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.080720+0000 osd.4 (osd.4) 55 : cluster 0 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 119'10943 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.081720+0000 osd.0 (osd.0) 1 : cluster 0 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 119'10689 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.082546+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.084667+0000 osd.1 (osd.1) 1 : cluster 0 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 119'10824 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.088838+0000 osd.6 (osd.6) 46 : cluster 0 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 119'10742 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.088849+0000 osd.6 (osd.6) 47 : cluster 0 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 119'10620 2023-10-09T08:54:18.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.088917+0000 osd.6 (osd.6) 48 : cluster 0 6.5 starting backfill to osd.2 from (0'0,0'0] MAX to 119'10744 2023-10-09T08:54:18.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.206382+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: audit 2023-10-09T08:54:18.033232+0000 mon.a (mon.0) 663 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:18.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:18 smithi016 bash[77892]: audit 2023-10-09T08:54:18.046975+0000 mon.a (mon.0) 664 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:18.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:18.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:18.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.074088+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.076841+0000 mon.a (mon.0) 662 : cluster 0 osdmap e122: 8 total, 8 up, 7 in 2023-10-09T08:54:18.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.076924+0000 osd.3 (osd.3) 1 : cluster 0 6.8 starting backfill to osd.6 from (0'0,0'0] MAX to 119'11009 2023-10-09T08:54:18.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.077216+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.077410+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.077865+0000 osd.3 (osd.3) 2 : cluster 0 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 119'10799 2023-10-09T08:54:18.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.078319+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.079648+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.080720+0000 osd.4 (osd.4) 55 : cluster 0 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 119'10943 2023-10-09T08:54:18.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.081720+0000 osd.0 (osd.0) 1 : cluster 0 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 119'10689 2023-10-09T08:54:18.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.082546+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.084667+0000 osd.1 (osd.1) 1 : cluster 0 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 119'10824 2023-10-09T08:54:18.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.088838+0000 osd.6 (osd.6) 46 : cluster 0 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 119'10742 2023-10-09T08:54:18.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.088849+0000 osd.6 (osd.6) 47 : cluster 0 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 119'10620 2023-10-09T08:54:18.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.088917+0000 osd.6 (osd.6) 48 : cluster 0 6.5 starting backfill to osd.2 from (0'0,0'0] MAX to 119'10744 2023-10-09T08:54:18.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.206382+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T08:54:18.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: audit 2023-10-09T08:54:18.033232+0000 mon.a (mon.0) 663 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:18.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:18 smithi094 bash[59677]: audit 2023-10-09T08:54:18.046975+0000 mon.a (mon.0) 664 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:18.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:18.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:18.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:18.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:18.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:18.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:18.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:18.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:18.698 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:18.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:18.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:18.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:18.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:18.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:18.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:18.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:19.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:19.134 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:19.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:19.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:19.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[75179]: cluster 2023-10-09T08:54:17.409984+0000 mgr.y (mgr.25421) 500 : cluster 0 pgmap v455: 89 pgs: 1 active+remapped, 63 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 18 peering, 1 unknown, 3 remapped+peering; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8261/88860 objects degraded (9.297%); 1813/88860 objects misplaced (2.040%); 1.9 MiB/s, 34 objects/s recovering 2023-10-09T08:54:19.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[75179]: audit 2023-10-09T08:54:18.623803+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[75179]: audit 2023-10-09T08:54:18.624958+0000 mon.a (mon.0) 666 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[75179]: audit 2023-10-09T08:54:18.637626+0000 mon.a (mon.0) 667 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[75179]: cluster 2023-10-09T08:54:18.957797+0000 mon.a (mon.0) 668 : cluster 3 Health check update: Degraded data redundancy: 8261/88860 objects degraded (9.297%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[77892]: cluster 2023-10-09T08:54:17.409984+0000 mgr.y (mgr.25421) 500 : cluster 0 pgmap v455: 89 pgs: 1 active+remapped, 63 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 18 peering, 1 unknown, 3 remapped+peering; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8261/88860 objects degraded (9.297%); 1813/88860 objects misplaced (2.040%); 1.9 MiB/s, 34 objects/s recovering 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[77892]: audit 2023-10-09T08:54:18.623803+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[77892]: audit 2023-10-09T08:54:18.624958+0000 mon.a (mon.0) 666 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[77892]: audit 2023-10-09T08:54:18.637626+0000 mon.a (mon.0) 667 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:19.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:19 smithi016 bash[77892]: cluster 2023-10-09T08:54:18.957797+0000 mon.a (mon.0) 668 : cluster 3 Health check update: Degraded data redundancy: 8261/88860 objects degraded (9.297%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:19.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:19.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:19.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:19.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:19.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:19.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:19.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:19 smithi094 bash[59677]: cluster 2023-10-09T08:54:17.409984+0000 mgr.y (mgr.25421) 500 : cluster 0 pgmap v455: 89 pgs: 1 active+remapped, 63 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 18 peering, 1 unknown, 3 remapped+peering; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8261/88860 objects degraded (9.297%); 1813/88860 objects misplaced (2.040%); 1.9 MiB/s, 34 objects/s recovering 2023-10-09T08:54:19.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:19 smithi094 bash[59677]: audit 2023-10-09T08:54:18.623803+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:19.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:19 smithi094 bash[59677]: audit 2023-10-09T08:54:18.624958+0000 mon.a (mon.0) 666 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:19.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:19 smithi094 bash[59677]: audit 2023-10-09T08:54:18.637626+0000 mon.a (mon.0) 667 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:19.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:19 smithi094 bash[59677]: cluster 2023-10-09T08:54:18.957797+0000 mon.a (mon.0) 668 : cluster 3 Health check update: Degraded data redundancy: 8261/88860 objects degraded (9.297%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:19.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:19.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:19.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:19.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:19.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:19.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:19.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:19.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:19.810 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:19.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:19.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:19.916 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:19.942 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:19.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:19.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:20.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:20.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:20.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:20.085 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T08:54:20.085 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:54:20.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:20.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:20.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:20.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:20.218 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:20 smithi094 bash[59677]: audit 2023-10-09T08:54:19.215815+0000 mon.a (mon.0) 669 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:20.218 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:20 smithi094 bash[59677]: audit 2023-10-09T08:54:19.219403+0000 mon.a (mon.0) 670 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:20.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:20.317 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:20 smithi016 bash[75179]: audit 2023-10-09T08:54:19.215815+0000 mon.a (mon.0) 669 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:20.317 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:20 smithi016 bash[75179]: audit 2023-10-09T08:54:19.219403+0000 mon.a (mon.0) 670 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:20.318 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:20 smithi016 bash[77892]: audit 2023-10-09T08:54:19.215815+0000 mon.a (mon.0) 669 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:20.318 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:20 smithi016 bash[77892]: audit 2023-10-09T08:54:19.219403+0000 mon.a (mon.0) 670 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:20.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:20.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:20.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:20.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:20.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:20.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:20.581 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:20.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:20.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:20.726 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:20.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:20.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:20.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:20.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:20.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:20.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:20.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:21.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:21.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:21.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:21.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:21.267 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:54:21.267 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:54:21.267 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T08:54:21.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:54:21.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:21.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:21.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:21.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:21 smithi016 bash[75179]: cluster 2023-10-09T08:54:19.410820+0000 mgr.y (mgr.25421) 501 : cluster 0 pgmap v456: 89 pgs: 1 active+remapped, 63 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 18 peering, 1 unknown, 3 remapped+peering; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 8261/88584 objects degraded (9.326%); 1813/88584 objects misplaced (2.047%); 1.7 MiB/s, 31 objects/s recovering 2023-10-09T08:54:21.415 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:21 smithi016 bash[77892]: cluster 2023-10-09T08:54:19.410820+0000 mgr.y (mgr.25421) 501 : cluster 0 pgmap v456: 89 pgs: 1 active+remapped, 63 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 18 peering, 1 unknown, 3 remapped+peering; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 8261/88584 objects degraded (9.326%); 1813/88584 objects misplaced (2.047%); 1.7 MiB/s, 31 objects/s recovering 2023-10-09T08:54:21.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T08:54:21.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:21 smithi094 bash[59677]: cluster 2023-10-09T08:54:19.410820+0000 mgr.y (mgr.25421) 501 : cluster 0 pgmap v456: 89 pgs: 1 active+remapped, 63 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 18 peering, 1 unknown, 3 remapped+peering; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 8261/88584 objects degraded (9.326%); 1813/88584 objects misplaced (2.047%); 1.7 MiB/s, 31 objects/s recovering 2023-10-09T08:54:21.472 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:21.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:21.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:21.636 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:54:21.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:21.670 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:54:21.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:21.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:21.709 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:54:21.738 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:54:21.783 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:54:21.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:21.821 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:54:21.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:21.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:21.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:21.866 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:54:21.893 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:54:21.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.898 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.898 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.898 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.898 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.898 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:54:21.934 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:21.967 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:21.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:22.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:22.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:22.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:22.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:22.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:22 smithi016 bash[75179]: audit 2023-10-09T08:54:21.597520+0000 mon.a (mon.0) 671 : audit 0 from='client.? 172.21.15.16:0/2292571175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:22.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:22 smithi016 bash[77892]: audit 2023-10-09T08:54:21.597520+0000 mon.a (mon.0) 671 : audit 0 from='client.? 172.21.15.16:0/2292571175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:22.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:22 smithi094 bash[59677]: audit 2023-10-09T08:54:21.597520+0000 mon.a (mon.0) 671 : audit 0 from='client.? 172.21.15.16:0/2292571175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:22.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:22.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:22.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:22.928 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:22.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:23.039 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:23.088 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:23.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:23.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:23.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:23.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:23.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:23.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:23.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:23.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:23.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:23 smithi016 bash[75179]: cluster 2023-10-09T08:54:21.411644+0000 mgr.y (mgr.25421) 502 : cluster 0 pgmap v457: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 11605/85479 objects degraded (13.576%); 14225/85479 objects misplaced (16.642%); 3.9 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T08:54:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:23 smithi016 bash[75179]: cluster 2023-10-09T08:54:22.229746+0000 mon.a (mon.0) 672 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T08:54:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:23 smithi016 bash[77892]: cluster 2023-10-09T08:54:21.411644+0000 mgr.y (mgr.25421) 502 : cluster 0 pgmap v457: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 11605/85479 objects degraded (13.576%); 14225/85479 objects misplaced (16.642%); 3.9 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T08:54:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:23 smithi016 bash[77892]: cluster 2023-10-09T08:54:22.229746+0000 mon.a (mon.0) 672 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T08:54:23.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:23.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:23.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:23 smithi094 bash[59677]: cluster 2023-10-09T08:54:21.411644+0000 mgr.y (mgr.25421) 502 : cluster 0 pgmap v457: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 11605/85479 objects degraded (13.576%); 14225/85479 objects misplaced (16.642%); 3.9 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T08:54:23.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:23 smithi094 bash[59677]: cluster 2023-10-09T08:54:22.229746+0000 mon.a (mon.0) 672 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T08:54:23.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:23.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:23.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:23.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:23.795 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:23.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:23.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:23.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:23.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:23.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:23.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:24.040 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:24.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:24.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:24.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:24.184 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:24.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:24.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:24.244 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:24 smithi016 bash[77892]: cluster 2023-10-09T08:54:23.958649+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 11605/85479 objects degraded (13.576%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:24.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:24 smithi016 bash[75179]: cluster 2023-10-09T08:54:23.958649+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 11605/85479 objects degraded (13.576%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:24.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:24.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:24 smithi094 bash[59677]: cluster 2023-10-09T08:54:23.958649+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 11605/85479 objects degraded (13.576%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:24.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:24.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:24.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:24.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:24.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:24.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:24.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:24.632 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:24.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:24.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:24.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:24.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:24.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:24.868 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:24.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:24.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:25.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:25.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:25.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:25.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:25.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:25.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:25.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:25.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:25.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:25.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:25.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:54:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:25.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:25.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:25 smithi016 bash[75179]: cluster 2023-10-09T08:54:23.412422+0000 mgr.y (mgr.25421) 503 : cluster 0 pgmap v458: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 11605/82689 objects degraded (14.035%); 14803/82689 objects misplaced (17.902%); 4.4 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T08:54:25.326 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:25 smithi016 bash[77892]: cluster 2023-10-09T08:54:23.412422+0000 mgr.y (mgr.25421) 503 : cluster 0 pgmap v458: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 11605/82689 objects degraded (14.035%); 14803/82689 objects misplaced (17.902%); 4.4 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T08:54:25.419 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:25.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:25 smithi094 bash[59677]: cluster 2023-10-09T08:54:23.412422+0000 mgr.y (mgr.25421) 503 : cluster 0 pgmap v458: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 11605/82689 objects degraded (14.035%); 14803/82689 objects misplaced (17.902%); 4.4 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T08:54:25.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:25.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:25.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:25.599 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:25.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:25.705 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:25.739 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:25.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:25.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:25.877 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:25.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:25.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:25.989 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:54:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:54:25] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.33.4" 2023-10-09T08:54:26.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:26.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:26.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:26.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:26.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:26.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:26.302 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:54:26.381 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:26.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:26.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:26.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:26.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:26.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:26.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:26.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:26.841 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:26.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:26.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:26.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:27.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:27.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:27.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:27.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:27.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:27.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:27.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:27.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:27.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:27.486 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:27.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:27 smithi016 bash[75179]: cluster 2023-10-09T08:54:25.413184+0000 mgr.y (mgr.25421) 504 : cluster 0 pgmap v459: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 11605/81066 objects degraded (14.315%); 14428/81066 objects misplaced (17.798%); 2.6 MiB/s, 1 keys/s, 60 objects/s recovering 2023-10-09T08:54:27.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:27 smithi016 bash[77892]: cluster 2023-10-09T08:54:25.413184+0000 mgr.y (mgr.25421) 504 : cluster 0 pgmap v459: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 11605/81066 objects degraded (14.315%); 14428/81066 objects misplaced (17.798%); 2.6 MiB/s, 1 keys/s, 60 objects/s recovering 2023-10-09T08:54:27.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:27.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:27 smithi094 bash[59677]: cluster 2023-10-09T08:54:25.413184+0000 mgr.y (mgr.25421) 504 : cluster 0 pgmap v459: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 11605/81066 objects degraded (14.315%); 14428/81066 objects misplaced (17.798%); 2.6 MiB/s, 1 keys/s, 60 objects/s recovering 2023-10-09T08:54:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:27.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:27.553 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:27.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:27.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:27.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:27.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:27.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:27.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:27.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:27.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:27.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:27.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:27.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:28.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:28.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:28.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:28.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:28.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:28.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:28.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:28.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:28.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:28.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:28.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:28.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:28.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:28.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:28.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:28.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:28.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:28.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:28.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:28.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:28.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:28.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:28.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:28.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:28.973 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:29.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:29.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:29.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:29.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:29.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:29.286 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:29.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:29.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:29.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:29.456 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:29 smithi016 bash[75179]: cluster 2023-10-09T08:54:27.414097+0000 mgr.y (mgr.25421) 505 : cluster 0 pgmap v460: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 11130/74904 objects degraded (14.859%); 13592/74904 objects misplaced (18.146%); 5.5 MiB/s, 0 keys/s, 116 objects/s recovering 2023-10-09T08:54:29.457 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:29 smithi016 bash[75179]: cluster 2023-10-09T08:54:28.959774+0000 mon.a (mon.0) 674 : cluster 3 Health check update: Degraded data redundancy: 11130/74904 objects degraded (14.859%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:29.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:29 smithi016 bash[77892]: cluster 2023-10-09T08:54:27.414097+0000 mgr.y (mgr.25421) 505 : cluster 0 pgmap v460: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 11130/74904 objects degraded (14.859%); 13592/74904 objects misplaced (18.146%); 5.5 MiB/s, 0 keys/s, 116 objects/s recovering 2023-10-09T08:54:29.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:29 smithi016 bash[77892]: cluster 2023-10-09T08:54:28.959774+0000 mon.a (mon.0) 674 : cluster 3 Health check update: Degraded data redundancy: 11130/74904 objects degraded (14.859%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:29.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:29.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:29 smithi094 bash[59677]: cluster 2023-10-09T08:54:27.414097+0000 mgr.y (mgr.25421) 505 : cluster 0 pgmap v460: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 11130/74904 objects degraded (14.859%); 13592/74904 objects misplaced (18.146%); 5.5 MiB/s, 0 keys/s, 116 objects/s recovering 2023-10-09T08:54:29.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:29 smithi094 bash[59677]: cluster 2023-10-09T08:54:28.959774+0000 mon.a (mon.0) 674 : cluster 3 Health check update: Degraded data redundancy: 11130/74904 objects degraded (14.859%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:29.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:29.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:29.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:29.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:29.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:29.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:29.900 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:29.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:29.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:29.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:29.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:30.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:30.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:30.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:30.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:30.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:30.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:30.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:30.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:30.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:30.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:30.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:30.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:30.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:30.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:30.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:30.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:30.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:30.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:30.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:30.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:30.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:30.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:30.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:30.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:30.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:30.934 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:30.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:31.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:31.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:31.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:31.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:31.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:31.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:31.282 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T08:54:31.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:31.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:31.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:31.354 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:31 smithi016 bash[75179]: cluster 2023-10-09T08:54:29.414977+0000 mgr.y (mgr.25421) 506 : cluster 0 pgmap v461: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 11130/74409 objects degraded (14.958%); 13592/74409 objects misplaced (18.267%); 4.7 MiB/s, 0 keys/s, 100 objects/s recovering 2023-10-09T08:54:31.354 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:31 smithi016 bash[75179]: audit 2023-10-09T08:54:30.642781+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.16:0/1431917815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:54:31.355 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:31 smithi016 bash[75179]: audit 2023-10-09T08:54:30.643703+0000 mon.a (mon.0) 675 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:54:31.414 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:31.444 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:31.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:31.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:31.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:31.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:31.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:31.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:31.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:31 smithi094 bash[59677]: cluster 2023-10-09T08:54:29.414977+0000 mgr.y (mgr.25421) 506 : cluster 0 pgmap v461: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 11130/74409 objects degraded (14.958%); 13592/74409 objects misplaced (18.267%); 4.7 MiB/s, 0 keys/s, 100 objects/s recovering 2023-10-09T08:54:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:31 smithi094 bash[59677]: audit 2023-10-09T08:54:30.642781+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.16:0/1431917815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:54:31.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:31 smithi094 bash[59677]: audit 2023-10-09T08:54:30.643703+0000 mon.a (mon.0) 675 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:54:31.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:31 smithi016 bash[77892]: cluster 2023-10-09T08:54:29.414977+0000 mgr.y (mgr.25421) 506 : cluster 0 pgmap v461: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 11130/74409 objects degraded (14.958%); 13592/74409 objects misplaced (18.267%); 4.7 MiB/s, 0 keys/s, 100 objects/s recovering 2023-10-09T08:54:31.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:31 smithi016 bash[77892]: audit 2023-10-09T08:54:30.642781+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.16:0/1431917815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:54:31.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:31 smithi016 bash[77892]: audit 2023-10-09T08:54:30.643703+0000 mon.a (mon.0) 675 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:54:31.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:31.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:31.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:31.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:31.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:32.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:32.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:32.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:32.206 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:32.232 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:32.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:32.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[77892]: cluster 2023-10-09T08:54:31.266476+0000 mon.a (mon.0) 676 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:54:32.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[77892]: audit 2023-10-09T08:54:31.276130+0000 mon.a (mon.0) 677 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:54:32.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[77892]: cluster 2023-10-09T08:54:31.276197+0000 mon.a (mon.0) 678 : cluster 0 osdmap e123: 8 total, 8 up, 7 in 2023-10-09T08:54:32.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[77892]: cluster 2023-10-09T08:54:31.276827+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T08:54:32.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[75179]: cluster 2023-10-09T08:54:31.266476+0000 mon.a (mon.0) 676 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:54:32.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[75179]: audit 2023-10-09T08:54:31.276130+0000 mon.a (mon.0) 677 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:54:32.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[75179]: cluster 2023-10-09T08:54:31.276197+0000 mon.a (mon.0) 678 : cluster 0 osdmap e123: 8 total, 8 up, 7 in 2023-10-09T08:54:32.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:32 smithi016 bash[75179]: cluster 2023-10-09T08:54:31.276827+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T08:54:32.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:32.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:32.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:32.389 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:54:32.390 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T08:54:32.390 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T08:54:32.390 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T08:54:32.390 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:54:32.427 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:32 smithi094 bash[59677]: cluster 2023-10-09T08:54:31.266476+0000 mon.a (mon.0) 676 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:54:32.427 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:32 smithi094 bash[59677]: audit 2023-10-09T08:54:31.276130+0000 mon.a (mon.0) 677 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:54:32.427 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:32 smithi094 bash[59677]: cluster 2023-10-09T08:54:31.276197+0000 mon.a (mon.0) 678 : cluster 0 osdmap e123: 8 total, 8 up, 7 in 2023-10-09T08:54:32.427 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:32 smithi094 bash[59677]: cluster 2023-10-09T08:54:31.276827+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T08:54:32.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:32.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:32.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:32.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:32.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:32.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:32.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:32.727 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:32.765 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:32.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:32.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:32.918 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:32.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:32.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:33.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:33.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:33.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:33.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:33.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:33.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:33.322 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:33.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:33.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:33 smithi016 bash[75179]: cluster 2023-10-09T08:54:31.415544+0000 mgr.y (mgr.25421) 507 : cluster 0 pgmap v463: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10653/69480 objects degraded (15.332%); 12761/69480 objects misplaced (18.366%); 6.2 MiB/s, 128 objects/s recovering 2023-10-09T08:54:33.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:33 smithi016 bash[77892]: cluster 2023-10-09T08:54:31.415544+0000 mgr.y (mgr.25421) 507 : cluster 0 pgmap v463: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10653/69480 objects degraded (15.332%); 12761/69480 objects misplaced (18.366%); 6.2 MiB/s, 128 objects/s recovering 2023-10-09T08:54:33.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:33 smithi094 bash[59677]: cluster 2023-10-09T08:54:31.415544+0000 mgr.y (mgr.25421) 507 : cluster 0 pgmap v463: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10653/69480 objects degraded (15.332%); 12761/69480 objects misplaced (18.366%); 6.2 MiB/s, 128 objects/s recovering 2023-10-09T08:54:33.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:33.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:33.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:33.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:33.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T08:54:33.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:33.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:33.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:33.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:33.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:33.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:34.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:34.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:34.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:34.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:34.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:34.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:34.356 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:34.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:34.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:34.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:34 smithi094 bash[59677]: cluster 2023-10-09T08:54:33.960677+0000 mon.a (mon.0) 679 : cluster 3 Health check update: Degraded data redundancy: 10653/69480 objects degraded (15.332%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:34 smithi094 bash[59677]: audit 2023-10-09T08:54:34.219537+0000 mon.a (mon.0) 680 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:34 smithi094 bash[59677]: audit 2023-10-09T08:54:34.221061+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:34.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:34 smithi016 bash[75179]: cluster 2023-10-09T08:54:33.960677+0000 mon.a (mon.0) 679 : cluster 3 Health check update: Degraded data redundancy: 10653/69480 objects degraded (15.332%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:34.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:34 smithi016 bash[75179]: audit 2023-10-09T08:54:34.219537+0000 mon.a (mon.0) 680 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:34.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:34 smithi016 bash[75179]: audit 2023-10-09T08:54:34.221061+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:34.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:34 smithi016 bash[77892]: cluster 2023-10-09T08:54:33.960677+0000 mon.a (mon.0) 679 : cluster 3 Health check update: Degraded data redundancy: 10653/69480 objects degraded (15.332%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:34.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:34 smithi016 bash[77892]: audit 2023-10-09T08:54:34.219537+0000 mon.a (mon.0) 680 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:54:34.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:34 smithi016 bash[77892]: audit 2023-10-09T08:54:34.221061+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:34.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:34.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:34.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:34.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:34.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:34.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:34.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:34.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:34.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:34.928 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:34.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:35.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:35.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:35.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:35.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:35.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:35.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:35.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:35.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:35.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:35.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:35.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:35.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:35.470 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:35 smithi094 bash[59677]: cluster 2023-10-09T08:54:33.416222+0000 mgr.y (mgr.25421) 508 : cluster 0 pgmap v464: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 10653/66696 objects degraded (15.972%); 12414/66696 objects misplaced (18.613%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:35 smithi016 bash[75179]: cluster 2023-10-09T08:54:33.416222+0000 mgr.y (mgr.25421) 508 : cluster 0 pgmap v464: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 10653/66696 objects degraded (15.972%); 12414/66696 objects misplaced (18.613%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:35 smithi016 bash[77892]: cluster 2023-10-09T08:54:33.416222+0000 mgr.y (mgr.25421) 508 : cluster 0 pgmap v464: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 10653/66696 objects degraded (15.972%); 12414/66696 objects misplaced (18.613%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:35.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:35.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:35.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:35.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:35.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:35.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:35.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:35.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:35.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:35.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:35.878 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:54:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:54:35] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.33.4" 2023-10-09T08:54:35.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:35.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:36.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:36.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:36.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:36.189 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:36.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:36.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:36.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:36.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:36.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:36.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:36.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:36.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:36.694 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:36.719 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:36.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:36.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:36.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:36.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T08:54:36.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:36.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:36.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:36.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:37.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:37.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:37.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:37.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:37.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:37.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:37.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:37.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:37.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:37.414 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:37.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:37.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:37.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:37.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:37.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:37.652 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:37 smithi094 bash[59677]: cluster 2023-10-09T08:54:35.416783+0000 mgr.y (mgr.25421) 509 : cluster 0 pgmap v465: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10653/65061 objects degraded (16.374%); 12054/65061 objects misplaced (18.527%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:37.677 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:37.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:37.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:37.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:37 smithi016 bash[75179]: cluster 2023-10-09T08:54:35.416783+0000 mgr.y (mgr.25421) 509 : cluster 0 pgmap v465: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10653/65061 objects degraded (16.374%); 12054/65061 objects misplaced (18.527%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:37.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:37 smithi016 bash[77892]: cluster 2023-10-09T08:54:35.416783+0000 mgr.y (mgr.25421) 509 : cluster 0 pgmap v465: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10653/65061 objects degraded (16.374%); 12054/65061 objects misplaced (18.527%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:37.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:37.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:37.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:37.918 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:37.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:37.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:38.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:38.083 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:38.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:38.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:38.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:38.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:38.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:38.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:38.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:38.353 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:38.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:38.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:38.456 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:38.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:38.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:38.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:38.529 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:54:38.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:38.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:38.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:38.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:38.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:38.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:38.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:38.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:38.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:38.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:39.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:39.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:39.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:39.183 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:39.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:39.209 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:54:39.223 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:54:39.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:39.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:39.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:39.427 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:39.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:39 smithi094 bash[59677]: cluster 2023-10-09T08:54:37.417694+0000 mgr.y (mgr.25421) 510 : cluster 0 pgmap v466: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 10170/59004 objects degraded (17.236%); 11243/59004 objects misplaced (19.055%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:39.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:39 smithi094 bash[59677]: cluster 2023-10-09T08:54:38.961816+0000 mon.a (mon.0) 682 : cluster 3 Health check update: Degraded data redundancy: 10170/59004 objects degraded (17.236%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:39.464 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:39.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:39.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:39.579 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:39 smithi016 bash[75179]: cluster 2023-10-09T08:54:37.417694+0000 mgr.y (mgr.25421) 510 : cluster 0 pgmap v466: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 10170/59004 objects degraded (17.236%); 11243/59004 objects misplaced (19.055%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:39.580 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:39 smithi016 bash[75179]: cluster 2023-10-09T08:54:38.961816+0000 mon.a (mon.0) 682 : cluster 3 Health check update: Degraded data redundancy: 10170/59004 objects degraded (17.236%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:39.585 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:39 smithi016 bash[77892]: cluster 2023-10-09T08:54:37.417694+0000 mgr.y (mgr.25421) 510 : cluster 0 pgmap v466: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 10170/59004 objects degraded (17.236%); 11243/59004 objects misplaced (19.055%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:39.585 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:39 smithi016 bash[77892]: cluster 2023-10-09T08:54:38.961816+0000 mon.a (mon.0) 682 : cluster 3 Health check update: Degraded data redundancy: 10170/59004 objects degraded (17.236%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:39.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:39.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:39.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:39.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:39.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:39.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:39.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:39.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:39.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:40.099 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:40.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:40.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:40.242 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:40.275 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:40.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:40.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:40.389 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:40 smithi094 bash[59677]: audit 2023-10-09T08:54:39.195346+0000 mgr.y (mgr.25421) 511 : audit 0 from='client.44840 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:54:40.411 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:40.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:40.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:40.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:40.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:40.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:40.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:40 smithi016 bash[75179]: audit 2023-10-09T08:54:39.195346+0000 mgr.y (mgr.25421) 511 : audit 0 from='client.44840 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:54:40.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:40 smithi016 bash[77892]: audit 2023-10-09T08:54:39.195346+0000 mgr.y (mgr.25421) 511 : audit 0 from='client.44840 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:54:40.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:40.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:40.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:40.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:40.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:40.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:40.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:40.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:40.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:41.060 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:41.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:41.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:41.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:41.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:41.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:41.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:41.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:41.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:41.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:41.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:41.477 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:41 smithi094 bash[59677]: cluster 2023-10-09T08:54:39.418261+0000 mgr.y (mgr.25421) 512 : cluster 0 pgmap v467: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 10170/58482 objects degraded (17.390%); 11243/58482 objects misplaced (19.225%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:41.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:41 smithi016 bash[75179]: cluster 2023-10-09T08:54:39.418261+0000 mgr.y (mgr.25421) 512 : cluster 0 pgmap v467: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 10170/58482 objects degraded (17.390%); 11243/58482 objects misplaced (19.225%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:41.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:41 smithi016 bash[77892]: cluster 2023-10-09T08:54:39.418261+0000 mgr.y (mgr.25421) 512 : cluster 0 pgmap v467: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 10170/58482 objects degraded (17.390%); 11243/58482 objects misplaced (19.225%); 6.3 MiB/s, 129 objects/s recovering 2023-10-09T08:54:41.504 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:41.509 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":466,"stamp":"2023-10-09T08:54:37.417055+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1287919544,"num_objects":19668,"num_object_clones":0,"num_object_copies":59004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10170,"num_objects_misplaced":11243,"num_objects_unfound":0,"num_objects_dirty":19668,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":182866,"num_write_kb":6474382,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7986,"num_bytes_recovered":480580992,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120862,"ondisk_log_size":120862,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6411188,"kb_used_data":5053436,"kb_used_omap":126,"kb_used_meta":1357505,"kb_avail":649804876,"statfs":{"total":671965249536,"available":665400193024,"internally_reserved":0,"allocated":5174718464,"data_stored":5171117379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1390085755},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,7,6,5,9,8,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-347340800,"num_objects":-5300,"num_object_clones":0,"num_object_copies":-15900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-960,"num_objects_misplaced":-2349,"num_objects_unfound":0,"num_objects_dirty":-5300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5300,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1299,"num_bytes_recovered":65798144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003641"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":106,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080819+0000","last_change":"2023-10-09T08:46:49.233483+0000","last_active":"2023-10-09T08:54:17.080819+0000","last_peered":"2023-10-09T08:54:17.080819+0000","last_clean":"2023-10-09T08:54:17.080819+0000","last_became_active":"2023-10-09T08:46:49.233033+0000","last_became_peered":"2023-10-09T08:46:49.233033+0000","last_unstale":"2023-10-09T08:54:17.080819+0000","last_undegraded":"2023-10-09T08:54:17.080819+0000","last_fullsized":"2023-10-09T08:54:17.080819+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":93,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.077373+0000","last_change":"2023-10-09T08:47:16.965051+0000","last_active":"2023-10-09T08:54:17.077373+0000","last_peered":"2023-10-09T08:54:17.077373+0000","last_clean":"2023-10-09T08:54:17.077373+0000","last_became_active":"2023-10-09T08:47:16.964664+0000","last_became_peered":"2023-10-09T08:47:16.964664+0000","last_unstale":"2023-10-09T08:54:17.077373+0000","last_undegraded":"2023-10-09T08:54:17.077373+0000","last_fullsized":"2023-10-09T08:54:17.077373+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.377202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.084415+0000","last_change":"2023-10-09T08:47:16.968800+0000","last_active":"2023-10-09T08:54:17.084415+0000","last_peered":"2023-10-09T08:54:17.084415+0000","last_clean":"2023-10-09T08:54:17.084415+0000","last_became_active":"2023-10-09T08:47:16.968430+0000","last_became_peered":"2023-10-09T08:47:16.968430+0000","last_unstale":"2023-10-09T08:54:17.084415+0000","last_undegraded":"2023-10-09T08:54:17.084415+0000","last_fullsized":"2023-10-09T08:54:17.084415+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:08.011498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":102,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080963+0000","last_change":"2023-10-09T08:54:16.096082+0000","last_active":"2023-10-09T08:54:17.080963+0000","last_peered":"2023-10-09T08:54:17.080963+0000","last_clean":"2023-10-09T08:54:17.080963+0000","last_became_active":"2023-10-09T08:54:16.095715+0000","last_became_peered":"2023-10-09T08:54:16.095715+0000","last_unstale":"2023-10-09T08:54:17.080963+0000","last_undegraded":"2023-10-09T08:54:17.080963+0000","last_fullsized":"2023-10-09T08:54:17.080963+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:40.399356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:30.314836+0000","last_change":"2023-10-09T08:47:16.965460+0000","last_active":"2023-10-09T08:54:30.314836+0000","last_peered":"2023-10-09T08:54:30.314836+0000","last_clean":"2023-10-09T08:54:30.314836+0000","last_became_active":"2023-10-09T08:47:16.965040+0000","last_became_peered":"2023-10-09T08:47:16.965040+0000","last_unstale":"2023-10-09T08:54:30.314836+0000","last_undegraded":"2023-10-09T08:54:30.314836+0000","last_fullsized":"2023-10-09T08:54:30.314836+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.025735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.089437+0000","last_change":"2023-10-09T08:46:18.680365+0000","last_active":"2023-10-09T08:54:17.089437+0000","last_peered":"2023-10-09T08:54:17.089437+0000","last_clean":"2023-10-09T08:54:17.089437+0000","last_became_active":"2023-10-09T08:46:18.679875+0000","last_became_peered":"2023-10-09T08:46:18.679875+0000","last_unstale":"2023-10-09T08:54:17.089437+0000","last_undegraded":"2023-10-09T08:54:17.089437+0000","last_fullsized":"2023-10-09T08:54:17.089437+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":83,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080780+0000","last_change":"2023-10-09T08:47:16.971965+0000","last_active":"2023-10-09T08:54:17.080780+0000","last_peered":"2023-10-09T08:54:17.080780+0000","last_clean":"2023-10-09T08:54:17.080780+0000","last_became_active":"2023-10-09T08:47:16.971651+0000","last_became_peered":"2023-10-09T08:47:16.971651+0000","last_unstale":"2023-10-09T08:54:17.080780+0000","last_undegraded":"2023-10-09T08:54:17.080780+0000","last_fullsized":"2023-10-09T08:54:17.080780+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:04.259603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":94,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.076488+0000","last_change":"2023-10-09T08:47:16.969948+0000","last_active":"2023-10-09T08:54:17.076488+0000","last_peered":"2023-10-09T08:54:17.076488+0000","last_clean":"2023-10-09T08:54:17.076488+0000","last_became_active":"2023-10-09T08:47:16.969666+0000","last_became_peered":"2023-10-09T08:47:16.969666+0000","last_unstale":"2023-10-09T08:54:17.076488+0000","last_undegraded":"2023-10-09T08:54:17.076488+0000","last_fullsized":"2023-10-09T08:54:17.076488+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:25.089633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.090601+0000","last_change":"2023-10-09T08:54:16.096551+0000","last_active":"2023-10-09T08:54:17.090601+0000","last_peered":"2023-10-09T08:54:17.090601+0000","last_clean":"2023-10-09T08:54:17.090601+0000","last_became_active":"2023-10-09T08:54:16.095982+0000","last_became_peered":"2023-10-09T08:54:16.095982+0000","last_unstale":"2023-10-09T08:54:17.090601+0000","last_undegraded":"2023-10-09T08:54:17.090601+0000","last_fullsized":"2023-10-09T08:54:17.090601+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:58.386123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.076487+0000","last_change":"2023-10-09T08:54:16.090726+0000","last_active":"2023-10-09T08:54:17.076487+0000","last_peered":"2023-10-09T08:54:17.076487+0000","last_clean":"2023-10-09T08:54:17.076487+0000","last_became_active":"2023-10-09T08:54:16.090390+0000","last_became_peered":"2023-10-09T08:54:16.090390+0000","last_unstale":"2023-10-09T08:54:17.076487+0000","last_undegraded":"2023-10-09T08:54:17.076487+0000","last_fullsized":"2023-10-09T08:54:17.076487+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:52.968772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":31,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.077250+0000","last_change":"2023-10-09T08:54:16.100639+0000","last_active":"2023-10-09T08:54:17.077250+0000","last_peered":"2023-10-09T08:54:17.077250+0000","last_clean":"2023-10-09T08:54:17.077250+0000","last_became_active":"2023-10-09T08:54:16.085401+0000","last_became_peered":"2023-10-09T08:54:16.085401+0000","last_unstale":"2023-10-09T08:54:17.077250+0000","last_undegraded":"2023-10-09T08:54:17.077250+0000","last_fullsized":"2023-10-09T08:54:17.077250+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.238266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":79,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.090672+0000","last_change":"2023-10-09T08:46:49.231880+0000","last_active":"2023-10-09T08:54:17.090672+0000","last_peered":"2023-10-09T08:54:17.090672+0000","last_clean":"2023-10-09T08:54:17.090672+0000","last_became_active":"2023-10-09T08:46:49.231471+0000","last_became_peered":"2023-10-09T08:46:49.231471+0000","last_unstale":"2023-10-09T08:54:17.090672+0000","last_undegraded":"2023-10-09T08:54:17.090672+0000","last_fullsized":"2023-10-09T08:54:17.090672+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:31.742745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":116,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081314+0000","last_change":"2023-10-09T08:54:16.167494+0000","last_active":"2023-10-09T08:54:17.081314+0000","last_peered":"2023-10-09T08:54:17.081314+0000","last_clean":"2023-10-09T08:54:17.081314+0000","last_became_active":"2023-10-09T08:54:16.098095+0000","last_became_peered":"2023-10-09T08:54:16.098095+0000","last_unstale":"2023-10-09T08:54:17.081314+0000","last_undegraded":"2023-10-09T08:54:17.081314+0000","last_fullsized":"2023-10-09T08:54:17.081314+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081290+0000","last_change":"2023-10-09T08:46:18.676902+0000","last_active":"2023-10-09T08:54:17.081290+0000","last_peered":"2023-10-09T08:54:17.081290+0000","last_clean":"2023-10-09T08:54:17.081290+0000","last_became_active":"2023-10-09T08:46:18.676560+0000","last_became_peered":"2023-10-09T08:46:18.676560+0000","last_unstale":"2023-10-09T08:54:17.081290+0000","last_undegraded":"2023-10-09T08:54:17.081290+0000","last_fullsized":"2023-10-09T08:54:17.081290+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":31,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081397+0000","last_change":"2023-10-09T08:54:16.254300+0000","last_active":"2023-10-09T08:54:17.081397+0000","last_peered":"2023-10-09T08:54:17.081397+0000","last_clean":"2023-10-09T08:54:17.081397+0000","last_became_active":"2023-10-09T08:54:16.098089+0000","last_became_peered":"2023-10-09T08:54:16.098089+0000","last_unstale":"2023-10-09T08:54:17.081397+0000","last_undegraded":"2023-10-09T08:54:17.081397+0000","last_fullsized":"2023-10-09T08:54:17.081397+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.077154+0000","last_change":"2023-10-09T08:54:16.087143+0000","last_active":"2023-10-09T08:54:17.077154+0000","last_peered":"2023-10-09T08:54:17.077154+0000","last_clean":"2023-10-09T08:54:17.077154+0000","last_became_active":"2023-10-09T08:54:16.086741+0000","last_became_peered":"2023-10-09T08:54:16.086741+0000","last_unstale":"2023-10-09T08:54:17.077154+0000","last_undegraded":"2023-10-09T08:54:17.077154+0000","last_fullsized":"2023-10-09T08:54:17.077154+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:53.402285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":101,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081230+0000","last_change":"2023-10-09T08:54:16.088635+0000","last_active":"2023-10-09T08:54:17.081230+0000","last_peered":"2023-10-09T08:54:17.081230+0000","last_clean":"2023-10-09T08:54:17.081230+0000","last_became_active":"2023-10-09T08:54:16.088295+0000","last_became_peered":"2023-10-09T08:54:16.088295+0000","last_unstale":"2023-10-09T08:54:17.081230+0000","last_undegraded":"2023-10-09T08:54:17.081230+0000","last_fullsized":"2023-10-09T08:54:17.081230+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:44.023596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.077604+0000","last_change":"2023-10-09T08:54:16.098961+0000","last_active":"2023-10-09T08:54:17.077604+0000","last_peered":"2023-10-09T08:54:17.077604+0000","last_clean":"2023-10-09T08:54:17.077604+0000","last_became_active":"2023-10-09T08:54:16.098689+0000","last_became_peered":"2023-10-09T08:54:16.098689+0000","last_unstale":"2023-10-09T08:54:17.077604+0000","last_undegraded":"2023-10-09T08:54:17.077604+0000","last_fullsized":"2023-10-09T08:54:17.077604+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:59.855841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081287+0000","last_change":"2023-10-09T08:54:16.092507+0000","last_active":"2023-10-09T08:54:17.081287+0000","last_peered":"2023-10-09T08:54:17.081287+0000","last_clean":"2023-10-09T08:54:17.081287+0000","last_became_active":"2023-10-09T08:54:16.092206+0000","last_became_peered":"2023-10-09T08:54:16.092206+0000","last_unstale":"2023-10-09T08:54:17.081287+0000","last_undegraded":"2023-10-09T08:54:17.081287+0000","last_fullsized":"2023-10-09T08:54:17.081287+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080896+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T08:54:17.080896+0000","last_peered":"2023-10-09T08:54:17.080896+0000","last_clean":"2023-10-09T08:54:17.080896+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T08:54:17.080896+0000","last_undegraded":"2023-10-09T08:54:17.080896+0000","last_fullsized":"2023-10-09T08:54:17.080896+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":136,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.090235+0000","last_change":"2023-10-09T08:51:53.462296+0000","last_active":"2023-10-09T08:54:17.090235+0000","last_peered":"2023-10-09T08:54:17.090235+0000","last_clean":"2023-10-09T08:54:17.090235+0000","last_became_active":"2023-10-09T08:51:53.454854+0000","last_became_peered":"2023-10-09T08:51:53.454854+0000","last_unstale":"2023-10-09T08:54:17.090235+0000","last_undegraded":"2023-10-09T08:54:17.090235+0000","last_fullsized":"2023-10-09T08:54:17.090235+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":70,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.092195+0000","last_change":"2023-10-09T08:54:16.094203+0000","last_active":"2023-10-09T08:54:17.092195+0000","last_peered":"2023-10-09T08:54:17.092195+0000","last_clean":"2023-10-09T08:54:17.092195+0000","last_became_active":"2023-10-09T08:54:16.090060+0000","last_became_peered":"2023-10-09T08:54:16.090060+0000","last_unstale":"2023-10-09T08:54:17.092195+0000","last_undegraded":"2023-10-09T08:54:17.092195+0000","last_fullsized":"2023-10-09T08:54:17.092195+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:17.599914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"123'11141","reported_seq":11184,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:35.708770+0000","last_change":"2023-10-09T08:53:53.430894+0000","last_active":"2023-10-09T08:54:35.708770+0000","last_peered":"2023-10-09T08:54:35.708770+0000","last_clean":"2023-10-09T08:54:35.708770+0000","last_became_active":"2023-10-09T08:53:53.380838+0000","last_became_peered":"2023-10-09T08:53:53.380838+0000","last_unstale":"2023-10-09T08:54:35.708770+0000","last_undegraded":"2023-10-09T08:54:35.708770+0000","last_fullsized":"2023-10-09T08:54:35.708770+0000","mapping_epoch":115,"log_start":"102'4161","ondisk_log_start":"102'4161","created":95,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79233024,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11141,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":29,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.082144+0000","last_change":"2023-10-09T08:54:16.151974+0000","last_active":"2023-10-09T08:54:17.082144+0000","last_peered":"2023-10-09T08:54:17.082144+0000","last_clean":"2023-10-09T08:54:17.082144+0000","last_became_active":"2023-10-09T08:54:16.091029+0000","last_became_peered":"2023-10-09T08:54:16.091029+0000","last_unstale":"2023-10-09T08:54:17.082144+0000","last_undegraded":"2023-10-09T08:54:17.082144+0000","last_fullsized":"2023-10-09T08:54:17.082144+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.089858+0000","last_change":"2023-10-09T08:54:16.092937+0000","last_active":"2023-10-09T08:54:17.089858+0000","last_peered":"2023-10-09T08:54:17.089858+0000","last_clean":"2023-10-09T08:54:17.089858+0000","last_became_active":"2023-10-09T08:54:16.090574+0000","last_became_peered":"2023-10-09T08:54:16.090574+0000","last_unstale":"2023-10-09T08:54:17.089858+0000","last_undegraded":"2023-10-09T08:54:17.089858+0000","last_fullsized":"2023-10-09T08:54:17.089858+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.b","version":"123'11433","reported_seq":11480,"reported_epoch":123,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T08:54:35.320202+0000","last_change":"2023-10-09T08:54:17.355797+0000","last_active":"2023-10-09T08:54:35.320202+0000","last_peered":"2023-10-09T08:54:35.320202+0000","last_clean":"2023-10-09T08:54:15.050128+0000","last_became_active":"2023-10-09T08:54:17.305178+0000","last_became_peered":"2023-10-09T08:54:17.305178+0000","last_unstale":"2023-10-09T08:54:35.320202+0000","last_undegraded":"2023-10-09T08:54:35.320202+0000","last_fullsized":"2023-10-09T08:54:35.320202+0000","mapping_epoch":121,"log_start":"101'2576","ondisk_log_start":"101'2576","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:49.527145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1269,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11433,"num_write_kb":406464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":1269}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":29,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.290304+0000","last_change":"2023-10-09T08:54:16.158765+0000","last_active":"2023-10-09T08:54:17.290304+0000","last_peered":"2023-10-09T08:54:17.290304+0000","last_clean":"2023-10-09T08:54:17.290304+0000","last_became_active":"2023-10-09T08:54:16.096769+0000","last_became_peered":"2023-10-09T08:54:16.096769+0000","last_unstale":"2023-10-09T08:54:17.290304+0000","last_undegraded":"2023-10-09T08:54:17.290304+0000","last_fullsized":"2023-10-09T08:54:17.290304+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":122,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081538+0000","last_change":"2023-10-09T08:51:33.439973+0000","last_active":"2023-10-09T08:54:17.081538+0000","last_peered":"2023-10-09T08:54:17.081538+0000","last_clean":"2023-10-09T08:54:17.081538+0000","last_became_active":"2023-10-09T08:51:33.439461+0000","last_became_peered":"2023-10-09T08:51:33.439461+0000","last_unstale":"2023-10-09T08:54:17.081538+0000","last_undegraded":"2023-10-09T08:54:17.081538+0000","last_fullsized":"2023-10-09T08:54:17.081538+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"123'11661","reported_seq":11732,"reported_epoch":123,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T08:54:35.255689+0000","last_change":"2023-10-09T08:54:17.496390+0000","last_active":"2023-10-09T08:54:35.255689+0000","last_peered":"2023-10-09T08:54:35.255689+0000","last_clean":"2023-10-09T08:54:15.020588+0000","last_became_active":"2023-10-09T08:54:17.279775+0000","last_became_peered":"2023-10-09T08:54:17.279775+0000","last_unstale":"2023-10-09T08:54:35.255689+0000","last_undegraded":"2023-10-09T08:54:35.255689+0000","last_fullsized":"2023-10-09T08:54:35.255689+0000","mapping_epoch":121,"log_start":"102'2900","ondisk_log_start":"102'2900","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8761,"log_dups_size":0,"ondisk_log_size":8761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:36.084761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1304,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11661,"num_write_kb":415104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1311}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":29,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.144746+0000","last_change":"2023-10-09T08:54:16.124228+0000","last_active":"2023-10-09T08:54:17.144746+0000","last_peered":"2023-10-09T08:54:17.144746+0000","last_clean":"2023-10-09T08:54:17.144746+0000","last_became_active":"2023-10-09T08:54:16.094739+0000","last_became_peered":"2023-10-09T08:54:16.094739+0000","last_unstale":"2023-10-09T08:54:17.144746+0000","last_undegraded":"2023-10-09T08:54:17.144746+0000","last_fullsized":"2023-10-09T08:54:17.144746+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":107,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.089517+0000","last_change":"2023-10-09T08:54:16.092901+0000","last_active":"2023-10-09T08:54:17.089517+0000","last_peered":"2023-10-09T08:54:17.089517+0000","last_clean":"2023-10-09T08:54:17.089517+0000","last_became_active":"2023-10-09T08:54:16.090513+0000","last_became_peered":"2023-10-09T08:54:16.090513+0000","last_unstale":"2023-10-09T08:54:17.089517+0000","last_undegraded":"2023-10-09T08:54:17.089517+0000","last_fullsized":"2023-10-09T08:54:17.089517+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"123'11366","reported_seq":11412,"reported_epoch":123,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T08:54:37.309041+0000","last_change":"2023-10-09T08:54:17.340952+0000","last_active":"2023-10-09T08:54:37.309041+0000","last_peered":"2023-10-09T08:54:37.309041+0000","last_clean":"2023-10-09T08:54:14.996820+0000","last_became_active":"2023-10-09T08:54:17.285103+0000","last_became_peered":"2023-10-09T08:54:17.285103+0000","last_unstale":"2023-10-09T08:54:37.309041+0000","last_undegraded":"2023-10-09T08:54:37.309041+0000","last_fullsized":"2023-10-09T08:54:37.309041+0000","mapping_epoch":121,"log_start":"102'2956","ondisk_log_start":"102'2956","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8410,"log_dups_size":0,"ondisk_log_size":8410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:25.779589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74448896,"num_objects":1136,"num_object_clones":0,"num_object_copies":3408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1136,"num_objects_unfound":0,"num_objects_dirty":1136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11366,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1136}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.077452+0000","last_change":"2023-10-09T08:53:53.386479+0000","last_active":"2023-10-09T08:54:17.077452+0000","last_peered":"2023-10-09T08:54:17.077452+0000","last_clean":"2023-10-09T08:54:17.077452+0000","last_became_active":"2023-10-09T08:53:53.385989+0000","last_became_peered":"2023-10-09T08:53:53.385989+0000","last_unstale":"2023-10-09T08:54:17.077452+0000","last_undegraded":"2023-10-09T08:54:17.077452+0000","last_fullsized":"2023-10-09T08:54:17.077452+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:10.062407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":83,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080382+0000","last_change":"2023-10-09T08:46:18.683057+0000","last_active":"2023-10-09T08:54:17.080382+0000","last_peered":"2023-10-09T08:54:17.080382+0000","last_clean":"2023-10-09T08:54:17.080382+0000","last_became_active":"2023-10-09T08:46:18.682672+0000","last_became_peered":"2023-10-09T08:46:18.682672+0000","last_unstale":"2023-10-09T08:54:17.080382+0000","last_undegraded":"2023-10-09T08:54:17.080382+0000","last_fullsized":"2023-10-09T08:54:17.080382+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:42.975534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"123'11385","reported_seq":13661,"reported_epoch":123,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T08:54:35.473916+0000","last_change":"2023-10-09T08:54:16.255574+0000","last_active":"2023-10-09T08:54:35.473916+0000","last_peered":"2023-10-09T08:54:35.473916+0000","last_clean":"2023-10-09T08:51:32.426090+0000","last_became_active":"2023-10-09T08:51:34.579869+0000","last_became_peered":"2023-10-09T08:51:34.579869+0000","last_unstale":"2023-10-09T08:54:35.473916+0000","last_undegraded":"2023-10-09T08:51:53.462339+0000","last_fullsized":"2023-10-09T08:51:34.434299+0000","mapping_epoch":105,"log_start":"102'4216","ondisk_log_start":"102'4216","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7169,"log_dups_size":0,"ondisk_log_size":7169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79101952,"num_objects":1207,"num_object_clones":0,"num_object_copies":3621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11385,"num_write_kb":402944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1114,"num_bytes_recovered":36044800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,4","objects":-2116}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":93,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.076861+0000","last_change":"2023-10-09T08:47:16.965045+0000","last_active":"2023-10-09T08:54:17.076861+0000","last_peered":"2023-10-09T08:54:17.076861+0000","last_clean":"2023-10-09T08:54:17.076861+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:54:17.076861+0000","last_undegraded":"2023-10-09T08:54:17.076861+0000","last_fullsized":"2023-10-09T08:54:17.076861+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:49.126847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081632+0000","last_change":"2023-10-09T08:47:16.972109+0000","last_active":"2023-10-09T08:54:17.081632+0000","last_peered":"2023-10-09T08:54:17.081632+0000","last_clean":"2023-10-09T08:54:17.081632+0000","last_became_active":"2023-10-09T08:47:16.971716+0000","last_became_peered":"2023-10-09T08:47:16.971716+0000","last_unstale":"2023-10-09T08:54:17.081632+0000","last_undegraded":"2023-10-09T08:54:17.081632+0000","last_fullsized":"2023-10-09T08:54:17.081632+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"123'11241","reported_seq":11279,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:32.448918+0000","last_change":"2023-10-09T08:50:24.945100+0000","last_active":"2023-10-09T08:54:32.448918+0000","last_peered":"2023-10-09T08:54:32.448918+0000","last_clean":"2023-10-09T08:54:32.448918+0000","last_became_active":"2023-10-09T08:50:24.944761+0000","last_became_peered":"2023-10-09T08:50:24.944761+0000","last_unstale":"2023-10-09T08:54:32.448918+0000","last_undegraded":"2023-10-09T08:54:32.448918+0000","last_fullsized":"2023-10-09T08:54:32.448918+0000","mapping_epoch":95,"log_start":"101'2090","ondisk_log_start":"101'2090","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":9151,"log_dups_size":0,"ondisk_log_size":9151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:11.053008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11241,"num_write_kb":403008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":90,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.089346+0000","last_change":"2023-10-09T08:46:18.681222+0000","last_active":"2023-10-09T08:54:17.089346+0000","last_peered":"2023-10-09T08:54:17.089346+0000","last_clean":"2023-10-09T08:54:17.089346+0000","last_became_active":"2023-10-09T08:46:18.680710+0000","last_became_peered":"2023-10-09T08:46:18.680710+0000","last_unstale":"2023-10-09T08:54:17.089346+0000","last_undegraded":"2023-10-09T08:54:17.089346+0000","last_fullsized":"2023-10-09T08:54:17.089346+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":104,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081801+0000","last_change":"2023-10-09T08:46:49.234714+0000","last_active":"2023-10-09T08:54:17.081801+0000","last_peered":"2023-10-09T08:54:17.081801+0000","last_clean":"2023-10-09T08:54:17.081801+0000","last_became_active":"2023-10-09T08:46:49.234307+0000","last_became_peered":"2023-10-09T08:46:49.234307+0000","last_unstale":"2023-10-09T08:54:17.081801+0000","last_undegraded":"2023-10-09T08:54:17.081801+0000","last_fullsized":"2023-10-09T08:54:17.081801+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"123'11254","reported_seq":13036,"reported_epoch":123,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T08:54:35.731380+0000","last_change":"2023-10-09T08:54:17.406963+0000","last_active":"2023-10-09T08:54:35.731380+0000","last_peered":"2023-10-09T08:54:35.731380+0000","last_clean":"2023-10-09T08:51:52.369065+0000","last_became_active":"2023-10-09T08:54:17.355379+0000","last_became_peered":"2023-10-09T08:54:17.355379+0000","last_unstale":"2023-10-09T08:54:35.731380+0000","last_undegraded":"2023-10-09T08:54:17.286955+0000","last_fullsized":"2023-10-09T08:54:35.731380+0000","mapping_epoch":121,"log_start":"102'4244","ondisk_log_start":"102'4244","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77725696,"num_objects":1186,"num_object_clones":0,"num_object_copies":3558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1186,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11254,"num_write_kb":398080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56623104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,7],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":311},{"shards":"5,6,7","objects":875}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081103+0000","last_change":"2023-10-09T08:54:16.087335+0000","last_active":"2023-10-09T08:54:17.081103+0000","last_peered":"2023-10-09T08:54:17.081103+0000","last_clean":"2023-10-09T08:54:17.081103+0000","last_became_active":"2023-10-09T08:54:16.086195+0000","last_became_peered":"2023-10-09T08:54:16.086195+0000","last_unstale":"2023-10-09T08:54:17.081103+0000","last_undegraded":"2023-10-09T08:54:17.081103+0000","last_fullsized":"2023-10-09T08:54:17.081103+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.290386+0000","last_change":"2023-10-09T08:46:18.676843+0000","last_active":"2023-10-09T08:54:17.290386+0000","last_peered":"2023-10-09T08:54:17.290386+0000","last_clean":"2023-10-09T08:54:17.290386+0000","last_became_active":"2023-10-09T08:46:18.676468+0000","last_became_peered":"2023-10-09T08:46:18.676468+0000","last_unstale":"2023-10-09T08:54:17.290386+0000","last_undegraded":"2023-10-09T08:54:17.290386+0000","last_fullsized":"2023-10-09T08:54:17.290386+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"123'11232","reported_seq":11277,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:37.091564+0000","last_change":"2023-10-09T08:53:53.441102+0000","last_active":"2023-10-09T08:54:37.091564+0000","last_peered":"2023-10-09T08:54:37.091564+0000","last_clean":"2023-10-09T08:54:37.091564+0000","last_became_active":"2023-10-09T08:53:53.374919+0000","last_became_peered":"2023-10-09T08:53:53.374919+0000","last_unstale":"2023-10-09T08:54:37.091564+0000","last_undegraded":"2023-10-09T08:54:37.091564+0000","last_fullsized":"2023-10-09T08:54:37.091564+0000","mapping_epoch":115,"log_start":"106'5083","ondisk_log_start":"106'5083","created":95,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":6149,"log_dups_size":0,"ondisk_log_size":6149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:28.771450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75169816,"num_objects":1148,"num_object_clones":0,"num_object_copies":3444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1148,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11232,"num_write_kb":396097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":94,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.091089+0000","last_change":"2023-10-09T08:54:16.088414+0000","last_active":"2023-10-09T08:54:17.091089+0000","last_peered":"2023-10-09T08:54:17.091089+0000","last_clean":"2023-10-09T08:54:17.091089+0000","last_became_active":"2023-10-09T08:54:16.088020+0000","last_became_peered":"2023-10-09T08:54:16.088020+0000","last_unstale":"2023-10-09T08:54:17.091089+0000","last_undegraded":"2023-10-09T08:54:17.091089+0000","last_fullsized":"2023-10-09T08:54:17.091089+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:42.977336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":79,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.092350+0000","last_change":"2023-10-09T08:47:16.965479+0000","last_active":"2023-10-09T08:54:17.092350+0000","last_peered":"2023-10-09T08:54:17.092350+0000","last_clean":"2023-10-09T08:54:17.092350+0000","last_became_active":"2023-10-09T08:47:16.965141+0000","last_became_peered":"2023-10-09T08:47:16.965141+0000","last_unstale":"2023-10-09T08:54:17.092350+0000","last_undegraded":"2023-10-09T08:54:17.092350+0000","last_fullsized":"2023-10-09T08:54:17.092350+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:50.145649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":114,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:31.365879+0000","last_change":"2023-10-09T08:54:16.095002+0000","last_active":"2023-10-09T08:54:31.365879+0000","last_peered":"2023-10-09T08:54:31.365879+0000","last_clean":"2023-10-09T08:54:31.365879+0000","last_became_active":"2023-10-09T08:54:16.093450+0000","last_became_peered":"2023-10-09T08:54:16.093450+0000","last_unstale":"2023-10-09T08:54:31.365879+0000","last_undegraded":"2023-10-09T08:54:31.365879+0000","last_fullsized":"2023-10-09T08:54:31.365879+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:19.952788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.077141+0000","last_change":"2023-10-09T08:53:53.373227+0000","last_active":"2023-10-09T08:54:17.077141+0000","last_peered":"2023-10-09T08:54:17.077141+0000","last_clean":"2023-10-09T08:54:17.077141+0000","last_became_active":"2023-10-09T08:53:53.372723+0000","last_became_peered":"2023-10-09T08:53:53.372723+0000","last_unstale":"2023-10-09T08:54:17.077141+0000","last_undegraded":"2023-10-09T08:54:17.077141+0000","last_fullsized":"2023-10-09T08:54:17.077141+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:50.784734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:31.359940+0000","last_change":"2023-10-09T08:46:18.684501+0000","last_active":"2023-10-09T08:54:31.359940+0000","last_peered":"2023-10-09T08:54:31.359940+0000","last_clean":"2023-10-09T08:54:31.359940+0000","last_became_active":"2023-10-09T08:46:18.684159+0000","last_became_peered":"2023-10-09T08:46:18.684159+0000","last_unstale":"2023-10-09T08:54:31.359940+0000","last_undegraded":"2023-10-09T08:54:31.359940+0000","last_fullsized":"2023-10-09T08:54:31.359940+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:21.270259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"123'11749","reported_seq":11788,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:37.243253+0000","last_change":"2023-10-09T08:50:24.945548+0000","last_active":"2023-10-09T08:54:37.243253+0000","last_peered":"2023-10-09T08:54:37.243253+0000","last_clean":"2023-10-09T08:54:37.243253+0000","last_became_active":"2023-10-09T08:50:24.945187+0000","last_became_peered":"2023-10-09T08:50:24.945187+0000","last_unstale":"2023-10-09T08:54:37.243253+0000","last_undegraded":"2023-10-09T08:54:37.243253+0000","last_fullsized":"2023-10-09T08:54:37.243253+0000","mapping_epoch":95,"log_start":"102'3358","ondisk_log_start":"102'3358","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8391,"log_dups_size":0,"ondisk_log_size":8391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:10:48.344698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83427328,"num_objects":1273,"num_object_clones":0,"num_object_copies":3819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11749,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"123'11541","reported_seq":12793,"reported_epoch":123,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T08:54:35.474017+0000","last_change":"2023-10-09T08:54:17.453001+0000","last_active":"2023-10-09T08:54:35.474017+0000","last_peered":"2023-10-09T08:54:35.474017+0000","last_clean":"2023-10-09T08:51:52.408850+0000","last_became_active":"2023-10-09T08:54:17.392980+0000","last_became_peered":"2023-10-09T08:54:17.392980+0000","last_unstale":"2023-10-09T08:54:35.474017+0000","last_undegraded":"2023-10-09T08:54:17.288877+0000","last_fullsized":"2023-10-09T08:54:35.474017+0000","mapping_epoch":121,"log_start":"102'4301","ondisk_log_start":"102'4301","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7240,"log_dups_size":0,"ondisk_log_size":7240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85262336,"num_objects":1301,"num_object_clones":0,"num_object_copies":3903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1301,"num_objects_misplaced":4888,"num_objects_unfound":0,"num_objects_dirty":1301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11541,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":599,"num_bytes_recovered":39256064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":366},{"shards":"4,6,7","objects":935}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.090421+0000","last_change":"2023-10-09T08:53:53.440504+0000","last_active":"2023-10-09T08:54:17.090421+0000","last_peered":"2023-10-09T08:54:17.090421+0000","last_clean":"2023-10-09T08:54:17.090421+0000","last_became_active":"2023-10-09T08:53:53.440129+0000","last_became_peered":"2023-10-09T08:53:53.440129+0000","last_unstale":"2023-10-09T08:54:17.090421+0000","last_undegraded":"2023-10-09T08:54:17.090421+0000","last_fullsized":"2023-10-09T08:54:17.090421+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:31.886056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":90,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081016+0000","last_change":"2023-10-09T08:45:49.184088+0000","last_active":"2023-10-09T08:54:17.081016+0000","last_peered":"2023-10-09T08:54:17.081016+0000","last_clean":"2023-10-09T08:54:17.081016+0000","last_became_active":"2023-10-09T08:45:49.183674+0000","last_became_peered":"2023-10-09T08:45:49.183674+0000","last_unstale":"2023-10-09T08:54:17.081016+0000","last_undegraded":"2023-10-09T08:54:17.081016+0000","last_fullsized":"2023-10-09T08:54:17.081016+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":102,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.120734+0000","last_change":"2023-10-09T08:53:53.386479+0000","last_active":"2023-10-09T08:54:17.120734+0000","last_peered":"2023-10-09T08:54:17.120734+0000","last_clean":"2023-10-09T08:54:17.120734+0000","last_became_active":"2023-10-09T08:53:53.386014+0000","last_became_peered":"2023-10-09T08:53:53.386014+0000","last_unstale":"2023-10-09T08:54:17.120734+0000","last_undegraded":"2023-10-09T08:54:17.120734+0000","last_fullsized":"2023-10-09T08:54:17.120734+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:24.175440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"123'11339","reported_seq":11393,"reported_epoch":123,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T08:54:35.675501+0000","last_change":"2023-10-09T08:54:17.408361+0000","last_active":"2023-10-09T08:54:35.675501+0000","last_peered":"2023-10-09T08:54:35.675501+0000","last_clean":"2023-10-09T08:54:15.064262+0000","last_became_active":"2023-10-09T08:54:17.292067+0000","last_became_peered":"2023-10-09T08:54:17.292067+0000","last_unstale":"2023-10-09T08:54:35.675501+0000","last_undegraded":"2023-10-09T08:54:35.675501+0000","last_fullsized":"2023-10-09T08:54:35.675501+0000","mapping_epoch":121,"log_start":"102'4268","ondisk_log_start":"102'4268","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77135872,"num_objects":1177,"num_object_clones":0,"num_object_copies":3531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1177,"num_objects_unfound":0,"num_objects_dirty":1177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11339,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":1177}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":124,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.121263+0000","last_change":"2023-10-09T08:51:33.431789+0000","last_active":"2023-10-09T08:54:17.121263+0000","last_peered":"2023-10-09T08:54:17.121263+0000","last_clean":"2023-10-09T08:54:17.121263+0000","last_became_active":"2023-10-09T08:51:33.431448+0000","last_became_peered":"2023-10-09T08:51:33.431448+0000","last_unstale":"2023-10-09T08:54:17.121263+0000","last_undegraded":"2023-10-09T08:54:17.121263+0000","last_fullsized":"2023-10-09T08:54:17.121263+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:42.314482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":80,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.091255+0000","last_change":"2023-10-09T08:46:49.231894+0000","last_active":"2023-10-09T08:54:17.091255+0000","last_peered":"2023-10-09T08:54:17.091255+0000","last_clean":"2023-10-09T08:54:17.091255+0000","last_became_active":"2023-10-09T08:46:49.231531+0000","last_became_peered":"2023-10-09T08:46:49.231531+0000","last_unstale":"2023-10-09T08:54:17.091255+0000","last_undegraded":"2023-10-09T08:54:17.091255+0000","last_fullsized":"2023-10-09T08:54:17.091255+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:50.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":99,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.092278+0000","last_change":"2023-10-09T08:46:18.681375+0000","last_active":"2023-10-09T08:54:17.092278+0000","last_peered":"2023-10-09T08:54:17.092278+0000","last_clean":"2023-10-09T08:54:17.092278+0000","last_became_active":"2023-10-09T08:46:18.680713+0000","last_became_peered":"2023-10-09T08:46:18.680713+0000","last_unstale":"2023-10-09T08:54:17.092278+0000","last_undegraded":"2023-10-09T08:54:17.092278+0000","last_fullsized":"2023-10-09T08:54:17.092278+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:59.671965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"123'11296","reported_seq":11339,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:35.265777+0000","last_change":"2023-10-09T08:53:53.414399+0000","last_active":"2023-10-09T08:54:35.265777+0000","last_peered":"2023-10-09T08:54:35.265777+0000","last_clean":"2023-10-09T08:54:35.265777+0000","last_became_active":"2023-10-09T08:53:53.376540+0000","last_became_peered":"2023-10-09T08:53:53.376540+0000","last_unstale":"2023-10-09T08:54:35.265777+0000","last_undegraded":"2023-10-09T08:54:35.265777+0000","last_fullsized":"2023-10-09T08:54:35.265777+0000","mapping_epoch":115,"log_start":"101'2378","ondisk_log_start":"101'2378","created":95,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":8918,"log_dups_size":0,"ondisk_log_size":8918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:09.798052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79036416,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11296,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":79,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.122015+0000","last_change":"2023-10-09T08:54:16.084456+0000","last_active":"2023-10-09T08:54:17.122015+0000","last_peered":"2023-10-09T08:54:17.122015+0000","last_clean":"2023-10-09T08:54:17.122015+0000","last_became_active":"2023-10-09T08:54:16.084031+0000","last_became_peered":"2023-10-09T08:54:16.084031+0000","last_unstale":"2023-10-09T08:54:17.122015+0000","last_undegraded":"2023-10-09T08:54:17.122015+0000","last_fullsized":"2023-10-09T08:54:17.122015+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:47.307634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":273,"reported_epoch":122,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T08:54:17.337330+0000","last_change":"2023-10-09T08:54:17.101183+0000","last_active":"2023-10-09T08:54:17.337330+0000","last_peered":"2023-10-09T08:54:17.337330+0000","last_clean":"2023-10-09T08:40:49.284447+0000","last_became_active":"2023-10-09T08:54:17.100791+0000","last_became_peered":"2023-10-09T08:54:17.100791+0000","last_unstale":"2023-10-09T08:54:17.337330+0000","last_undegraded":"2023-10-09T08:54:17.081352+0000","last_fullsized":"2023-10-09T08:54:17.080414+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:50.057877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":35,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.089621+0000","last_change":"2023-10-09T08:53:53.432093+0000","last_active":"2023-10-09T08:54:17.089621+0000","last_peered":"2023-10-09T08:54:17.089621+0000","last_clean":"2023-10-09T08:54:17.089621+0000","last_became_active":"2023-10-09T08:53:53.431670+0000","last_became_peered":"2023-10-09T08:53:53.431670+0000","last_unstale":"2023-10-09T08:54:17.089621+0000","last_undegraded":"2023-10-09T08:54:17.089621+0000","last_fullsized":"2023-10-09T08:54:17.089621+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":83,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.144821+0000","last_change":"2023-10-09T08:54:16.097597+0000","last_active":"2023-10-09T08:54:17.144821+0000","last_peered":"2023-10-09T08:54:17.144821+0000","last_clean":"2023-10-09T08:54:17.144821+0000","last_became_active":"2023-10-09T08:54:16.097205+0000","last_became_peered":"2023-10-09T08:54:16.097205+0000","last_unstale":"2023-10-09T08:54:17.144821+0000","last_undegraded":"2023-10-09T08:54:17.144821+0000","last_fullsized":"2023-10-09T08:54:17.144821+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":74,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.091400+0000","last_change":"2023-10-09T08:54:16.095375+0000","last_active":"2023-10-09T08:54:17.091400+0000","last_peered":"2023-10-09T08:54:17.091400+0000","last_clean":"2023-10-09T08:54:17.091400+0000","last_became_active":"2023-10-09T08:54:16.095020+0000","last_became_peered":"2023-10-09T08:54:16.095020+0000","last_unstale":"2023-10-09T08:54:17.091400+0000","last_undegraded":"2023-10-09T08:54:17.091400+0000","last_fullsized":"2023-10-09T08:54:17.091400+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:28.039593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":102,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081428+0000","last_change":"2023-10-09T08:51:33.441477+0000","last_active":"2023-10-09T08:54:17.081428+0000","last_peered":"2023-10-09T08:54:17.081428+0000","last_clean":"2023-10-09T08:54:17.081428+0000","last_became_active":"2023-10-09T08:51:33.441144+0000","last_became_peered":"2023-10-09T08:51:33.441144+0000","last_unstale":"2023-10-09T08:54:17.081428+0000","last_undegraded":"2023-10-09T08:54:17.081428+0000","last_fullsized":"2023-10-09T08:54:17.081428+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:54.567772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":99,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:27.340349+0000","last_change":"2023-10-09T08:46:49.232279+0000","last_active":"2023-10-09T08:54:27.340349+0000","last_peered":"2023-10-09T08:54:27.340349+0000","last_clean":"2023-10-09T08:54:27.340349+0000","last_became_active":"2023-10-09T08:46:49.231911+0000","last_became_peered":"2023-10-09T08:46:49.231911+0000","last_unstale":"2023-10-09T08:54:27.340349+0000","last_undegraded":"2023-10-09T08:54:27.340349+0000","last_fullsized":"2023-10-09T08:54:27.340349+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:38.497218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"123'11506","reported_seq":20539,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:37.088150+0000","last_change":"2023-10-09T08:54:06.508075+0000","last_active":"2023-10-09T08:54:37.088150+0000","last_peered":"2023-10-09T08:54:37.088150+0000","last_clean":"2023-10-09T08:54:37.088150+0000","last_became_active":"2023-10-09T08:54:06.483889+0000","last_became_peered":"2023-10-09T08:54:06.483889+0000","last_unstale":"2023-10-09T08:54:37.088150+0000","last_undegraded":"2023-10-09T08:54:37.088150+0000","last_fullsized":"2023-10-09T08:54:37.088150+0000","mapping_epoch":117,"log_start":"106'5331","ondisk_log_start":"106'5331","created":95,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":6175,"log_dups_size":0,"ondisk_log_size":6175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:12:03.385300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77594624,"num_objects":1184,"num_object_clones":0,"num_object_copies":3552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11506,"num_write_kb":406080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4488,"num_bytes_recovered":292290560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":55,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.337458+0000","last_change":"2023-10-09T08:54:16.094534+0000","last_active":"2023-10-09T08:54:17.337458+0000","last_peered":"2023-10-09T08:54:17.337458+0000","last_clean":"2023-10-09T08:54:17.337458+0000","last_became_active":"2023-10-09T08:54:16.094095+0000","last_became_peered":"2023-10-09T08:54:16.094095+0000","last_unstale":"2023-10-09T08:54:17.337458+0000","last_undegraded":"2023-10-09T08:54:17.337458+0000","last_fullsized":"2023-10-09T08:54:17.337458+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:42.851572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":80,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T08:54:31.366323+0000","last_change":"2023-10-09T08:46:49.233112+0000","last_active":"2023-10-09T08:54:31.366323+0000","last_peered":"2023-10-09T08:54:31.366323+0000","last_clean":"2023-10-09T08:54:31.366323+0000","last_became_active":"2023-10-09T08:46:49.232724+0000","last_became_peered":"2023-10-09T08:46:49.232724+0000","last_unstale":"2023-10-09T08:54:31.366323+0000","last_undegraded":"2023-10-09T08:54:31.366323+0000","last_fullsized":"2023-10-09T08:54:31.366323+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.285159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":78,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080221+0000","last_change":"2023-10-09T08:54:16.091388+0000","last_active":"2023-10-09T08:54:17.080221+0000","last_peered":"2023-10-09T08:54:17.080221+0000","last_clean":"2023-10-09T08:54:17.080221+0000","last_became_active":"2023-10-09T08:54:16.088188+0000","last_became_peered":"2023-10-09T08:54:16.088188+0000","last_unstale":"2023-10-09T08:54:17.080221+0000","last_undegraded":"2023-10-09T08:54:17.080221+0000","last_fullsized":"2023-10-09T08:54:17.080221+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:50.065824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"123'11462","reported_seq":12850,"reported_epoch":123,"state":"active+recovering+remapped","last_fresh":"2023-10-09T08:54:37.072139+0000","last_change":"2023-10-09T08:54:17.520691+0000","last_active":"2023-10-09T08:54:37.072139+0000","last_peered":"2023-10-09T08:54:37.072139+0000","last_clean":"2023-10-09T08:51:52.421304+0000","last_became_active":"2023-10-09T08:54:17.326511+0000","last_became_peered":"2023-10-09T08:54:17.326511+0000","last_unstale":"2023-10-09T08:54:37.072139+0000","last_undegraded":"2023-10-09T08:54:37.072139+0000","last_fullsized":"2023-10-09T08:54:37.072139+0000","mapping_epoch":121,"log_start":"106'5302","ondisk_log_start":"106'5302","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":6160,"log_dups_size":0,"ondisk_log_size":6160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:48.799180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77463552,"num_objects":1182,"num_object_clones":0,"num_object_copies":3546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":241,"num_objects_unfound":0,"num_objects_dirty":1182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11462,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":666,"num_bytes_recovered":43646976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":241},{"shards":"1,6,7","objects":941}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":106,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.077794+0000","last_change":"2023-10-09T08:54:16.155171+0000","last_active":"2023-10-09T08:54:17.077794+0000","last_peered":"2023-10-09T08:54:17.077794+0000","last_clean":"2023-10-09T08:54:17.077794+0000","last_became_active":"2023-10-09T08:54:16.090138+0000","last_became_peered":"2023-10-09T08:54:16.090138+0000","last_unstale":"2023-10-09T08:54:17.077794+0000","last_undegraded":"2023-10-09T08:54:17.077794+0000","last_fullsized":"2023-10-09T08:54:17.077794+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:44.395778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":64,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.090499+0000","last_change":"2023-10-09T08:46:18.678094+0000","last_active":"2023-10-09T08:54:17.090499+0000","last_peered":"2023-10-09T08:54:17.090499+0000","last_clean":"2023-10-09T08:54:17.090499+0000","last_became_active":"2023-10-09T08:46:18.677705+0000","last_became_peered":"2023-10-09T08:46:18.677705+0000","last_unstale":"2023-10-09T08:54:17.090499+0000","last_undegraded":"2023-10-09T08:54:17.090499+0000","last_fullsized":"2023-10-09T08:54:17.090499+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:31.647828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.337389+0000","last_change":"2023-10-09T08:54:16.091706+0000","last_active":"2023-10-09T08:54:17.337389+0000","last_peered":"2023-10-09T08:54:17.337389+0000","last_clean":"2023-10-09T08:54:17.337389+0000","last_became_active":"2023-10-09T08:54:16.091344+0000","last_became_peered":"2023-10-09T08:54:16.091344+0000","last_unstale":"2023-10-09T08:54:17.337389+0000","last_undegraded":"2023-10-09T08:54:17.337389+0000","last_fullsized":"2023-10-09T08:54:17.337389+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:11.816910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"123'11640","reported_seq":12123,"reported_epoch":123,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T08:54:35.479185+0000","last_change":"2023-10-09T08:54:16.091742+0000","last_active":"2023-10-09T08:54:35.479185+0000","last_peered":"2023-10-09T08:54:35.479185+0000","last_clean":"2023-10-09T08:51:32.405237+0000","last_became_active":"2023-10-09T08:51:34.760613+0000","last_became_peered":"2023-10-09T08:51:34.760613+0000","last_unstale":"2023-10-09T08:54:35.479185+0000","last_undegraded":"2023-10-09T08:51:34.471265+0000","last_fullsized":"2023-10-09T08:51:34.434298+0000","mapping_epoch":105,"log_start":"102'4408","ondisk_log_start":"102'4408","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":7232,"log_dups_size":0,"ondisk_log_size":7232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83492864,"num_objects":1274,"num_object_clones":0,"num_object_copies":3822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11640,"num_write_kb":413248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":11862016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,4","objects":-3084}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":103,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.089401+0000","last_change":"2023-10-09T08:50:37.369204+0000","last_active":"2023-10-09T08:54:17.089401+0000","last_peered":"2023-10-09T08:54:17.089401+0000","last_clean":"2023-10-09T08:54:17.089401+0000","last_became_active":"2023-10-09T08:50:37.368197+0000","last_became_peered":"2023-10-09T08:50:37.368197+0000","last_unstale":"2023-10-09T08:54:17.089401+0000","last_undegraded":"2023-10-09T08:54:17.089401+0000","last_fullsized":"2023-10-09T08:54:17.089401+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":88,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:27.340327+0000","last_change":"2023-10-09T08:46:18.683082+0000","last_active":"2023-10-09T08:54:27.340327+0000","last_peered":"2023-10-09T08:54:27.340327+0000","last_clean":"2023-10-09T08:54:27.340327+0000","last_became_active":"2023-10-09T08:46:18.682657+0000","last_became_peered":"2023-10-09T08:46:18.682657+0000","last_unstale":"2023-10-09T08:54:27.340327+0000","last_undegraded":"2023-10-09T08:54:27.340327+0000","last_fullsized":"2023-10-09T08:54:27.340327+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:45.531585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":74,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080945+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:54:17.080945+0000","last_peered":"2023-10-09T08:54:17.080945+0000","last_clean":"2023-10-09T08:54:17.080945+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:54:17.080945+0000","last_undegraded":"2023-10-09T08:54:17.080945+0000","last_fullsized":"2023-10-09T08:54:17.080945+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"123'11360","reported_seq":11401,"reported_epoch":123,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T08:54:35.729032+0000","last_change":"2023-10-09T08:54:17.427948+0000","last_active":"2023-10-09T08:54:35.729032+0000","last_peered":"2023-10-09T08:54:35.729032+0000","last_clean":"2023-10-09T08:54:15.044021+0000","last_became_active":"2023-10-09T08:54:17.353187+0000","last_became_peered":"2023-10-09T08:54:17.353187+0000","last_unstale":"2023-10-09T08:54:35.729032+0000","last_undegraded":"2023-10-09T08:54:35.729032+0000","last_fullsized":"2023-10-09T08:54:35.729032+0000","mapping_epoch":121,"log_start":"103'4366","ondisk_log_start":"103'4366","created":95,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:50:23.916171+0000","last_clean_scrub_stamp":"2023-10-09T08:50:23.916171+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80478208,"num_objects":1228,"num_object_clones":0,"num_object_copies":3684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1228,"num_objects_unfound":0,"num_objects_dirty":1228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11360,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1228}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":81,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080532+0000","last_change":"2023-10-09T08:54:16.091839+0000","last_active":"2023-10-09T08:54:17.080532+0000","last_peered":"2023-10-09T08:54:17.080532+0000","last_clean":"2023-10-09T08:54:17.080532+0000","last_became_active":"2023-10-09T08:54:16.091292+0000","last_became_peered":"2023-10-09T08:54:16.091292+0000","last_unstale":"2023-10-09T08:54:17.080532+0000","last_undegraded":"2023-10-09T08:54:17.080532+0000","last_fullsized":"2023-10-09T08:54:17.080532+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:04.866003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.087106+0000","last_change":"2023-10-09T08:51:53.456605+0000","last_active":"2023-10-09T08:54:17.087106+0000","last_peered":"2023-10-09T08:54:17.087106+0000","last_clean":"2023-10-09T08:54:17.087106+0000","last_became_active":"2023-10-09T08:51:53.456254+0000","last_became_peered":"2023-10-09T08:51:53.456254+0000","last_unstale":"2023-10-09T08:54:17.087106+0000","last_undegraded":"2023-10-09T08:54:17.087106+0000","last_fullsized":"2023-10-09T08:54:17.087106+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:22.564027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":30,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081202+0000","last_change":"2023-10-09T08:53:53.379774+0000","last_active":"2023-10-09T08:54:17.081202+0000","last_peered":"2023-10-09T08:54:17.081202+0000","last_clean":"2023-10-09T08:54:17.081202+0000","last_became_active":"2023-10-09T08:53:53.379368+0000","last_became_peered":"2023-10-09T08:53:53.379368+0000","last_unstale":"2023-10-09T08:54:17.081202+0000","last_undegraded":"2023-10-09T08:54:17.081202+0000","last_fullsized":"2023-10-09T08:54:17.081202+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":104,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081196+0000","last_change":"2023-10-09T08:47:16.968742+0000","last_active":"2023-10-09T08:54:17.081196+0000","last_peered":"2023-10-09T08:54:17.081196+0000","last_clean":"2023-10-09T08:54:17.081196+0000","last_became_active":"2023-10-09T08:47:16.968317+0000","last_became_peered":"2023-10-09T08:47:16.968317+0000","last_unstale":"2023-10-09T08:54:17.081196+0000","last_undegraded":"2023-10-09T08:54:17.081196+0000","last_fullsized":"2023-10-09T08:54:17.081196+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":30,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.081927+0000","last_change":"2023-10-09T08:53:53.383401+0000","last_active":"2023-10-09T08:54:17.081927+0000","last_peered":"2023-10-09T08:54:17.081927+0000","last_clean":"2023-10-09T08:54:17.081927+0000","last_became_active":"2023-10-09T08:53:53.382955+0000","last_became_peered":"2023-10-09T08:53:53.382955+0000","last_unstale":"2023-10-09T08:54:17.081927+0000","last_undegraded":"2023-10-09T08:54:17.081927+0000","last_fullsized":"2023-10-09T08:54:17.081927+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":28,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080938+0000","last_change":"2023-10-09T08:51:33.443239+0000","last_active":"2023-10-09T08:54:17.080938+0000","last_peered":"2023-10-09T08:54:17.080938+0000","last_clean":"2023-10-09T08:54:17.080938+0000","last_became_active":"2023-10-09T08:51:33.442063+0000","last_became_peered":"2023-10-09T08:51:33.442063+0000","last_unstale":"2023-10-09T08:54:17.080938+0000","last_undegraded":"2023-10-09T08:54:17.080938+0000","last_fullsized":"2023-10-09T08:54:17.080938+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":100,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.076854+0000","last_change":"2023-10-09T08:47:16.970132+0000","last_active":"2023-10-09T08:54:17.076854+0000","last_peered":"2023-10-09T08:54:17.076854+0000","last_clean":"2023-10-09T08:54:17.076854+0000","last_became_active":"2023-10-09T08:47:16.969760+0000","last_became_peered":"2023-10-09T08:47:16.969760+0000","last_unstale":"2023-10-09T08:54:17.076854+0000","last_undegraded":"2023-10-09T08:54:17.076854+0000","last_fullsized":"2023-10-09T08:54:17.076854+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.801830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":100,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:22.471083+0000","last_change":"2023-10-09T08:51:53.455827+0000","last_active":"2023-10-09T08:54:22.471083+0000","last_peered":"2023-10-09T08:54:22.471083+0000","last_clean":"2023-10-09T08:54:22.471083+0000","last_became_active":"2023-10-09T08:51:53.455328+0000","last_became_peered":"2023-10-09T08:51:53.455328+0000","last_unstale":"2023-10-09T08:54:22.471083+0000","last_undegraded":"2023-10-09T08:54:22.471083+0000","last_fullsized":"2023-10-09T08:54:22.471083+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:46.784739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":103,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080790+0000","last_change":"2023-10-09T08:51:33.445775+0000","last_active":"2023-10-09T08:54:17.080790+0000","last_peered":"2023-10-09T08:54:17.080790+0000","last_clean":"2023-10-09T08:54:17.080790+0000","last_became_active":"2023-10-09T08:51:33.440135+0000","last_became_peered":"2023-10-09T08:51:33.440135+0000","last_unstale":"2023-10-09T08:54:17.080790+0000","last_undegraded":"2023-10-09T08:54:17.080790+0000","last_fullsized":"2023-10-09T08:54:17.080790+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:02.419868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T08:54:17.080612+0000","last_change":"2023-10-09T08:46:18.676602+0000","last_active":"2023-10-09T08:54:17.080612+0000","last_peered":"2023-10-09T08:54:17.080612+0000","last_clean":"2023-10-09T08:54:17.080612+0000","last_became_active":"2023-10-09T08:46:18.676122+0000","last_became_peered":"2023-10-09T08:46:18.676122+0000","last_unstale":"2023-10-09T08:54:17.080612+0000","last_undegraded":"2023-10-09T08:54:17.080612+0000","last_fullsized":"2023-10-09T08:54:17.080612+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1287323672,"num_objects":19644,"num_object_clones":0,"num_object_copies":58932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10168,"num_objects_misplaced":11243,"num_objects_unfound":0,"num_objects_dirty":19644,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":182606,"num_write_kb":6471937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7964,"num_bytes_recovered":480575488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5732839424,"data_stored":5732827208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120668,"ondisk_log_size":120668,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5504,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430489,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626722,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":930332,"kb_used_data":616724,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92814820,"statfs":{"total":95995035648,"available":95042375680,"internally_reserved":0,"allocated":631525376,"data_stored":631067485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,7,2,3,6,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:54:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2810000000000001,"5min":1.508,"15min":1.1240000000000001},"min":{"1min":0.251,"5min":0.249,"15min":0.249},"max":{"1min":7.8959999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.2410000000000001,"15min":1.0800000000000001},"min":{"1min":0.41999999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.3580000000000005,"5min":10.776,"15min":10.776},"last":0.63700000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:54:29 2023","interfaces":[{"interface":"back","average":{"1min":5.5739999999999998,"5min":2.2210000000000001,"15min":1.339},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":62.219000000000001,"5min":62.219000000000001,"15min":62.219000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":4.1829999999999998,"5min":2.1240000000000001,"15min":1.3480000000000001},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":57.021000000000001,"5min":57.021000000000001,"15min":57.021000000000001},"last":0.68999999999999995}]},{"osd":2,"last update":"Mon Oct 9 08:54:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.621,"15min":1.151},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.0220000000000002,"5min":8.0220000000000002,"15min":8.0220000000000002},"last":0.375},{"interface":"front","average":{"1min":1.613,"5min":1.3129999999999999,"15min":1.0660000000000001},"min":{"1min":0.49299999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.3859999999999992,"5min":10.747,"15min":10.747},"last":0.71499999999999997}]},{"osd":3,"last update":"Mon Oct 9 08:54:29 2023","interfaces":[{"interface":"back","average":{"1min":5.5819999999999999,"5min":3.0699999999999998,"15min":1.6319999999999999},"min":{"1min":0.433,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":55.713000000000001,"5min":57.636000000000003,"15min":57.636000000000003},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.867,"5min":2.4350000000000001,"15min":1.4430000000000001},"min":{"1min":0.39300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":56.064,"5min":58.393000000000001,"15min":58.393000000000001},"last":0.46800000000000003}]},{"osd":4,"last update":"Mon Oct 9 08:54:24 2023","interfaces":[{"interface":"back","average":{"1min":3.4209999999999998,"5min":2.1920000000000002,"15min":1.327},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.462999999999999,"5min":12.462999999999999,"15min":12.462999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.296,"5min":1.3859999999999999,"15min":1.087},"min":{"1min":0.375,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.3150000000000004,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":3.125}]},{"osd":5,"last update":"Mon Oct 9 08:54:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.6579999999999999,"15min":1.2110000000000001},"min":{"1min":0.54000000000000004,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":9.2550000000000008,"5min":12.404999999999999,"15min":12.404999999999999},"last":9.702},{"interface":"front","average":{"1min":1.431,"5min":1.466,"15min":1.1779999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.8569999999999993,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":6.9800000000000004}]},{"osd":7,"last update":"Mon Oct 9 08:54:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":1.7290000000000001,"15min":1.274},"min":{"1min":0.442,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.4469999999999992,"5min":10.678000000000001,"15min":10.678000000000001},"last":9.6549999999999994},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.6779999999999999,"15min":1.2689999999999999},"min":{"1min":0.45100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":7.9169999999999998,"5min":7.9169999999999998,"15min":7.9169999999999998},"last":1.7050000000000001}]}]},{"osd":1,"up_from":83,"seq":356482285670,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":503312,"kb_used_data":377416,"kb_used_omap":9,"kb_used_meta":125878,"kb_avail":93241840,"statfs":{"total":95995035648,"available":95479644160,"internally_reserved":0,"allocated":386473984,"data_stored":385910277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":128900073},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:54:36 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.0569999999999999,"15min":0.77200000000000002},"min":{"1min":0.23200000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.327,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":0.37},{"interface":"front","average":{"1min":1.615,"5min":1.123,"15min":0.82699999999999996},"min":{"1min":0.254,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.9020000000000001,"5min":6.4420000000000002,"15min":6.4420000000000002},"last":0.41699999999999998}]},{"osd":2,"last update":"Mon Oct 9 08:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.087,"15min":0.85199999999999998},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":5.6689999999999996,"5min":7.468,"15min":7.468},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.877,"5min":1.038,"15min":0.81499999999999995},"min":{"1min":0.29599999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.242,"5min":6.2800000000000002,"15min":6.2800000000000002},"last":0.53200000000000003}]},{"osd":3,"last update":"Mon Oct 9 08:54:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.427,"15min":1.044},"min":{"1min":0.38400000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.4660000000000002,"5min":13.01,"15min":13.01},"last":0.443},{"interface":"front","average":{"1min":1.171,"5min":1.429,"15min":1.046},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.5490000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.46400000000000002}]},{"osd":4,"last update":"Mon Oct 9 08:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.4450000000000001,"15min":1.1259999999999999},"min":{"1min":0.377,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.1080000000000005,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":5.0549999999999997},{"interface":"front","average":{"1min":1.887,"5min":1.6379999999999999,"15min":1.1970000000000001},"min":{"1min":0.46300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.5839999999999996,"5min":8.3019999999999996,"15min":8.3019999999999996},"last":5.0899999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.7839999999999998,"5min":2.004,"15min":1.3759999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":12.673,"5min":12.673,"15min":12.673},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.671,"5min":1.847,"15min":1.3240000000000001},"min":{"1min":0.443,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":5.4119999999999999,"5min":14.439,"15min":14.439},"last":0.57099999999999995}]},{"osd":6,"last update":"Mon Oct 9 08:54:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":2.024,"15min":1.3999999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.7560000000000002,"5min":12.619999999999999,"15min":12.619999999999999},"last":1.294},{"interface":"front","average":{"1min":2.7480000000000002,"5min":2.3540000000000001,"15min":1.526},"min":{"1min":0.47799999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":12.718,"5min":12.718,"15min":12.718},"last":1.304}]},{"osd":7,"last update":"Mon Oct 9 08:54:36 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":1.837,"15min":1.3089999999999999},"min":{"1min":0.47699999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":7.2409999999999997,"5min":10.855,"15min":10.855},"last":1.3799999999999999},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.6220000000000001,"15min":1.2789999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":5.2770000000000001,"5min":8.8390000000000004,"15min":8.8390000000000004},"last":0.624}]}]},{"osd":0,"up_from":78,"seq":335007449196,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":367904,"kb_used_data":304216,"kb_used_omap":28,"kb_used_meta":63587,"kb_avail":93377248,"statfs":{"total":95995035648,"available":95618301952,"internally_reserved":0,"allocated":311517184,"data_stored":310939342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":65113936},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.468,"15min":0.96599999999999997},"min":{"1min":0.26900000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.2579999999999991,"5min":37.380000000000003,"15min":37.380000000000003},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.536,"5min":1.4379999999999999,"15min":0.996},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":12.615,"5min":38.012999999999998,"15min":38.012999999999998},"last":0.31}]},{"osd":2,"last update":"Mon Oct 9 08:54:10 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.002,"15min":0.89600000000000002},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.7279999999999998,"5min":6.032,"15min":6.032},"last":0.746},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.091,"15min":0.99399999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.6970000000000001,"5min":5.4089999999999998,"15min":5.4089999999999998},"last":0.94499999999999995}]},{"osd":3,"last update":"Mon Oct 9 08:54:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":3.2170000000000001,"15min":1.603},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.9990000000000006,"5min":110.988,"15min":110.988},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.2889999999999999,"5min":3.1789999999999998,"15min":1.613},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.9450000000000003,"5min":107.66500000000001,"15min":107.66500000000001},"last":0.35099999999999998}]},{"osd":4,"last update":"Mon Oct 9 08:54:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":1.252,"15min":1.113},"min":{"1min":0.38700000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.2840000000000007,"5min":8.2840000000000007,"15min":8.2840000000000007},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.4369999999999998,"5min":1.458,"15min":1.123},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.4610000000000003,"5min":6.5700000000000003,"15min":6.5700000000000003},"last":0.46899999999999997}]},{"osd":5,"last update":"Mon Oct 9 08:54:10 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.1439999999999999,"15min":1.0600000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.5649999999999999,"5min":7.9710000000000001,"15min":7.9710000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.407,"15min":1.21},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.1550000000000002,"5min":6.1550000000000002,"15min":6.1550000000000002},"last":0.78000000000000003}]},{"osd":6,"last update":"Mon Oct 9 08:54:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":1.4870000000000001,"15min":1.2649999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":6.2519999999999998,"5min":6.2519999999999998,"15min":6.2519999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.758,"5min":1.466,"15min":1.2949999999999999},"min":{"1min":0.45800000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.5350000000000001,"5min":5.7569999999999997,"15min":5.7569999999999997},"last":0.68000000000000005}]},{"osd":7,"last update":"Mon Oct 9 08:54:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3610000000000002,"5min":1.5549999999999999,"15min":1.337},"min":{"1min":0.42699999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":7.774,"5min":7.774,"15min":7.774},"last":0.84099999999999997},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.7170000000000001,"15min":1.3320000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.0250000000000004,"5min":9.0250000000000004,"15min":9.0250000000000004},"last":0.51400000000000001}]}]},{"osd":2,"up_from":88,"seq":377957122143,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1288456,"kb_used_data":1158336,"kb_used_omap":81,"kb_used_meta":130030,"kb_avail":92456696,"statfs":{"total":95995035648,"available":94675656704,"internally_reserved":0,"allocated":1186136064,"data_stored":1185583246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":133150938},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.038,"15min":0.81499999999999995},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":6.21,"5min":6.21,"15min":6.21},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.84799999999999998,"15min":0.81999999999999995},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.0990000000000002,"5min":7.2450000000000001,"15min":7.2450000000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.1659999999999999,"15min":0.91000000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.0999999999999996,"5min":11.728,"15min":11.728},"last":0.436},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.069,"15min":0.91500000000000004},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.1060000000000001,"5min":8.266,"15min":8.266},"last":0.42399999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:53:30 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.0669999999999999,"15min":0.90500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.915,"5min":7.3019999999999996,"15min":7.3019999999999996},"last":0.499},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.772,"15min":1.1299999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.8300000000000001,"5min":45.512999999999998,"15min":45.512999999999998},"last":0.39200000000000002}]},{"osd":4,"last update":"Mon Oct 9 08:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.6890000000000001,"15min":1.21},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":12.191000000000001,"5min":12.191000000000001,"15min":12.191000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.4930000000000001,"15min":1.2010000000000001},"min":{"1min":0.40500000000000003,"5min":0.314,"15min":0.314},"max":{"1min":10.598000000000001,"5min":10.598000000000001,"15min":10.598000000000001},"last":14.071999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.403,"15min":1.2350000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.9080000000000004,"5min":7.3419999999999996,"15min":7.3419999999999996},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.659,"5min":1.6930000000000001,"15min":1.28},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":9.4860000000000007,"5min":9.6080000000000005,"15min":9.6080000000000005},"last":1.633}]},{"osd":6,"last update":"Mon Oct 9 08:54:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":1.5609999999999999,"15min":1.276},"min":{"1min":0.437,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.2430000000000003,"5min":7.2430000000000003,"15min":7.2430000000000003},"last":3.5609999999999999},{"interface":"front","average":{"1min":1.7410000000000001,"5min":1.532,"15min":1.3109999999999999},"min":{"1min":0.53300000000000003,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":7.9660000000000002,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":1.27}]},{"osd":7,"last update":"Mon Oct 9 08:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.581,"15min":1.3},"min":{"1min":0.48999999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":6.2380000000000004,"5min":7.6440000000000001,"15min":7.6440000000000001},"last":3.9769999999999999},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.4099999999999999,"15min":1.29},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":5.1790000000000003,"5min":7.415,"15min":7.415},"last":3.8159999999999998}]}]},{"osd":3,"up_from":93,"seq":399431958618,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1837940,"kb_used_data":1712044,"kb_used_omap":7,"kb_used_meta":125880,"kb_avail":91907212,"statfs":{"total":95995035648,"available":94112985088,"internally_reserved":0,"allocated":1753133056,"data_stored":1752593437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":128901736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:54:30 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.3640000000000001,"15min":0.93799999999999994},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.9910000000000001,"5min":13.916,"15min":13.916},"last":1.163},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.1279999999999999,"15min":0.90300000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.3530000000000002,"5min":7.5919999999999996,"15min":7.5919999999999996},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 08:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":1.4950000000000001,"15min":1.0209999999999999},"min":{"1min":0.33300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.0909999999999993,"5min":12.300000000000001,"15min":12.300000000000001},"last":2.5350000000000001},{"interface":"front","average":{"1min":1.167,"5min":1.4450000000000001,"15min":1.022},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.2830000000000004,"5min":12.564,"15min":12.564},"last":1.2190000000000001}]},{"osd":2,"last update":"Mon Oct 9 08:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.9890000000000001,"15min":1.175},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.9260000000000002,"5min":28.655000000000001,"15min":28.655000000000001},"last":2.6040000000000001},{"interface":"front","average":{"1min":1.4399999999999999,"5min":1.982,"15min":1.2},"min":{"1min":0.35699999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8670000000000009,"5min":30.785,"15min":30.785},"last":1.256}]},{"osd":4,"last update":"Mon Oct 9 08:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.7529999999999999,"15min":1.2609999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":7.6840000000000002,"5min":12.877000000000001,"15min":12.877000000000001},"last":1.323},{"interface":"front","average":{"1min":1.3140000000000001,"5min":1.7430000000000001,"15min":1.266},"min":{"1min":0.40999999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.5110000000000001,"5min":10.026,"15min":10.026},"last":4.0800000000000001}]},{"osd":5,"last update":"Mon Oct 9 08:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.544,"15min":1.198},"min":{"1min":0.45300000000000001,"5min":0.374,"15min":0.374},"max":{"1min":3.9769999999999999,"5min":8.0050000000000008,"15min":8.0050000000000008},"last":3.169},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.9650000000000001,"15min":1.369},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.1189999999999998,"5min":12.204000000000001,"15min":12.204000000000001},"last":2.641}]},{"osd":6,"last update":"Mon Oct 9 08:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":1.8160000000000001,"15min":1.351},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.9450000000000003,"5min":10.048,"15min":10.048},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.6669999999999998,"5min":2.0289999999999999,"15min":1.472},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.1980000000000004,"5min":12.275,"15min":12.275},"last":2.7090000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":1.849,"15min":1.385},"min":{"1min":0.45000000000000001,"5min":0.437,"15min":0.437},"max":{"1min":8.1430000000000007,"5min":12.244,"15min":12.244},"last":2.673},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.9590000000000001,"15min":1.4279999999999999},"min":{"1min":0.47499999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":4.9880000000000004,"5min":13.339,"15min":13.339},"last":1.2909999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019188,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":702172,"kb_used_data":404948,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93042980,"statfs":{"total":95995035648,"available":95276011520,"internally_reserved":0,"allocated":414666752,"data_stored":414208120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.5669999999999999,"15min":1.1080000000000001},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.617,"5min":22.815000000000001,"15min":22.815000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.5790000000000002,"5min":1.573,"15min":1.1220000000000001},"min":{"1min":0.42799999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":12.893000000000001,"5min":22.843,"15min":22.843},"last":0.501}]},{"osd":1,"last update":"Mon Oct 9 08:54:25 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.7250000000000001,"15min":1.1439999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.851,"5min":8.9169999999999998,"15min":8.9169999999999998},"last":0.44},{"interface":"front","average":{"1min":1.845,"5min":1.5509999999999999,"15min":1.161},"min":{"1min":0.32800000000000001,"5min":0.313,"15min":0.313},"max":{"1min":11.712999999999999,"5min":11.712999999999999,"15min":11.712999999999999},"last":0.59099999999999997}]},{"osd":2,"last update":"Mon Oct 9 08:54:06 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.9419999999999999,"15min":1.272},"min":{"1min":0.36099999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":9.0380000000000003,"5min":15.163,"15min":15.163},"last":0.70999999999999996},{"interface":"front","average":{"1min":2.5990000000000002,"5min":1.6060000000000001,"15min":1.1739999999999999},"min":{"1min":0.45000000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.1229999999999993,"5min":9.1229999999999993,"15min":9.1229999999999993},"last":0.57499999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:54:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.974,"15min":1.319},"min":{"1min":0.38100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.883,"5min":9.5969999999999995,"15min":9.5969999999999995},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.5350000000000001,"5min":1.7190000000000001,"15min":1.2609999999999999},"min":{"1min":0.47399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.1199999999999992,"5min":9.1199999999999992,"15min":9.1199999999999992},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:53:46 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.5149999999999999,"15min":1.109},"min":{"1min":0.438,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":11.762,"5min":11.762,"15min":11.762},"last":0.66400000000000003},{"interface":"front","average":{"1min":3.8340000000000001,"5min":1.6559999999999999,"15min":1.216},"min":{"1min":0.49099999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.85,"5min":12.85,"15min":12.85},"last":0.68700000000000006}]},{"osd":6,"last update":"Mon Oct 9 08:54:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":2.0299999999999998,"15min":1.321},"min":{"1min":0.33500000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.8140000000000001,"5min":9.0060000000000002,"15min":9.0060000000000002},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.399,"5min":1.823,"15min":1.3120000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7300000000000004,"5min":8.6310000000000002,"15min":8.6310000000000002},"last":0.626}]},{"osd":7,"last update":"Mon Oct 9 08:54:06 2023","interfaces":[{"interface":"back","average":{"1min":3.133,"5min":1.7849999999999999,"15min":1.3100000000000001},"min":{"1min":0.41399999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.933999999999999,"5min":13.933999999999999,"15min":13.933999999999999},"last":1.3680000000000001},{"interface":"front","average":{"1min":3.2269999999999999,"5min":2.2709999999999999,"15min":1.482},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":12.923,"5min":13.911,"15min":13.911},"last":1.5429999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822956,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781072,"kb_used_data":479752,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92964080,"statfs":{"total":95995035648,"available":95195217920,"internally_reserved":0,"allocated":491266048,"data_stored":490815472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.452,"15min":1.0680000000000001},"min":{"1min":0.34399999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.2729999999999997,"5min":6.5599999999999996,"15min":6.5599999999999996},"last":4.1950000000000003},{"interface":"front","average":{"1min":2.5009999999999999,"5min":1.387,"15min":1.0669999999999999},"min":{"1min":0.29999999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.4900000000000002,"5min":7.7949999999999999,"15min":7.7949999999999999},"last":0.67500000000000004}]},{"osd":1,"last update":"Mon Oct 9 08:53:34 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":1.403,"15min":1.077},"min":{"1min":0.32800000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.1630000000000003,"5min":9.1630000000000003,"15min":9.1630000000000003},"last":4.2430000000000003},{"interface":"front","average":{"1min":2.9380000000000002,"5min":1.917,"15min":1.25},"min":{"1min":0.41999999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3979999999999997,"5min":35.042000000000002,"15min":35.042000000000002},"last":0.53800000000000003}]},{"osd":2,"last update":"Mon Oct 9 08:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.5349999999999999,"15min":1.1279999999999999},"min":{"1min":0.34799999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.6689999999999996,"5min":8.7949999999999999,"15min":8.7949999999999999},"last":4.2759999999999998},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.379,"15min":1.1240000000000001},"min":{"1min":0.32900000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.5570000000000004,"5min":7.8719999999999999,"15min":7.8719999999999999},"last":0.58399999999999996}]},{"osd":3,"last update":"Mon Oct 9 08:54:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.5900000000000001,"15min":1.198},"min":{"1min":0.36899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6050000000000004,"5min":7.915,"15min":7.915},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.921,"5min":1.639,"15min":1.1759999999999999},"min":{"1min":0.38400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.0010000000000003,"5min":7.9509999999999996,"15min":7.9509999999999996},"last":0.60099999999999998}]},{"osd":4,"last update":"Mon Oct 9 08:53:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2989999999999999,"5min":1.615,"15min":1.1180000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.2379999999999995,"5min":9.2379999999999995,"15min":9.2379999999999995},"last":4.3070000000000004},{"interface":"front","average":{"1min":2.8210000000000002,"5min":1.5800000000000001,"15min":1.1639999999999999},"min":{"1min":0.41099999999999998,"5min":0.375,"15min":0.375},"max":{"1min":8.141,"5min":8.141,"15min":8.141},"last":0.746}]},{"osd":6,"last update":"Mon Oct 9 08:54:20 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":2.008,"15min":1.3080000000000001},"min":{"1min":0.40699999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.919,"5min":11.919,"15min":11.919},"last":0.69699999999999995},{"interface":"front","average":{"1min":2.8399999999999999,"5min":1.8320000000000001,"15min":1.3029999999999999},"min":{"1min":0.46300000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.5810000000000004,"5min":11.234,"15min":11.234},"last":0.64700000000000002}]},{"osd":7,"last update":"Mon Oct 9 08:54:20 2023","interfaces":[{"interface":"back","average":{"1min":2.698,"5min":2.1819999999999999,"15min":1.427},"min":{"1min":0.48999999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.6280000000000001,"5min":15.849,"15min":15.849},"last":0.78000000000000003},{"interface":"front","average":{"1min":2.6160000000000001,"5min":2.1880000000000002,"15min":1.4410000000000001},"min":{"1min":0.40600000000000003,"5min":0.377,"15min":0.377},"max":{"1min":8.1329999999999991,"5min":12.346,"15min":12.346},"last":0.66000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":308154368,"data_stored":308150296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":384897024,"data_stored":384892952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1183186944,"data_stored":1183186944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1741619200,"data_stored":1741619200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":407437312,"data_stored":407437312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":477040640,"data_stored":477036568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":621281280,"data_stored":621281280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":609222656,"data_stored":609222656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:54:41.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-10-09T08:54:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:41.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:41.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:41.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:41.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:41.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:41.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:41.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:41.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:41.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:41.898 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:41.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:41.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:41.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:41.984 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:41.999 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:54:42.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:42.051 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:54:42.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:42.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:42.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:42.096 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:54:42.135 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:54:42.166 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:54:42.181 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:42.201 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:54:42.219 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:42.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:42.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:42.246 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:54:42.283 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:54:42.288 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.288 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.288 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.288 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.289 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.289 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.289 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.289 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:54:42.309 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:42 smithi016 bash[75179]: audit 2023-10-09T08:54:41.947137+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.16:0/1388584582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:42.310 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:42 smithi094 bash[59677]: audit 2023-10-09T08:54:41.947137+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.16:0/1388584582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:42.326 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:42.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:42.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:42.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:42.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:42.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:42.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:42.611 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:42.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:42.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:42.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:42.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:42.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:42 smithi016 bash[77892]: audit 2023-10-09T08:54:41.947137+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.16:0/1388584582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:54:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:42.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:42.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:42.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:42.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:42.985 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:43.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:43.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:43.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:43.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:43.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:43.310 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:43.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:43.425 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:43 smithi016 bash[75179]: cluster 2023-10-09T08:54:41.419082+0000 mgr.y (mgr.25421) 513 : cluster 0 pgmap v468: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 9658/53367 objects degraded (18.097%); 10391/53367 objects misplaced (19.471%); 6.4 MiB/s, 131 objects/s recovering 2023-10-09T08:54:43.425 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:43 smithi016 bash[77892]: cluster 2023-10-09T08:54:41.419082+0000 mgr.y (mgr.25421) 513 : cluster 0 pgmap v468: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 9658/53367 objects degraded (18.097%); 10391/53367 objects misplaced (19.471%); 6.4 MiB/s, 131 objects/s recovering 2023-10-09T08:54:43.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:43.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:43.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:43.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:43.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:43.692 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:43 smithi094 bash[59677]: cluster 2023-10-09T08:54:41.419082+0000 mgr.y (mgr.25421) 513 : cluster 0 pgmap v468: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 9658/53367 objects degraded (18.097%); 10391/53367 objects misplaced (19.471%); 6.4 MiB/s, 131 objects/s recovering 2023-10-09T08:54:43.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:43.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:43.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:43.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:43.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:43.974 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:43.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:44.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:44.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:44.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:44.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:44.280 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:44.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:44.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:44.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:44.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:44.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:44.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:44.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:44.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:44.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:44 smithi094 bash[59677]: cluster 2023-10-09T08:54:43.962986+0000 mon.a (mon.0) 683 : cluster 3 Health check update: Degraded data redundancy: 9658/53367 objects degraded (18.097%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:44.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:44.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:44 smithi016 bash[75179]: cluster 2023-10-09T08:54:43.962986+0000 mon.a (mon.0) 683 : cluster 3 Health check update: Degraded data redundancy: 9658/53367 objects degraded (18.097%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:44.705 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:44 smithi016 bash[77892]: cluster 2023-10-09T08:54:43.962986+0000 mon.a (mon.0) 683 : cluster 3 Health check update: Degraded data redundancy: 9658/53367 objects degraded (18.097%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:44.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:44.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:44.826 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:44.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:44.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:44.927 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T08:54:44.927 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:54:44.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:45.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:45.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:45.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:45.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:45.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:45.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:45.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:45 smithi094 bash[59677]: cluster 2023-10-09T08:54:43.420093+0000 mgr.y (mgr.25421) 514 : cluster 0 pgmap v469: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 9658/50364 objects degraded (19.176%); 10079/50364 objects misplaced (20.012%); 6.2 MiB/s, 124 objects/s recovering 2023-10-09T08:54:45.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:45.540 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:45 smithi016 bash[77892]: cluster 2023-10-09T08:54:43.420093+0000 mgr.y (mgr.25421) 514 : cluster 0 pgmap v469: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 9658/50364 objects degraded (19.176%); 10079/50364 objects misplaced (20.012%); 6.2 MiB/s, 124 objects/s recovering 2023-10-09T08:54:45.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:45 smithi016 bash[75179]: cluster 2023-10-09T08:54:43.420093+0000 mgr.y (mgr.25421) 514 : cluster 0 pgmap v469: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 9658/50364 objects degraded (19.176%); 10079/50364 objects misplaced (20.012%); 6.2 MiB/s, 124 objects/s recovering 2023-10-09T08:54:45.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:45.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:45.690 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:45.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:45.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:45.806 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:54:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:54:45] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.33.4" 2023-10-09T08:54:45.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:45.936 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:45.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:45.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:46.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:46.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:46.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:46.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:46.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:46.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:46.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:46.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:46.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:46.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:46.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:46.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T08:54:46.480 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:46.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:46.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:46.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:46.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:46.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:46.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:46.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:54:46.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:46.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:46.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:46.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:46.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:46.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:46.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:47.040 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:47.079 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:47.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:47.185 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:47.212 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:47.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:47.267 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:47.318 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:47.344 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:47.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:47.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:47.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:47.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:47.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:47.597 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:47 smithi094 bash[59677]: cluster 2023-10-09T08:54:45.420763+0000 mgr.y (mgr.25421) 515 : cluster 0 pgmap v470: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1016 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 9658/48828 objects degraded (19.780%); 9731/48828 objects misplaced (19.929%); 5.3 MiB/s, 110 objects/s recovering 2023-10-09T08:54:47.613 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:47.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:47.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:47.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:47.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:47.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:47.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:47.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:47 smithi016 bash[77892]: cluster 2023-10-09T08:54:45.420763+0000 mgr.y (mgr.25421) 515 : cluster 0 pgmap v470: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1016 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 9658/48828 objects degraded (19.780%); 9731/48828 objects misplaced (19.929%); 5.3 MiB/s, 110 objects/s recovering 2023-10-09T08:54:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:47 smithi016 bash[75179]: cluster 2023-10-09T08:54:45.420763+0000 mgr.y (mgr.25421) 515 : cluster 0 pgmap v470: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 1016 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 9658/48828 objects degraded (19.780%); 9731/48828 objects misplaced (19.929%); 5.3 MiB/s, 110 objects/s recovering 2023-10-09T08:54:47.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:47.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:47.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:47.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:48.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:48.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:48.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:48.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:48.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:48.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:48.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:48.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:48.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:48.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:48.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:48.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:48.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:48.802 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:48.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:48.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:48.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:48.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:48.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:49.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:49.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:49.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:49.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:49.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:49.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:49.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:49.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:49.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:49.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:49 smithi094 bash[59677]: cluster 2023-10-09T08:54:47.421477+0000 mgr.y (mgr.25421) 516 : cluster 0 pgmap v471: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 880 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 9159/42285 objects degraded (21.660%); 8880/42285 objects misplaced (21.000%); 7.4 MiB/s, 163 objects/s recovering 2023-10-09T08:54:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:49 smithi094 bash[59677]: cluster 2023-10-09T08:54:48.964123+0000 mon.a (mon.0) 684 : cluster 3 Health check update: Degraded data redundancy: 9159/42285 objects degraded (21.660%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:49 smithi094 bash[59677]: audit 2023-10-09T08:54:49.212824+0000 mon.a (mon.0) 685 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:49.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:49 smithi016 bash[75179]: cluster 2023-10-09T08:54:47.421477+0000 mgr.y (mgr.25421) 516 : cluster 0 pgmap v471: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 880 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 9159/42285 objects degraded (21.660%); 8880/42285 objects misplaced (21.000%); 7.4 MiB/s, 163 objects/s recovering 2023-10-09T08:54:49.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:49 smithi016 bash[75179]: cluster 2023-10-09T08:54:48.964123+0000 mon.a (mon.0) 684 : cluster 3 Health check update: Degraded data redundancy: 9159/42285 objects degraded (21.660%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:49.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:49 smithi016 bash[75179]: audit 2023-10-09T08:54:49.212824+0000 mon.a (mon.0) 685 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:49.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:49 smithi016 bash[77892]: cluster 2023-10-09T08:54:47.421477+0000 mgr.y (mgr.25421) 516 : cluster 0 pgmap v471: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 880 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 9159/42285 objects degraded (21.660%); 8880/42285 objects misplaced (21.000%); 7.4 MiB/s, 163 objects/s recovering 2023-10-09T08:54:49.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:49 smithi016 bash[77892]: cluster 2023-10-09T08:54:48.964123+0000 mon.a (mon.0) 684 : cluster 3 Health check update: Degraded data redundancy: 9159/42285 objects degraded (21.660%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:49.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:49 smithi016 bash[77892]: audit 2023-10-09T08:54:49.212824+0000 mon.a (mon.0) 685 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:54:49.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:49.553 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:49.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:49.660 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:49.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:49.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:49.785 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:49.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:49.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:49.922 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:49.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:49.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:50.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:50.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:50.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:50.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:50.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:50.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:50.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:50.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:50.352 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T08:54:50.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:50.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:50.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:50.485 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:50 smithi016 bash[75179]: audit 2023-10-09T08:54:50.290073+0000 mon.a (mon.0) 686 : audit 1 from='client.? 172.21.15.16:0/349652684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:54:50.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:50 smithi016 bash[77892]: audit 2023-10-09T08:54:50.290073+0000 mon.a (mon.0) 686 : audit 1 from='client.? 172.21.15.16:0/349652684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:54:50.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:50 smithi094 bash[59677]: audit 2023-10-09T08:54:50.290073+0000 mon.a (mon.0) 686 : audit 1 from='client.? 172.21.15.16:0/349652684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:54:50.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:50.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:50.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:50.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:50.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:50.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:50.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:50.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:50.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:50.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:50.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:50.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:51.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:51.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:51.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:51.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:51.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:51.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:51.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:51.273 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:54:51.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:51.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:51.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:51.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:51.444 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:51.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:49.422108+0000 mgr.y (mgr.25421) 517 : cluster 0 pgmap v472: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 869 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 9159/41748 objects degraded (21.939%); 8880/41748 objects misplaced (21.270%); 4.8 MiB/s, 109 objects/s recovering 2023-10-09T08:54:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:50.324919+0000 mon.a (mon.0) 687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:54:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:50.337305+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: audit 2023-10-09T08:54:50.338560+0000 mon.a (mon.0) 688 : audit 1 from='client.? 172.21.15.16:0/349652684' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:54:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:50.340956+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:50.342697+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:50.343911+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:50.343975+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[75179]: cluster 2023-10-09T08:54:50.345189+0000 mon.a (mon.0) 689 : cluster 0 osdmap e124: 8 total, 8 up, 7 in 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:49.422108+0000 mgr.y (mgr.25421) 517 : cluster 0 pgmap v472: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 869 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 9159/41748 objects degraded (21.939%); 8880/41748 objects misplaced (21.270%); 4.8 MiB/s, 109 objects/s recovering 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:50.324919+0000 mon.a (mon.0) 687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:50.337305+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: audit 2023-10-09T08:54:50.338560+0000 mon.a (mon.0) 688 : audit 1 from='client.? 172.21.15.16:0/349652684' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:50.340956+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:50.342697+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:50.343911+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:50.343975+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:51 smithi016 bash[77892]: cluster 2023-10-09T08:54:50.345189+0000 mon.a (mon.0) 689 : cluster 0 osdmap e124: 8 total, 8 up, 7 in 2023-10-09T08:54:51.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:51.598 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:49.422108+0000 mgr.y (mgr.25421) 517 : cluster 0 pgmap v472: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 869 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 9159/41748 objects degraded (21.939%); 8880/41748 objects misplaced (21.270%); 4.8 MiB/s, 109 objects/s recovering 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:50.324919+0000 mon.a (mon.0) 687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:50.337305+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: audit 2023-10-09T08:54:50.338560+0000 mon.a (mon.0) 688 : audit 1 from='client.? 172.21.15.16:0/349652684' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:50.340956+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:50.342697+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:50.343911+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:50.343975+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T08:54:51.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:51 smithi094 bash[59677]: cluster 2023-10-09T08:54:50.345189+0000 mon.a (mon.0) 689 : cluster 0 osdmap e124: 8 total, 8 up, 7 in 2023-10-09T08:54:51.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:51.705 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:51.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:51.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:51.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:51.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:51.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:51.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:51.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:51.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:52.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:52.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:52.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.293 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:52.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:52.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:52.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:52.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:52.661 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:52.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:52.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:52.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:52.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:52.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:53.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:53.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:53.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:53.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:53.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:53.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:53.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:53.338 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:53 smithi016 bash[75179]: cluster 2023-10-09T08:54:51.422731+0000 mgr.y (mgr.25421) 518 : cluster 0 pgmap v474: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 756 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 8614/36339 objects degraded (23.705%); 8162/36339 objects misplaced (22.461%); 3.5 MiB/s, 98 objects/s recovering 2023-10-09T08:54:53.338 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:53 smithi016 bash[77892]: cluster 2023-10-09T08:54:51.422731+0000 mgr.y (mgr.25421) 518 : cluster 0 pgmap v474: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 756 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 8614/36339 objects degraded (23.705%); 8162/36339 objects misplaced (22.461%); 3.5 MiB/s, 98 objects/s recovering 2023-10-09T08:54:53.385 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:53.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:53.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:53.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:53.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:53 smithi094 bash[59677]: cluster 2023-10-09T08:54:51.422731+0000 mgr.y (mgr.25421) 518 : cluster 0 pgmap v474: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 756 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 8614/36339 objects degraded (23.705%); 8162/36339 objects misplaced (22.461%); 3.5 MiB/s, 98 objects/s recovering 2023-10-09T08:54:53.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:53.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:53.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:53.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:53.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:53.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:53.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:53.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:53.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:53.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:53.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:53.831 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:54:53.899 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:53.927 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:53.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:54.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:54.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:54.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:54.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:54.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:54.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:54.300 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:54.339 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:54.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:54.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:54.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:54 smithi016 bash[75179]: cluster 2023-10-09T08:54:53.965126+0000 mon.a (mon.0) 690 : cluster 3 Health check update: Degraded data redundancy: 8614/36339 objects degraded (23.705%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:54.352 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:54 smithi016 bash[77892]: cluster 2023-10-09T08:54:53.965126+0000 mon.a (mon.0) 690 : cluster 3 Health check update: Degraded data redundancy: 8614/36339 objects degraded (23.705%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:54.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:54.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:54.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:54.520 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:54 smithi094 bash[59677]: cluster 2023-10-09T08:54:53.965126+0000 mon.a (mon.0) 690 : cluster 3 Health check update: Degraded data redundancy: 8614/36339 objects degraded (23.705%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:54.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:54.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:54.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:54.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:54.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:54.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:54.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:54.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:54.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:54.875 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:54.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:54.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:54.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:55.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:55.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:55.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:55.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:55.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:55.276 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:55.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:55.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:55 smithi016 bash[75179]: cluster 2023-10-09T08:54:53.423329+0000 mgr.y (mgr.25421) 519 : cluster 0 pgmap v475: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 687 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 8614/33042 objects degraded (26.070%); 8007/33042 objects misplaced (24.233%); 2.5 MiB/s, 97 objects/s recovering 2023-10-09T08:54:55.389 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:55 smithi016 bash[77892]: cluster 2023-10-09T08:54:53.423329+0000 mgr.y (mgr.25421) 519 : cluster 0 pgmap v475: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 687 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 8614/33042 objects degraded (26.070%); 8007/33042 objects misplaced (24.233%); 2.5 MiB/s, 97 objects/s recovering 2023-10-09T08:54:55.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:55.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:55.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:55.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:55.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:55.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:55.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:55.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T08:54:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:55 smithi094 bash[59677]: cluster 2023-10-09T08:54:53.423329+0000 mgr.y (mgr.25421) 519 : cluster 0 pgmap v475: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 687 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 8614/33042 objects degraded (26.070%); 8007/33042 objects misplaced (24.233%); 2.5 MiB/s, 97 objects/s recovering 2023-10-09T08:54:55.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:55.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:55.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:55.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:55.937 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:54:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:54:55] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-09T08:54:55.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:55.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:56.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:56.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:56.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:56.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:56.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:56.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:56.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:56.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:56.356 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:56.388 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:56.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:56.495 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:56.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:56.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:56.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:56.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:56.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:56.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:56.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:56.788 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:56.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:56.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:56.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:56.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:56.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:57.004 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T08:54:57.004 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:54:57.032 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:57.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:57.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:57.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:57.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:57.205 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:57.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:57.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:57.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:57.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:57.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:57.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:57.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:57 smithi094 bash[59677]: cluster 2023-10-09T08:54:55.423937+0000 mgr.y (mgr.25421) 520 : cluster 0 pgmap v476: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 650 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 8614/31245 objects degraded (27.569%); 7593/31245 objects misplaced (24.301%); 2.5 MiB/s, 97 objects/s recovering 2023-10-09T08:54:57.364 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:57 smithi094 bash[59677]: audit 2023-10-09T08:54:56.818893+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.16:0/3997179195' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:54:57.364 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:57 smithi094 bash[59677]: audit 2023-10-09T08:54:56.819396+0000 mon.a (mon.0) 691 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:54:57.390 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 6 pgp_num to 16 2023-10-09T08:54:57.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:57.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:57.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:57.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:57.579 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:57.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:57.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:57.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:57 smithi016 bash[75179]: cluster 2023-10-09T08:54:55.423937+0000 mgr.y (mgr.25421) 520 : cluster 0 pgmap v476: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 650 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 8614/31245 objects degraded (27.569%); 7593/31245 objects misplaced (24.301%); 2.5 MiB/s, 97 objects/s recovering 2023-10-09T08:54:57.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:57 smithi016 bash[75179]: audit 2023-10-09T08:54:56.818893+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.16:0/3997179195' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:54:57.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:57 smithi016 bash[75179]: audit 2023-10-09T08:54:56.819396+0000 mon.a (mon.0) 691 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:54:57.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:57 smithi016 bash[77892]: cluster 2023-10-09T08:54:55.423937+0000 mgr.y (mgr.25421) 520 : cluster 0 pgmap v476: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 650 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 8614/31245 objects degraded (27.569%); 7593/31245 objects misplaced (24.301%); 2.5 MiB/s, 97 objects/s recovering 2023-10-09T08:54:57.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:57 smithi016 bash[77892]: audit 2023-10-09T08:54:56.818893+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.16:0/3997179195' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:54:57.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:57 smithi016 bash[77892]: audit 2023-10-09T08:54:56.819396+0000 mon.a (mon.0) 691 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:54:57.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:57.751 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:57.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:57.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:57.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:57.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:57.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:58.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:54:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:54:58.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:58.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:58.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T08:54:58.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.294 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:58.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:58.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:58.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:58.522 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:58.546 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:58.566 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[75179]: audit 2023-10-09T08:54:57.371409+0000 mon.a (mon.0) 692 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[75179]: cluster 2023-10-09T08:54:57.371486+0000 mon.a (mon.0) 693 : cluster 0 osdmap e125: 8 total, 8 up, 7 in 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[75179]: cluster 2023-10-09T08:54:57.371702+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[75179]: cluster 2023-10-09T08:54:57.380586+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[75179]: cluster 2023-10-09T08:54:57.384182+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[75179]: cluster 2023-10-09T08:54:57.394176+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[77892]: audit 2023-10-09T08:54:57.371409+0000 mon.a (mon.0) 692 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[77892]: cluster 2023-10-09T08:54:57.371486+0000 mon.a (mon.0) 693 : cluster 0 osdmap e125: 8 total, 8 up, 7 in 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[77892]: cluster 2023-10-09T08:54:57.371702+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[77892]: cluster 2023-10-09T08:54:57.380586+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[77892]: cluster 2023-10-09T08:54:57.384182+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:58 smithi016 bash[77892]: cluster 2023-10-09T08:54:57.394176+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:58.659 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:58 smithi094 bash[59677]: audit 2023-10-09T08:54:57.371409+0000 mon.a (mon.0) 692 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T08:54:58.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:58 smithi094 bash[59677]: cluster 2023-10-09T08:54:57.371486+0000 mon.a (mon.0) 693 : cluster 0 osdmap e125: 8 total, 8 up, 7 in 2023-10-09T08:54:58.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:58 smithi094 bash[59677]: cluster 2023-10-09T08:54:57.371702+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:58 smithi094 bash[59677]: cluster 2023-10-09T08:54:57.380586+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:58 smithi094 bash[59677]: cluster 2023-10-09T08:54:57.384182+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:58 smithi094 bash[59677]: cluster 2023-10-09T08:54:57.394176+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T08:54:58.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:54:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:54:58.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:58.822 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:58.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:58.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:54:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:58.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:54:59.059 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:59.096 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:54:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:59.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:54:59.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:59.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:59.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:59.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:59.334 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:59.358 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:54:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:59.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:54:59.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:59.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:54:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:59.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:54:59.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:59.620 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:59.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:59.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:59.732 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:59 smithi016 bash[75179]: cluster 2023-10-09T08:54:57.424624+0000 mgr.y (mgr.25421) 521 : cluster 0 pgmap v478: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 504 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 8082/24246 objects degraded (33.333%); 6977/24246 objects misplaced (28.776%); 0 B/s, 83 objects/s recovering 2023-10-09T08:54:59.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:54:59 smithi016 bash[75179]: cluster 2023-10-09T08:54:58.966080+0000 mon.a (mon.0) 694 : cluster 3 Health check update: Degraded data redundancy: 8082/24246 objects degraded (33.333%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:59.733 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:59 smithi016 bash[77892]: cluster 2023-10-09T08:54:57.424624+0000 mgr.y (mgr.25421) 521 : cluster 0 pgmap v478: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 504 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 8082/24246 objects degraded (33.333%); 6977/24246 objects misplaced (28.776%); 0 B/s, 83 objects/s recovering 2023-10-09T08:54:59.733 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:54:59 smithi016 bash[77892]: cluster 2023-10-09T08:54:58.966080+0000 mon.a (mon.0) 694 : cluster 3 Health check update: Degraded data redundancy: 8082/24246 objects degraded (33.333%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:59.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:59 smithi094 bash[59677]: cluster 2023-10-09T08:54:57.424624+0000 mgr.y (mgr.25421) 521 : cluster 0 pgmap v478: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 504 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 8082/24246 objects degraded (33.333%); 6977/24246 objects misplaced (28.776%); 0 B/s, 83 objects/s recovering 2023-10-09T08:54:59.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:54:59 smithi094 bash[59677]: cluster 2023-10-09T08:54:58.966080+0000 mon.a (mon.0) 694 : cluster 3 Health check update: Degraded data redundancy: 8082/24246 objects degraded (33.333%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:54:59.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:54:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:59.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:54:59.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:59.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:54:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:54:59.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:54:59.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:00.046 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:00.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:00.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:00.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:00.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:00.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:00.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:00.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:00.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:00.455 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:00.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:00.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:00.597 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:00.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:00.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:00.705 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:00.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:00.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:00.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:00.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:00.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:00.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:00.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:01.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:01.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:01.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:01.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:01.155 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:55:01.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:01.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:01.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:01.190 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:55:01.237 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:55:01.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:01.286 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:55:01.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:01.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:01.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:01.321 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:55:01.365 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:55:01.390 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:01 smithi094 bash[59677]: cluster 2023-10-09T08:54:59.427005+0000 mgr.y (mgr.25421) 522 : cluster 0 pgmap v479: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 491 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 8082/23607 objects degraded (34.236%); 6977/23607 objects misplaced (29.555%); 0 B/s, 73 objects/s recovering 2023-10-09T08:55:01.390 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:01 smithi094 bash[59677]: audit 2023-10-09T08:55:01.122818+0000 mon.a (mon.0) 695 : audit 0 from='client.? 172.21.15.16:0/4128336449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:01.406 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:55:01.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:01.441 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:55:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.446 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:55:01.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:01.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:01.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:01.587 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:01 smithi016 bash[75179]: cluster 2023-10-09T08:54:59.427005+0000 mgr.y (mgr.25421) 522 : cluster 0 pgmap v479: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 491 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 8082/23607 objects degraded (34.236%); 6977/23607 objects misplaced (29.555%); 0 B/s, 73 objects/s recovering 2023-10-09T08:55:01.587 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:01 smithi016 bash[75179]: audit 2023-10-09T08:55:01.122818+0000 mon.a (mon.0) 695 : audit 0 from='client.? 172.21.15.16:0/4128336449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:01.587 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:01 smithi016 bash[77892]: cluster 2023-10-09T08:54:59.427005+0000 mgr.y (mgr.25421) 522 : cluster 0 pgmap v479: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 491 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 8082/23607 objects degraded (34.236%); 6977/23607 objects misplaced (29.555%); 0 B/s, 73 objects/s recovering 2023-10-09T08:55:01.587 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:01 smithi016 bash[77892]: audit 2023-10-09T08:55:01.122818+0000 mon.a (mon.0) 695 : audit 0 from='client.? 172.21.15.16:0/4128336449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:01.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:01.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:01.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:01.734 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:01.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:01.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:01.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:01.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:01.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:01.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:01.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:02.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:02.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:02.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:02.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:02.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:02.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:02.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:02.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:02.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:02.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:02.411 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:02.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:02.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:02.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:02.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:02.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:02.656 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:02.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:02.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:02.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:02.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:02.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:02.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:03.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:03.049 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:03.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:03.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:03.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:03.285 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:03.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:03.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:03.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:03.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:03.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:03.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:03 smithi094 bash[59677]: cluster 2023-10-09T08:55:01.427667+0000 mgr.y (mgr.25421) 523 : cluster 0 pgmap v480: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 373 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 7505/17949 objects degraded (41.813%); 6242/17949 objects misplaced (34.776%); 0 B/s, 66 objects/s recovering 2023-10-09T08:55:03.572 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:03.676 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:03.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:03 smithi016 bash[75179]: cluster 2023-10-09T08:55:01.427667+0000 mgr.y (mgr.25421) 523 : cluster 0 pgmap v480: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 373 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 7505/17949 objects degraded (41.813%); 6242/17949 objects misplaced (34.776%); 0 B/s, 66 objects/s recovering 2023-10-09T08:55:03.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:03 smithi016 bash[77892]: cluster 2023-10-09T08:55:01.427667+0000 mgr.y (mgr.25421) 523 : cluster 0 pgmap v480: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 373 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 7505/17949 objects degraded (41.813%); 6242/17949 objects misplaced (34.776%); 0 B/s, 66 objects/s recovering 2023-10-09T08:55:03.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:03.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:03.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:03.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:03.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:04.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:04.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:04.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:04.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:04.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:04.402 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:04 smithi016 bash[75179]: cluster 2023-10-09T08:55:03.967330+0000 mon.a (mon.0) 696 : cluster 3 Health check update: Degraded data redundancy: 7505/17949 objects degraded (41.813%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:04.403 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:04 smithi016 bash[75179]: audit 2023-10-09T08:55:04.217035+0000 mon.a (mon.0) 697 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:04.403 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:04 smithi016 bash[75179]: audit 2023-10-09T08:55:04.218327+0000 mon.a (mon.0) 698 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:04.403 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:04 smithi016 bash[77892]: cluster 2023-10-09T08:55:03.967330+0000 mon.a (mon.0) 696 : cluster 3 Health check update: Degraded data redundancy: 7505/17949 objects degraded (41.813%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:04.403 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:04 smithi016 bash[77892]: audit 2023-10-09T08:55:04.217035+0000 mon.a (mon.0) 697 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:04.403 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:04 smithi016 bash[77892]: audit 2023-10-09T08:55:04.218327+0000 mon.a (mon.0) 698 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:04.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:04.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:04.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:04.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:04.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:04.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:04.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:04.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:04.658 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:04 smithi094 bash[59677]: cluster 2023-10-09T08:55:03.967330+0000 mon.a (mon.0) 696 : cluster 3 Health check update: Degraded data redundancy: 7505/17949 objects degraded (41.813%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:04.658 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:04 smithi094 bash[59677]: audit 2023-10-09T08:55:04.217035+0000 mon.a (mon.0) 697 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:04.658 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:04 smithi094 bash[59677]: audit 2023-10-09T08:55:04.218327+0000 mon.a (mon.0) 698 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:04.671 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:04.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:04.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:55:04.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:04.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:04.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:04.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:04.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:05.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:05.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:05.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:05.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:05.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:05.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:05.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:05.361 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:55:05.362 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:55:05.362 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T08:55:05.362 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T08:55:05.362 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T08:55:05.362 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T08:55:05.367 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:05.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:05.404 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:05 smithi016 bash[77892]: cluster 2023-10-09T08:55:03.428466+0000 mgr.y (mgr.25421) 524 : cluster 0 pgmap v481: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 302 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 7505/14541 objects degraded (51.613%); 6054/14541 objects misplaced (41.634%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:05.481 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:05.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:05 smithi016 bash[75179]: cluster 2023-10-09T08:55:03.428466+0000 mgr.y (mgr.25421) 524 : cluster 0 pgmap v481: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 302 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 7505/14541 objects degraded (51.613%); 6054/14541 objects misplaced (41.634%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:05.513 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:05.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:05.616 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:05.633 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:05 smithi094 bash[59677]: cluster 2023-10-09T08:55:03.428466+0000 mgr.y (mgr.25421) 524 : cluster 0 pgmap v481: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 302 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 7505/14541 objects degraded (51.613%); 6054/14541 objects misplaced (41.634%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:05.648 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:05.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:05.762 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:05.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:05.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:05.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:05.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:05.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:05.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:55:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:55:05] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.33.4" 2023-10-09T08:55:06.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:06.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:06.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:06.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:06.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:06.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:06.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:06.332 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:06.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:06.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:06.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:06.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:06.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:06.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:06.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:06.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:06.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:06.855 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:06.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:06.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:06.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:07.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:07.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:07.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:07.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:07.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:07.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:07.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:07.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:07.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:07.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:07.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:07.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:07.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:07.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:07.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:07.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:07 smithi016 bash[75179]: cluster 2023-10-09T08:55:05.429393+0000 mgr.y (mgr.25421) 525 : cluster 0 pgmap v482: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 266 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 7505/12813 objects degraded (58.573%); 5664/12813 objects misplaced (44.205%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:07.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:07 smithi016 bash[77892]: cluster 2023-10-09T08:55:05.429393+0000 mgr.y (mgr.25421) 525 : cluster 0 pgmap v482: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 266 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 7505/12813 objects degraded (58.573%); 5664/12813 objects misplaced (44.205%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:07.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:07.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:07.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:07 smithi094 bash[59677]: cluster 2023-10-09T08:55:05.429393+0000 mgr.y (mgr.25421) 525 : cluster 0 pgmap v482: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 266 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 7505/12813 objects degraded (58.573%); 5664/12813 objects misplaced (44.205%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:07.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:07.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:07.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:07.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:07.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:07.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:07.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:07.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:08.046 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:08.082 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:08.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:08.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:08.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:08.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:08.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:08.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:08.371 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:08.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:08.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:08.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:08.504 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:08.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:08.608 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:08.639 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:08.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:08.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:08.780 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:08.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:08.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:08.892 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:55:08.892 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T08:55:08.892 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T08:55:08.892 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T08:55:08.892 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:55:08.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:08.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:08.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:09.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:09.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:09.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:09.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:09.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:09.193 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 101124 objects 2023-10-09T08:55:09.193 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :172.727 2023-10-09T08:55:09.288 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:09.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:09.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:09.424 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T08:55:09.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:09.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:09 smithi094 bash[59677]: cluster 2023-10-09T08:55:07.430107+0000 mgr.y (mgr.25421) 526 : cluster 0 pgmap v483: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 126 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 6966/6075 objects degraded (114.667%); 5117/6075 objects misplaced (84.230%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:09.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:09 smithi094 bash[59677]: audit 2023-10-09T08:55:08.588789+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.16:0/485661599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:09.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:09 smithi094 bash[59677]: audit 2023-10-09T08:55:08.589356+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:09.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:09 smithi094 bash[59677]: cluster 2023-10-09T08:55:08.968454+0000 mon.a (mon.0) 700 : cluster 3 Health check update: Degraded data redundancy: 6966/6075 objects degraded (114.667%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:09.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:09.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:09.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:09.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:09.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:09.670 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[75179]: cluster 2023-10-09T08:55:07.430107+0000 mgr.y (mgr.25421) 526 : cluster 0 pgmap v483: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 126 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 6966/6075 objects degraded (114.667%); 5117/6075 objects misplaced (84.230%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:09.670 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[75179]: audit 2023-10-09T08:55:08.588789+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.16:0/485661599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:09.670 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[75179]: audit 2023-10-09T08:55:08.589356+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:09.670 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[75179]: cluster 2023-10-09T08:55:08.968454+0000 mon.a (mon.0) 700 : cluster 3 Health check update: Degraded data redundancy: 6966/6075 objects degraded (114.667%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:09.670 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[77892]: cluster 2023-10-09T08:55:07.430107+0000 mgr.y (mgr.25421) 526 : cluster 0 pgmap v483: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 126 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 6966/6075 objects degraded (114.667%); 5117/6075 objects misplaced (84.230%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:09.670 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[77892]: audit 2023-10-09T08:55:08.588789+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.16:0/485661599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:09.670 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[77892]: audit 2023-10-09T08:55:08.589356+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:09.671 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:09 smithi016 bash[77892]: cluster 2023-10-09T08:55:08.968454+0000 mon.a (mon.0) 700 : cluster 3 Health check update: Degraded data redundancy: 6966/6075 objects degraded (114.667%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:09.678 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:09.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:09.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:09.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:09.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:09.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:09.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:10.072 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:10.094 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:10.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:10.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:10.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T08:55:10.325 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:10.348 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:10.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:10.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:10.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:10.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:10.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[75179]: cluster 2023-10-09T08:55:09.408850+0000 mon.a (mon.0) 701 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:55:10.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[75179]: cluster 2023-10-09T08:55:09.411469+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:10.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[75179]: cluster 2023-10-09T08:55:09.411833+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:10.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[75179]: audit 2023-10-09T08:55:09.415981+0000 mon.a (mon.0) 702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:55:10.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[75179]: cluster 2023-10-09T08:55:09.416100+0000 mon.a (mon.0) 703 : cluster 0 osdmap e126: 8 total, 8 up, 7 in 2023-10-09T08:55:10.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[75179]: cluster 2023-10-09T08:55:09.430691+0000 mgr.y (mgr.25421) 527 : cluster 0 pgmap v485: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 114 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 6966/5520 objects degraded (126.196%); 5117/5520 objects misplaced (92.699%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:10.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[77892]: cluster 2023-10-09T08:55:09.408850+0000 mon.a (mon.0) 701 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:55:10.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[77892]: cluster 2023-10-09T08:55:09.411469+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:10.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[77892]: cluster 2023-10-09T08:55:09.411833+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:10.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[77892]: audit 2023-10-09T08:55:09.415981+0000 mon.a (mon.0) 702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:55:10.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[77892]: cluster 2023-10-09T08:55:09.416100+0000 mon.a (mon.0) 703 : cluster 0 osdmap e126: 8 total, 8 up, 7 in 2023-10-09T08:55:10.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:10 smithi016 bash[77892]: cluster 2023-10-09T08:55:09.430691+0000 mgr.y (mgr.25421) 527 : cluster 0 pgmap v485: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 114 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 6966/5520 objects degraded (126.196%); 5117/5520 objects misplaced (92.699%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:10.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:10.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:10 smithi094 bash[59677]: cluster 2023-10-09T08:55:09.408850+0000 mon.a (mon.0) 701 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:55:10.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:10 smithi094 bash[59677]: cluster 2023-10-09T08:55:09.411469+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:10.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:10 smithi094 bash[59677]: cluster 2023-10-09T08:55:09.411833+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:10.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:10 smithi094 bash[59677]: audit 2023-10-09T08:55:09.415981+0000 mon.a (mon.0) 702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:55:10.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:10 smithi094 bash[59677]: cluster 2023-10-09T08:55:09.416100+0000 mon.a (mon.0) 703 : cluster 0 osdmap e126: 8 total, 8 up, 7 in 2023-10-09T08:55:10.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:10 smithi094 bash[59677]: cluster 2023-10-09T08:55:09.430691+0000 mgr.y (mgr.25421) 527 : cluster 0 pgmap v485: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 114 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 6966/5520 objects degraded (126.196%); 5117/5520 objects misplaced (92.699%); 0 B/s, 67 objects/s recovering 2023-10-09T08:55:10.613 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:10.652 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:10.716 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:10.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:10.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:10.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:10.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:10.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:10.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:10.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:10.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:11.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:11.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:11.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:11.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:11.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:11.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:11.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:11.370 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:11.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:11.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:11.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:11.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:11.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:11.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:11.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:11.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:11.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:11.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:11.726 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:11.733 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:11 smithi094 bash[59677]: cluster 2023-10-09T08:55:10.287200+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:11.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:11 smithi016 bash[75179]: cluster 2023-10-09T08:55:10.287200+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:11 smithi016 bash[77892]: cluster 2023-10-09T08:55:10.287200+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T08:55:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:11.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:11.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:11.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:11.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:11.978 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:12.000 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:12.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:12.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:12.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:12.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:12.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:12.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:12.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:12.481 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:12.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:12.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:12.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:12.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:12 smithi016 bash[75179]: cluster 2023-10-09T08:55:11.431285+0000 mgr.y (mgr.25421) 528 : cluster 0 pgmap v486: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 28 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6232/1410 objects degraded (441.986%); 4589/1410 objects misplaced (325.461%); 0 B/s, 95 objects/s recovering 2023-10-09T08:55:12.631 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:12 smithi016 bash[77892]: cluster 2023-10-09T08:55:11.431285+0000 mgr.y (mgr.25421) 528 : cluster 0 pgmap v486: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 28 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6232/1410 objects degraded (441.986%); 4589/1410 objects misplaced (325.461%); 0 B/s, 95 objects/s recovering 2023-10-09T08:55:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:12.732 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:12.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:12 smithi094 bash[59677]: cluster 2023-10-09T08:55:11.431285+0000 mgr.y (mgr.25421) 528 : cluster 0 pgmap v486: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 28 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6232/1410 objects degraded (441.986%); 4589/1410 objects misplaced (325.461%); 0 B/s, 95 objects/s recovering 2023-10-09T08:55:12.758 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:12.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:12.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:12.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:12.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:12.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:13.009 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:13.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:13.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:13.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:13.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:13.240 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:13.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:13.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:13.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:13.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:13.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:13.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:13.656 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:13.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:13.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:13.855 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T08:55:13.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T08:55:13.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:13.914 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:13.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:14.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:14.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:14.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:14.173 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:14.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:14.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:14.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:14.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:14.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:14.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:14.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:14.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:14.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:14.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:14.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:14.674 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:14 smithi016 bash[75179]: cluster 2023-10-09T08:55:13.431843+0000 mgr.y (mgr.25421) 529 : cluster 0 pgmap v487: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 6232/72 objects degraded (8655.556%); 4522/72 objects misplaced (6280.556%); 0 B/s, 154 objects/s recovering 2023-10-09T08:55:14.674 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:14 smithi016 bash[75179]: cluster 2023-10-09T08:55:13.969590+0000 mon.a (mon.0) 704 : cluster 3 Health check update: Degraded data redundancy: 6232/72 objects degraded (8655.556%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:14.674 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:14 smithi016 bash[77892]: cluster 2023-10-09T08:55:13.431843+0000 mgr.y (mgr.25421) 529 : cluster 0 pgmap v487: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 6232/72 objects degraded (8655.556%); 4522/72 objects misplaced (6280.556%); 0 B/s, 154 objects/s recovering 2023-10-09T08:55:14.674 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:14 smithi016 bash[77892]: cluster 2023-10-09T08:55:13.969590+0000 mon.a (mon.0) 704 : cluster 3 Health check update: Degraded data redundancy: 6232/72 objects degraded (8655.556%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:14.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:14.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:14 smithi094 bash[59677]: cluster 2023-10-09T08:55:13.431843+0000 mgr.y (mgr.25421) 529 : cluster 0 pgmap v487: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 6232/72 objects degraded (8655.556%); 4522/72 objects misplaced (6280.556%); 0 B/s, 154 objects/s recovering 2023-10-09T08:55:14.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:14 smithi094 bash[59677]: cluster 2023-10-09T08:55:13.969590+0000 mon.a (mon.0) 704 : cluster 3 Health check update: Degraded data redundancy: 6232/72 objects degraded (8655.556%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:14.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:14.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:14.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:14.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:14.950 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:14.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:14.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:15.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:15.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:15.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:15.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:15.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:15.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:15.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T08:55:15.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:15.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:15.383 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:15.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:15.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:15.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:15.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:15.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:15.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:15.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:15.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:15.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:15.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:15.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:15.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:15.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:55:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:55:15] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.33.4" 2023-10-09T08:55:16.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:16.100 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:16.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:16.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:16.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:16.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:16.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:16.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:16.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:16.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:16 smithi016 bash[77892]: cluster 2023-10-09T08:55:15.432598+0000 mgr.y (mgr.25421) 530 : cluster 0 pgmap v488: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 6232/72 objects degraded (8655.556%); 4522/72 objects misplaced (6280.556%); 0 B/s, 154 objects/s recovering 2023-10-09T08:55:16.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:16.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:16.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:16.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:16 smithi094 bash[59677]: cluster 2023-10-09T08:55:15.432598+0000 mgr.y (mgr.25421) 530 : cluster 0 pgmap v488: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 6232/72 objects degraded (8655.556%); 4522/72 objects misplaced (6280.556%); 0 B/s, 154 objects/s recovering 2023-10-09T08:55:16.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:16.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:16 smithi016 bash[75179]: cluster 2023-10-09T08:55:15.432598+0000 mgr.y (mgr.25421) 530 : cluster 0 pgmap v488: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 6232/72 objects degraded (8655.556%); 4522/72 objects misplaced (6280.556%); 0 B/s, 154 objects/s recovering 2023-10-09T08:55:16.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:16.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:16.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:16.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:16.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:17.099 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:17.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:17.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:17.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:17.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:17.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:17.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:17.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:17.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:17.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:17.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:17.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:17.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:17.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:17.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:17.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:17.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:17.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:17.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:17.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:17.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:17.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:18.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:18.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:18.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:18.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:18.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:18.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:18.176 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:55:18.176 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T08:55:18.176 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:55:18.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:18.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:18.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:18.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:18.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:18.497 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:18 smithi016 bash[77892]: cluster 2023-10-09T08:55:17.433794+0000 mgr.y (mgr.25421) 531 : cluster 0 pgmap v489: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 5132/72 objects degraded (7127.778%); 4522/72 objects misplaced (6280.556%); 0 B/s, 392 objects/s recovering 2023-10-09T08:55:18.513 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:18.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:18.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:18.661 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:18 smithi094 bash[59677]: cluster 2023-10-09T08:55:17.433794+0000 mgr.y (mgr.25421) 531 : cluster 0 pgmap v489: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 5132/72 objects degraded (7127.778%); 4522/72 objects misplaced (6280.556%); 0 B/s, 392 objects/s recovering 2023-10-09T08:55:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:18 smithi016 bash[75179]: cluster 2023-10-09T08:55:17.433794+0000 mgr.y (mgr.25421) 531 : cluster 0 pgmap v489: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 5132/72 objects degraded (7127.778%); 4522/72 objects misplaced (6280.556%); 0 B/s, 392 objects/s recovering 2023-10-09T08:55:18.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:18.788 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:18.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:18.891 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:18.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:18.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:18.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:19.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:19.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:19.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:19.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:19.268 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:19.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:19.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:19.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:19.416 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:19.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:19.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:19.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:19.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:19.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:19 smithi094 bash[59677]: audit 2023-10-09T08:55:18.719143+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:19 smithi094 bash[59677]: cluster 2023-10-09T08:55:18.970711+0000 mon.a (mon.0) 706 : cluster 3 Health check update: Degraded data redundancy: 5132/72 objects degraded (7127.778%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:19 smithi094 bash[59677]: cluster 2023-10-09T08:55:18.979238+0000 mon.a (mon.0) 707 : cluster 0 osdmap e127: 8 total, 8 up, 7 in 2023-10-09T08:55:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:19 smithi094 bash[59677]: audit 2023-10-09T08:55:19.092182+0000 mon.a (mon.0) 708 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:19 smithi094 bash[59677]: audit 2023-10-09T08:55:19.100547+0000 mon.a (mon.0) 709 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:19 smithi094 bash[59677]: audit 2023-10-09T08:55:19.213859+0000 mon.a (mon.0) 710 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:19.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:19.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[75179]: audit 2023-10-09T08:55:18.719143+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[75179]: cluster 2023-10-09T08:55:18.970711+0000 mon.a (mon.0) 706 : cluster 3 Health check update: Degraded data redundancy: 5132/72 objects degraded (7127.778%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[75179]: cluster 2023-10-09T08:55:18.979238+0000 mon.a (mon.0) 707 : cluster 0 osdmap e127: 8 total, 8 up, 7 in 2023-10-09T08:55:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[75179]: audit 2023-10-09T08:55:19.092182+0000 mon.a (mon.0) 708 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[75179]: audit 2023-10-09T08:55:19.100547+0000 mon.a (mon.0) 709 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[75179]: audit 2023-10-09T08:55:19.213859+0000 mon.a (mon.0) 710 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[77892]: audit 2023-10-09T08:55:18.719143+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[77892]: cluster 2023-10-09T08:55:18.970711+0000 mon.a (mon.0) 706 : cluster 3 Health check update: Degraded data redundancy: 5132/72 objects degraded (7127.778%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[77892]: cluster 2023-10-09T08:55:18.979238+0000 mon.a (mon.0) 707 : cluster 0 osdmap e127: 8 total, 8 up, 7 in 2023-10-09T08:55:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[77892]: audit 2023-10-09T08:55:19.092182+0000 mon.a (mon.0) 708 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[77892]: audit 2023-10-09T08:55:19.100547+0000 mon.a (mon.0) 709 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:19 smithi016 bash[77892]: audit 2023-10-09T08:55:19.213859+0000 mon.a (mon.0) 710 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:19.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:19.810 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:19.914 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:19.941 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:19.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:19.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:20.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:20.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:20.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:20.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:20.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:20.310 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:20.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:20.437 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:20.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:20.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:20.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:20.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:20.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:20.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:20 smithi094 bash[59677]: cluster 2023-10-09T08:55:18.973556+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:20 smithi094 bash[59677]: cluster 2023-10-09T08:55:18.976812+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:20 smithi094 bash[59677]: cluster 2023-10-09T08:55:18.978193+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:20 smithi094 bash[59677]: cluster 2023-10-09T08:55:18.979102+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:20 smithi094 bash[59677]: cluster 2023-10-09T08:55:19.434381+0000 mgr.y (mgr.25421) 532 : cluster 0 pgmap v491: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 181 op/s; 5132/72 objects degraded (7127.778%); 4522/72 objects misplaced (6280.556%); 0 B/s, 392 objects/s recovering 2023-10-09T08:55:20.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:20 smithi094 bash[59677]: cluster 2023-10-09T08:55:19.987875+0000 mon.a (mon.0) 711 : cluster 0 osdmap e128: 8 total, 8 up, 7 in 2023-10-09T08:55:20.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[75179]: cluster 2023-10-09T08:55:18.973556+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[75179]: cluster 2023-10-09T08:55:18.976812+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[75179]: cluster 2023-10-09T08:55:18.978193+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[75179]: cluster 2023-10-09T08:55:18.979102+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[75179]: cluster 2023-10-09T08:55:19.434381+0000 mgr.y (mgr.25421) 532 : cluster 0 pgmap v491: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 181 op/s; 5132/72 objects degraded (7127.778%); 4522/72 objects misplaced (6280.556%); 0 B/s, 392 objects/s recovering 2023-10-09T08:55:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[75179]: cluster 2023-10-09T08:55:19.987875+0000 mon.a (mon.0) 711 : cluster 0 osdmap e128: 8 total, 8 up, 7 in 2023-10-09T08:55:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[77892]: cluster 2023-10-09T08:55:18.973556+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[77892]: cluster 2023-10-09T08:55:18.976812+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[77892]: cluster 2023-10-09T08:55:18.978193+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[77892]: cluster 2023-10-09T08:55:18.979102+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T08:55:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[77892]: cluster 2023-10-09T08:55:19.434381+0000 mgr.y (mgr.25421) 532 : cluster 0 pgmap v491: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 181 op/s; 5132/72 objects degraded (7127.778%); 4522/72 objects misplaced (6280.556%); 0 B/s, 392 objects/s recovering 2023-10-09T08:55:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:20 smithi016 bash[77892]: cluster 2023-10-09T08:55:19.987875+0000 mon.a (mon.0) 711 : cluster 0 osdmap e128: 8 total, 8 up, 7 in 2023-10-09T08:55:20.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:20.861 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:20.869 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:55:20.893 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:55:20.916 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:55:20.940 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:55:20.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:20.967 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:55:20.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:20.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:20.996 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:55:21.024 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:55:21.049 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:55:21.052 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.052 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.053 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.053 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.053 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.053 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.053 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.053 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:55:21.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:21.124 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:21.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:21.213 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T08:55:21.213 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:55:21.218 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:21.218 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:21.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:21.255 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:21.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:21.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:21.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:21.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:21.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:21.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:21.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:21.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:21.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:21.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:21.631 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[75179]: cluster 2023-10-09T08:55:19.983014+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.631 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[75179]: cluster 2023-10-09T08:55:19.986906+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[75179]: cluster 2023-10-09T08:55:19.988010+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[75179]: cluster 2023-10-09T08:55:19.988386+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[75179]: cluster 2023-10-09T08:55:20.122003+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[75179]: audit 2023-10-09T08:55:20.838687+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.16:0/2798881331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[75179]: cluster 2023-10-09T08:55:21.105886+0000 mon.a (mon.0) 712 : cluster 0 osdmap e129: 8 total, 8 up, 7 in 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[77892]: cluster 2023-10-09T08:55:19.983014+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[77892]: cluster 2023-10-09T08:55:19.986906+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[77892]: cluster 2023-10-09T08:55:19.988010+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.632 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[77892]: cluster 2023-10-09T08:55:19.988386+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.633 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[77892]: cluster 2023-10-09T08:55:20.122003+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.633 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[77892]: audit 2023-10-09T08:55:20.838687+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.16:0/2798881331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:21.633 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:21 smithi016 bash[77892]: cluster 2023-10-09T08:55:21.105886+0000 mon.a (mon.0) 712 : cluster 0 osdmap e129: 8 total, 8 up, 7 in 2023-10-09T08:55:21.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:21.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:21.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:21 smithi094 bash[59677]: cluster 2023-10-09T08:55:19.983014+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:21 smithi094 bash[59677]: cluster 2023-10-09T08:55:19.986906+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:21 smithi094 bash[59677]: cluster 2023-10-09T08:55:19.988010+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:21 smithi094 bash[59677]: cluster 2023-10-09T08:55:19.988386+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:21 smithi094 bash[59677]: cluster 2023-10-09T08:55:20.122003+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T08:55:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:21 smithi094 bash[59677]: audit 2023-10-09T08:55:20.838687+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.16:0/2798881331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:21 smithi094 bash[59677]: cluster 2023-10-09T08:55:21.105886+0000 mon.a (mon.0) 712 : cluster 0 osdmap e129: 8 total, 8 up, 7 in 2023-10-09T08:55:21.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:21.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:21.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:21.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:21.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:21.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:22.015 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:22.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:22.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T08:55:22.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:22.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:22.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:22.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:22.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:22.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:22.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:22.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:22.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:22.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:22.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:22.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:22.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[75179]: cluster 2023-10-09T08:55:21.100877+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[75179]: cluster 2023-10-09T08:55:21.105449+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[75179]: cluster 2023-10-09T08:55:21.107486+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[75179]: cluster 2023-10-09T08:55:21.240397+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[75179]: cluster 2023-10-09T08:55:21.435145+0000 mgr.y (mgr.25421) 533 : cluster 0 pgmap v494: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 583 objects/s recovering 2023-10-09T08:55:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[75179]: cluster 2023-10-09T08:55:22.106374+0000 mon.a (mon.0) 713 : cluster 0 osdmap e130: 8 total, 8 up, 7 in 2023-10-09T08:55:22.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[77892]: cluster 2023-10-09T08:55:21.100877+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[77892]: cluster 2023-10-09T08:55:21.105449+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[77892]: cluster 2023-10-09T08:55:21.107486+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[77892]: cluster 2023-10-09T08:55:21.240397+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[77892]: cluster 2023-10-09T08:55:21.435145+0000 mgr.y (mgr.25421) 533 : cluster 0 pgmap v494: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 583 objects/s recovering 2023-10-09T08:55:22.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:22 smithi016 bash[77892]: cluster 2023-10-09T08:55:22.106374+0000 mon.a (mon.0) 713 : cluster 0 osdmap e130: 8 total, 8 up, 7 in 2023-10-09T08:55:22.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:22.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:22.925 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:22.935 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:22 smithi094 bash[59677]: cluster 2023-10-09T08:55:21.100877+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:22 smithi094 bash[59677]: cluster 2023-10-09T08:55:21.105449+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:22 smithi094 bash[59677]: cluster 2023-10-09T08:55:21.107486+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:22 smithi094 bash[59677]: cluster 2023-10-09T08:55:21.240397+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T08:55:22.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:22 smithi094 bash[59677]: cluster 2023-10-09T08:55:21.435145+0000 mgr.y (mgr.25421) 533 : cluster 0 pgmap v494: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 583 objects/s recovering 2023-10-09T08:55:22.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:22 smithi094 bash[59677]: cluster 2023-10-09T08:55:22.106374+0000 mon.a (mon.0) 713 : cluster 0 osdmap e130: 8 total, 8 up, 7 in 2023-10-09T08:55:22.955 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:22.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:23.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:23.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:23.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:23.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:23.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:23.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:23.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:23.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:23.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:23.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:23.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:23.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:23.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:23.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:23.606 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:23.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:23.709 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:23.722 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:23 smithi094 bash[59677]: cluster 2023-10-09T08:55:22.110772+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:23.722 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:23 smithi094 bash[59677]: cluster 2023-10-09T08:55:22.815352+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:23.746 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:23.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:23.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:23 smithi016 bash[77892]: cluster 2023-10-09T08:55:22.110772+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:23 smithi016 bash[77892]: cluster 2023-10-09T08:55:22.815352+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:23 smithi016 bash[75179]: cluster 2023-10-09T08:55:22.110772+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:23 smithi016 bash[75179]: cluster 2023-10-09T08:55:22.815352+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:23.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:23.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:23.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:23.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:23.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:24.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:24.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:24.134 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:24.160 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:24.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:24.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:24.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:24.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:24.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:24.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:24.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:24.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:24.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:24.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:24 smithi016 bash[75179]: cluster 2023-10-09T08:55:22.833154+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:24.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:24 smithi016 bash[75179]: cluster 2023-10-09T08:55:23.436246+0000 mgr.y (mgr.25421) 534 : cluster 0 pgmap v496: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 482 objects/s recovering 2023-10-09T08:55:24.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:24 smithi016 bash[75179]: cluster 2023-10-09T08:55:23.972290+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 4354/72 objects degraded (6047.222%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:24.648 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:24 smithi016 bash[77892]: cluster 2023-10-09T08:55:22.833154+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:24.648 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:24 smithi016 bash[77892]: cluster 2023-10-09T08:55:23.436246+0000 mgr.y (mgr.25421) 534 : cluster 0 pgmap v496: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 482 objects/s recovering 2023-10-09T08:55:24.648 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:24 smithi016 bash[77892]: cluster 2023-10-09T08:55:23.972290+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 4354/72 objects degraded (6047.222%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:24.650 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:24 smithi094 bash[59677]: cluster 2023-10-09T08:55:22.833154+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T08:55:24.650 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:24 smithi094 bash[59677]: cluster 2023-10-09T08:55:23.436246+0000 mgr.y (mgr.25421) 534 : cluster 0 pgmap v496: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 482 objects/s recovering 2023-10-09T08:55:24.650 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:24 smithi094 bash[59677]: cluster 2023-10-09T08:55:23.972290+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 4354/72 objects degraded (6047.222%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:24.666 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:24.769 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:24.783 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:55:24.783 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:55:24.783 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T08:55:24.783 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T08:55:24.784 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T08:55:24.784 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T08:55:24.794 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:24.904 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:24.930 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:24.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:24.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:25.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:25.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:25.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:55:25.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:25.193 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:25.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:25.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:25.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:25.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:25.452 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:25.557 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:25.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:25.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:25.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:25.719 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:25.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:25.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:25.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:25.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:25.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:25.969 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:25 smithi094 bash[59677]: audit 2023-10-09T08:55:24.687656+0000 mon.a (mon.0) 715 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:25.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:25 smithi094 bash[59677]: audit 2023-10-09T08:55:24.693445+0000 mon.a (mon.0) 716 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:25.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:25 smithi094 bash[59677]: audit 2023-10-09T08:55:24.866873+0000 mon.a (mon.0) 717 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:25.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:25 smithi094 bash[59677]: audit 2023-10-09T08:55:24.872474+0000 mon.a (mon.0) 718 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:25.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:25 smithi094 bash[59677]: audit 2023-10-09T08:55:25.344721+0000 mon.a (mon.0) 719 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:25.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:25 smithi094 bash[59677]: audit 2023-10-09T08:55:25.345863+0000 mon.a (mon.0) 720 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:25.971 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:25 smithi094 bash[59677]: audit 2023-10-09T08:55:25.353571+0000 mon.a (mon.0) 721 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:25.988 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:25.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[75179]: audit 2023-10-09T08:55:24.687656+0000 mon.a (mon.0) 715 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[75179]: audit 2023-10-09T08:55:24.693445+0000 mon.a (mon.0) 716 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[75179]: audit 2023-10-09T08:55:24.866873+0000 mon.a (mon.0) 717 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[75179]: audit 2023-10-09T08:55:24.872474+0000 mon.a (mon.0) 718 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[75179]: audit 2023-10-09T08:55:25.344721+0000 mon.a (mon.0) 719 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[75179]: audit 2023-10-09T08:55:25.345863+0000 mon.a (mon.0) 720 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[75179]: audit 2023-10-09T08:55:25.353571+0000 mon.a (mon.0) 721 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:55:25] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-09T08:55:26.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[77892]: audit 2023-10-09T08:55:24.687656+0000 mon.a (mon.0) 715 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[77892]: audit 2023-10-09T08:55:24.693445+0000 mon.a (mon.0) 716 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[77892]: audit 2023-10-09T08:55:24.866873+0000 mon.a (mon.0) 717 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[77892]: audit 2023-10-09T08:55:24.872474+0000 mon.a (mon.0) 718 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[77892]: audit 2023-10-09T08:55:25.344721+0000 mon.a (mon.0) 719 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:26.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[77892]: audit 2023-10-09T08:55:25.345863+0000 mon.a (mon.0) 720 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:26.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:25 smithi016 bash[77892]: audit 2023-10-09T08:55:25.353571+0000 mon.a (mon.0) 721 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:26.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:26.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:26.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:26.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:26.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:26.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:26.357 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:26.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:26.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:26.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:26.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:26.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:26.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:26.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:26.623 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:26.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:26.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:26.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:26.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:26.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:26.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:26.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:26.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:27.012 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:27.036 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:27.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:27.139 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:27.162 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:27.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:27.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:27.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:27.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:27.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:27.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:27.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:27.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:27 smithi094 bash[59677]: cluster 2023-10-09T08:55:25.436986+0000 mgr.y (mgr.25421) 535 : cluster 0 pgmap v497: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 359 objects/s recovering 2023-10-09T08:55:27.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:27.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:27.666 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:27 smithi016 bash[75179]: cluster 2023-10-09T08:55:25.436986+0000 mgr.y (mgr.25421) 535 : cluster 0 pgmap v497: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 359 objects/s recovering 2023-10-09T08:55:27.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:27 smithi016 bash[77892]: cluster 2023-10-09T08:55:25.436986+0000 mgr.y (mgr.25421) 535 : cluster 0 pgmap v497: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4354/72 objects degraded (6047.222%); 4522/72 objects misplaced (6280.556%); 0 B/s, 359 objects/s recovering 2023-10-09T08:55:27.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:27.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:27.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:27.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:27.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:28.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:28.075 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:28.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:28.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:28.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:28.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:28.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:28.443 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:28.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:28.572 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:28.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:28.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:28.644 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:28.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:28.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:28.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:28.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:28.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:28.988 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T08:55:28.990 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_0' removed 2023-10-09T08:55:28.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:28.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:28.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:29.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:29.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:29.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:29.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:29.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:29.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:29.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:29.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:29.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:29.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:29.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:29.518 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: cluster 2023-10-09T08:55:27.438209+0000 mgr.y (mgr.25421) 536 : cluster 0 pgmap v498: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4237/72 objects degraded (5884.722%); 4522/72 objects misplaced (6280.556%); 0 B/s, 304 objects/s recovering 2023-10-09T08:55:29.518 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: audit 2023-10-09T08:55:28.717951+0000 mon.c (mon.2) 56 : audit 1 from='client.? 172.21.15.16:0/3749127128' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T08:55:29.518 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: audit 2023-10-09T08:55:28.718626+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T08:55:29.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: audit 2023-10-09T08:55:28.720571+0000 mon.a (mon.0) 723 : audit 1 from='client.? 172.21.15.16:0/3690124806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:55:29.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: cluster 2023-10-09T08:55:28.973920+0000 mon.a (mon.0) 724 : cluster 3 Health check update: Degraded data redundancy: 4237/72 objects degraded (5884.722%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:29.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: cluster 2023-10-09T08:55:28.974899+0000 mon.a (mon.0) 725 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:55:29.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: audit 2023-10-09T08:55:28.981050+0000 mon.a (mon.0) 726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T08:55:29.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: audit 2023-10-09T08:55:28.981138+0000 mon.a (mon.0) 727 : audit 1 from='client.? 172.21.15.16:0/3690124806' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:55:29.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[75179]: cluster 2023-10-09T08:55:28.981235+0000 mon.a (mon.0) 728 : cluster 0 osdmap e131: 8 total, 8 up, 7 in 2023-10-09T08:55:29.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: cluster 2023-10-09T08:55:27.438209+0000 mgr.y (mgr.25421) 536 : cluster 0 pgmap v498: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4237/72 objects degraded (5884.722%); 4522/72 objects misplaced (6280.556%); 0 B/s, 304 objects/s recovering 2023-10-09T08:55:29.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: audit 2023-10-09T08:55:28.717951+0000 mon.c (mon.2) 56 : audit 1 from='client.? 172.21.15.16:0/3749127128' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T08:55:29.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: audit 2023-10-09T08:55:28.718626+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T08:55:29.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: audit 2023-10-09T08:55:28.720571+0000 mon.a (mon.0) 723 : audit 1 from='client.? 172.21.15.16:0/3690124806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:55:29.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: cluster 2023-10-09T08:55:28.973920+0000 mon.a (mon.0) 724 : cluster 3 Health check update: Degraded data redundancy: 4237/72 objects degraded (5884.722%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:29.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: cluster 2023-10-09T08:55:28.974899+0000 mon.a (mon.0) 725 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:55:29.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: audit 2023-10-09T08:55:28.981050+0000 mon.a (mon.0) 726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T08:55:29.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: audit 2023-10-09T08:55:28.981138+0000 mon.a (mon.0) 727 : audit 1 from='client.? 172.21.15.16:0/3690124806' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:55:29.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:29 smithi016 bash[77892]: cluster 2023-10-09T08:55:28.981235+0000 mon.a (mon.0) 728 : cluster 0 osdmap e131: 8 total, 8 up, 7 in 2023-10-09T08:55:29.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:29.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:29.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:29.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:29.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: cluster 2023-10-09T08:55:27.438209+0000 mgr.y (mgr.25421) 536 : cluster 0 pgmap v498: 89 pgs: 2 active+recovery_wait+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 4237/72 objects degraded (5884.722%); 4522/72 objects misplaced (6280.556%); 0 B/s, 304 objects/s recovering 2023-10-09T08:55:29.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: audit 2023-10-09T08:55:28.717951+0000 mon.c (mon.2) 56 : audit 1 from='client.? 172.21.15.16:0/3749127128' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T08:55:29.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: audit 2023-10-09T08:55:28.718626+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T08:55:29.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: audit 2023-10-09T08:55:28.720571+0000 mon.a (mon.0) 723 : audit 1 from='client.? 172.21.15.16:0/3690124806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:55:29.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: cluster 2023-10-09T08:55:28.973920+0000 mon.a (mon.0) 724 : cluster 3 Health check update: Degraded data redundancy: 4237/72 objects degraded (5884.722%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:55:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: cluster 2023-10-09T08:55:28.974899+0000 mon.a (mon.0) 725 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:55:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: audit 2023-10-09T08:55:28.981050+0000 mon.a (mon.0) 726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T08:55:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: audit 2023-10-09T08:55:28.981138+0000 mon.a (mon.0) 727 : audit 1 from='client.? 172.21.15.16:0/3690124806' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:55:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:29 smithi094 bash[59677]: cluster 2023-10-09T08:55:28.981235+0000 mon.a (mon.0) 728 : cluster 0 osdmap e131: 8 total, 8 up, 7 in 2023-10-09T08:55:29.660 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:29.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:29.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:29.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:29.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:29.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:29.891 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:29.916 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:29.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:30.019 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:30.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:30.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:30.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:30.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:30.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:30.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:30.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:30.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:30.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:30 smithi016 bash[77892]: cluster 2023-10-09T08:55:28.978297+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T08:55:30.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:30 smithi016 bash[77892]: cluster 2023-10-09T08:55:28.978732+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T08:55:30.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:30 smithi016 bash[77892]: cluster 2023-10-09T08:55:29.990962+0000 mon.a (mon.0) 729 : cluster 0 osdmap e132: 8 total, 8 up, 7 in 2023-10-09T08:55:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:30 smithi016 bash[75179]: cluster 2023-10-09T08:55:28.978297+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T08:55:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:30 smithi016 bash[75179]: cluster 2023-10-09T08:55:28.978732+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T08:55:30.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:30 smithi016 bash[75179]: cluster 2023-10-09T08:55:29.990962+0000 mon.a (mon.0) 729 : cluster 0 osdmap e132: 8 total, 8 up, 7 in 2023-10-09T08:55:30.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:30.544 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T08:55:30.544 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T08:55:30.544 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T08:55:30.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T08:55:30.553 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:30 smithi094 bash[59677]: cluster 2023-10-09T08:55:28.978297+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T08:55:30.553 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:30 smithi094 bash[59677]: cluster 2023-10-09T08:55:28.978732+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T08:55:30.553 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:30 smithi094 bash[59677]: cluster 2023-10-09T08:55:29.990962+0000 mon.a (mon.0) 729 : cluster 0 osdmap e132: 8 total, 8 up, 7 in 2023-10-09T08:55:30.565 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:30.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:30.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:30.806 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:30.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:30.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:30.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:30.963 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:30.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:30.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:31.045 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:31.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:31.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:31.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:31.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:31.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:31.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:31.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:31.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:31.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:31.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:31.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:31.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:31.486 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:31.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:31.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:31.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:31.618 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:31.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:31.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:31 smithi094 bash[59677]: cluster 2023-10-09T08:55:29.438869+0000 mgr.y (mgr.25421) 537 : cluster 0 pgmap v500: 73 pgs: 72 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:55:31.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:31 smithi094 bash[59677]: cluster 2023-10-09T08:55:29.987815+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:31 smithi094 bash[59677]: cluster 2023-10-09T08:55:29.994502+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:31 smithi094 bash[59677]: cluster 2023-10-09T08:55:29.994550+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:31 smithi094 bash[59677]: cluster 2023-10-09T08:55:30.209574+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.656 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:31 smithi094 bash[59677]: cluster 2023-10-09T08:55:30.995497+0000 mon.a (mon.0) 730 : cluster 0 osdmap e133: 8 total, 8 up, 7 in 2023-10-09T08:55:31.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:31.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[77892]: cluster 2023-10-09T08:55:29.438869+0000 mgr.y (mgr.25421) 537 : cluster 0 pgmap v500: 73 pgs: 72 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[77892]: cluster 2023-10-09T08:55:29.987815+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[77892]: cluster 2023-10-09T08:55:29.994502+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[77892]: cluster 2023-10-09T08:55:29.994550+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[77892]: cluster 2023-10-09T08:55:30.209574+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[77892]: cluster 2023-10-09T08:55:30.995497+0000 mon.a (mon.0) 730 : cluster 0 osdmap e133: 8 total, 8 up, 7 in 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[75179]: cluster 2023-10-09T08:55:29.438869+0000 mgr.y (mgr.25421) 537 : cluster 0 pgmap v500: 73 pgs: 72 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[75179]: cluster 2023-10-09T08:55:29.987815+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[75179]: cluster 2023-10-09T08:55:29.994502+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.736 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[75179]: cluster 2023-10-09T08:55:29.994550+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.736 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[75179]: cluster 2023-10-09T08:55:30.209574+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T08:55:31.736 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:31 smithi016 bash[75179]: cluster 2023-10-09T08:55:30.995497+0000 mon.a (mon.0) 730 : cluster 0 osdmap e133: 8 total, 8 up, 7 in 2023-10-09T08:55:31.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:31.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:31.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:31.883 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:31.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:31.986 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:32.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:32.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:32.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:32.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:32.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:32.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:32.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:32.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:32.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:32.398 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:32 smithi016 bash[77892]: cluster 2023-10-09T08:55:30.989770+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.398 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:32 smithi016 bash[77892]: cluster 2023-10-09T08:55:30.997009+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.398 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:32 smithi016 bash[77892]: cluster 2023-10-09T08:55:31.020407+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:32 smithi016 bash[75179]: cluster 2023-10-09T08:55:30.989770+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:32 smithi016 bash[75179]: cluster 2023-10-09T08:55:30.997009+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.399 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:32 smithi016 bash[75179]: cluster 2023-10-09T08:55:31.020407+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:32.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:32.519 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:32.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:32.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:32.555 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T08:55:32.555 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:55:32.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T08:55:32.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:32.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:32 smithi094 bash[59677]: cluster 2023-10-09T08:55:30.989770+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:32 smithi094 bash[59677]: cluster 2023-10-09T08:55:30.997009+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:32 smithi094 bash[59677]: cluster 2023-10-09T08:55:31.020407+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T08:55:32.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:32.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:32.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:32.930 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:32.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:33.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:33.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:33.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:33.164 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:33.196 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:33.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:33.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:33.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:33.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:33.434 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:33.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:33 smithi094 bash[59677]: cluster 2023-10-09T08:55:31.439523+0000 mgr.y (mgr.25421) 538 : cluster 0 pgmap v503: 73 pgs: 72 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:55:33.565 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:33.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:33.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:33.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:33.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:33.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:33 smithi016 bash[75179]: cluster 2023-10-09T08:55:31.439523+0000 mgr.y (mgr.25421) 538 : cluster 0 pgmap v503: 73 pgs: 72 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:55:33.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:33 smithi016 bash[77892]: cluster 2023-10-09T08:55:31.439523+0000 mgr.y (mgr.25421) 538 : cluster 0 pgmap v503: 73 pgs: 72 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T08:55:33.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:33.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:33.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:33.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:33.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:34.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:34.107 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:34.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:34.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:34.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:34.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:34.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:34.370 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:34.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:34.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:34.474 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:34.497 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:34.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:34 smithi016 bash[75179]: cluster 2023-10-09T08:55:33.976536+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:55:34.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:34 smithi016 bash[75179]: audit 2023-10-09T08:55:34.213746+0000 mon.a (mon.0) 732 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:34.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:34 smithi016 bash[77892]: cluster 2023-10-09T08:55:33.976536+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:55:34.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:34 smithi016 bash[77892]: audit 2023-10-09T08:55:34.213746+0000 mon.a (mon.0) 732 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:34.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:34.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:34.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:34.628 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:34.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:34.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:34.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:34 smithi094 bash[59677]: cluster 2023-10-09T08:55:33.976536+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:55:34.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:34 smithi094 bash[59677]: audit 2023-10-09T08:55:34.213746+0000 mon.a (mon.0) 732 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:34.758 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:34.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:34.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:34.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T08:55:34.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:34.886 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:55:34.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:34.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:34.911 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:55:34.936 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:55:34.968 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:55:34.990 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:55:34.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:35.019 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:55:35.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:35.048 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:55:35.073 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:55:35.076 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.076 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.076 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.076 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.076 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.076 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.076 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.077 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:55:35.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:35.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:35.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:35.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:35.285 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:55:35.285 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T08:55:35.285 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:55:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:35.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:35.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:35.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:35.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:35.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:35.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:35.661 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:35 smithi016 bash[75179]: cluster 2023-10-09T08:55:33.440112+0000 mgr.y (mgr.25421) 539 : cluster 0 pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:35.661 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:35 smithi016 bash[75179]: cluster 2023-10-09T08:55:34.375753+0000 mon.a (mon.0) 733 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T08:55:35.661 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:35 smithi016 bash[75179]: audit 2023-10-09T08:55:34.858162+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.16:0/531113353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:35.662 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:35 smithi016 bash[77892]: cluster 2023-10-09T08:55:33.440112+0000 mgr.y (mgr.25421) 539 : cluster 0 pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:35.662 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:35 smithi016 bash[77892]: cluster 2023-10-09T08:55:34.375753+0000 mon.a (mon.0) 733 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T08:55:35.662 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:35 smithi016 bash[77892]: audit 2023-10-09T08:55:34.858162+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.16:0/531113353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:35.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:35.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:35.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:35 smithi094 bash[59677]: cluster 2023-10-09T08:55:33.440112+0000 mgr.y (mgr.25421) 539 : cluster 0 pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:35.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:35 smithi094 bash[59677]: cluster 2023-10-09T08:55:34.375753+0000 mon.a (mon.0) 733 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T08:55:35.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:35 smithi094 bash[59677]: audit 2023-10-09T08:55:34.858162+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.16:0/531113353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:35.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:35.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:35.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:35.930 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:55:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:55:35] "GET /metrics HTTP/1.1" 200 34056 "" "Prometheus/2.33.4" 2023-10-09T08:55:35.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:35.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:35.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:36.050 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:36.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:36.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:36.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:36.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:36.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:36.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:36.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:36.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:36.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:36.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:36.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:36.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:36.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:36.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:36.616 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:36.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:36.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:36.629 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:36.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:36.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:36.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:36.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:36.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:36.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:36.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:37.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:37.122 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:37.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:37.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:37.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:37.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:37.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:37.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:37.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:37.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:37.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:37.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:37.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:37.646 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:37.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:37.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:37 smithi094 bash[59677]: cluster 2023-10-09T08:55:35.440726+0000 mgr.y (mgr.25421) 540 : cluster 0 pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-10-09T08:55:37.677 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:37 smithi016 bash[75179]: cluster 2023-10-09T08:55:35.440726+0000 mgr.y (mgr.25421) 540 : cluster 0 pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-10-09T08:55:37.677 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:37 smithi016 bash[77892]: cluster 2023-10-09T08:55:35.440726+0000 mgr.y (mgr.25421) 540 : cluster 0 pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-10-09T08:55:37.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:37.788 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:37.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:37.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:37.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:37.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:38.047 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:38.079 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:38.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:38.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:55:38.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:38.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:38.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:38.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:38.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:38.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:38.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:38.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:38.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:38.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:38.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:38.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:38.572 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:38.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:38.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:38.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:38.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:38.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:38.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:38.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:38.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:38.973 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:39.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:39.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:39.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:39.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:39.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:39.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:39.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:39.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:39.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:39.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:39.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:39.646 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:39 smithi016 bash[77892]: cluster 2023-10-09T08:55:37.441892+0000 mgr.y (mgr.25421) 541 : cluster 0 pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T08:55:39.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:39 smithi016 bash[75179]: cluster 2023-10-09T08:55:37.441892+0000 mgr.y (mgr.25421) 541 : cluster 0 pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T08:55:39.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:39.664 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:39.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:39.699 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:39 smithi094 bash[59677]: cluster 2023-10-09T08:55:37.441892+0000 mgr.y (mgr.25421) 541 : cluster 0 pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T08:55:39.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:39.796 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:39.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:39.930 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:39.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:40.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:40.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:40.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:40.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:40.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:40.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:40.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:40.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:40.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:40.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:40.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:40.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:40.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:40.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:40.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:40 smithi016 bash[75179]: cluster 2023-10-09T08:55:39.442383+0000 mgr.y (mgr.25421) 542 : cluster 0 pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T08:55:40.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:40 smithi016 bash[75179]: audit 2023-10-09T08:55:39.447808+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:40.559 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:40 smithi016 bash[77892]: cluster 2023-10-09T08:55:39.442383+0000 mgr.y (mgr.25421) 542 : cluster 0 pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T08:55:40.559 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:40 smithi016 bash[77892]: audit 2023-10-09T08:55:39.447808+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:40.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:40.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:40.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:40.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:40.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:40.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:40 smithi094 bash[59677]: cluster 2023-10-09T08:55:39.442383+0000 mgr.y (mgr.25421) 542 : cluster 0 pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T08:55:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:40 smithi094 bash[59677]: audit 2023-10-09T08:55:39.447808+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:40.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:40.830 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:40.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:40.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:41.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:41.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:41.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:41.137 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:55:41.138 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:55:41.138 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T08:55:41.138 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T08:55:41.138 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T08:55:41.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T08:55:41.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:41.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:41.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:41.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:41.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:41.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:41.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:41.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:41.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:41.681 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:41.784 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:41.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:41.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:41.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:41.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:42.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:42.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:42.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:42.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:42.209 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:42.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:42.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:42.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:42.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:42.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:42.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:42.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:42.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:42.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:42.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:42.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:42.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:42.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:42.722 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:42 smithi094 bash[59677]: cluster 2023-10-09T08:55:41.443257+0000 mgr.y (mgr.25421) 543 : cluster 0 pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T08:55:42.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:42.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:42.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:42 smithi016 bash[75179]: cluster 2023-10-09T08:55:41.443257+0000 mgr.y (mgr.25421) 543 : cluster 0 pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T08:55:42.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:42 smithi016 bash[77892]: cluster 2023-10-09T08:55:41.443257+0000 mgr.y (mgr.25421) 543 : cluster 0 pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T08:55:42.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:42.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:42.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:42.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:43.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:43.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:43.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:43.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:43.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:43.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:43.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:43.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:43.332 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:55:43.332 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T08:55:43.333 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T08:55:43.333 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T08:55:43.333 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:55:43.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:43.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:43.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:43.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:43.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:43.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:43.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:43.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:43.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:43.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:43.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:43.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:43.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:43.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:44.022 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:44.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:44.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:44.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:44.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:44.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T08:55:44.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:44.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:44.474 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_1' created 2023-10-09T08:55:44.476 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T08:55:44.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:44.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:44.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[75179]: cluster 2023-10-09T08:55:43.443904+0000 mgr.y (mgr.25421) 544 : cluster 0 pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T08:55:44.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[75179]: audit 2023-10-09T08:55:43.913682+0000 mon.a (mon.0) 735 : audit 1 from='client.? 172.21.15.16:0/1737460746' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T08:55:44.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[75179]: audit 2023-10-09T08:55:44.265636+0000 mon.c (mon.2) 58 : audit 1 from='client.? 172.21.15.16:0/792707459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:44.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[75179]: audit 2023-10-09T08:55:44.387773+0000 mon.a (mon.0) 736 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:44.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[77892]: cluster 2023-10-09T08:55:43.443904+0000 mgr.y (mgr.25421) 544 : cluster 0 pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T08:55:44.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[77892]: audit 2023-10-09T08:55:43.913682+0000 mon.a (mon.0) 735 : audit 1 from='client.? 172.21.15.16:0/1737460746' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T08:55:44.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[77892]: audit 2023-10-09T08:55:44.265636+0000 mon.c (mon.2) 58 : audit 1 from='client.? 172.21.15.16:0/792707459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:44.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:44 smithi016 bash[77892]: audit 2023-10-09T08:55:44.387773+0000 mon.a (mon.0) 736 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:44.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:44.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:44 smithi094 bash[59677]: cluster 2023-10-09T08:55:43.443904+0000 mgr.y (mgr.25421) 544 : cluster 0 pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T08:55:44.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:44 smithi094 bash[59677]: audit 2023-10-09T08:55:43.913682+0000 mon.a (mon.0) 735 : audit 1 from='client.? 172.21.15.16:0/1737460746' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T08:55:44.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:44 smithi094 bash[59677]: audit 2023-10-09T08:55:44.265636+0000 mon.c (mon.2) 58 : audit 1 from='client.? 172.21.15.16:0/792707459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:44.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:44 smithi094 bash[59677]: audit 2023-10-09T08:55:44.387773+0000 mon.a (mon.0) 736 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:55:44.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:44.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:44.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:44.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:44.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:45.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:45.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:45.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:45.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:45.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:45.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:45.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:45.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:45.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:45.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:45.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:45.490 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: cluster 2023-10-09T08:55:44.452323+0000 mon.a (mon.0) 737 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: cluster 2023-10-09T08:55:44.452381+0000 mon.a (mon.0) 738 : cluster 1 Cluster is now healthy 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: cluster 2023-10-09T08:55:44.462721+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: cluster 2023-10-09T08:55:44.463275+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: audit 2023-10-09T08:55:44.463481+0000 mon.a (mon.0) 739 : audit 1 from='client.? 172.21.15.16:0/1737460746' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: audit 2023-10-09T08:55:44.463678+0000 mon.a (mon.0) 740 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: cluster 2023-10-09T08:55:44.463995+0000 mon.a (mon.0) 741 : cluster 0 osdmap e134: 8 total, 8 up, 7 in 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: cluster 2023-10-09T08:55:44.465621+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[75179]: cluster 2023-10-09T08:55:44.609290+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.491 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: cluster 2023-10-09T08:55:44.452323+0000 mon.a (mon.0) 737 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: cluster 2023-10-09T08:55:44.452381+0000 mon.a (mon.0) 738 : cluster 1 Cluster is now healthy 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: cluster 2023-10-09T08:55:44.462721+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: cluster 2023-10-09T08:55:44.463275+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: audit 2023-10-09T08:55:44.463481+0000 mon.a (mon.0) 739 : audit 1 from='client.? 172.21.15.16:0/1737460746' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: audit 2023-10-09T08:55:44.463678+0000 mon.a (mon.0) 740 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: cluster 2023-10-09T08:55:44.463995+0000 mon.a (mon.0) 741 : cluster 0 osdmap e134: 8 total, 8 up, 7 in 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: cluster 2023-10-09T08:55:44.465621+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[77892]: cluster 2023-10-09T08:55:44.609290+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:45.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:45.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:45.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:45.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:45.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: cluster 2023-10-09T08:55:44.452323+0000 mon.a (mon.0) 737 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:55:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: cluster 2023-10-09T08:55:44.452381+0000 mon.a (mon.0) 738 : cluster 1 Cluster is now healthy 2023-10-09T08:55:45.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: cluster 2023-10-09T08:55:44.462721+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: cluster 2023-10-09T08:55:44.463275+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: audit 2023-10-09T08:55:44.463481+0000 mon.a (mon.0) 739 : audit 1 from='client.? 172.21.15.16:0/1737460746' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T08:55:45.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: audit 2023-10-09T08:55:44.463678+0000 mon.a (mon.0) 740 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:55:45.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: cluster 2023-10-09T08:55:44.463995+0000 mon.a (mon.0) 741 : cluster 0 osdmap e134: 8 total, 8 up, 7 in 2023-10-09T08:55:45.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: cluster 2023-10-09T08:55:44.465621+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:45 smithi094 bash[59677]: cluster 2023-10-09T08:55:44.609290+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T08:55:45.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:45.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:55:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:55:45] "GET /metrics HTTP/1.1" 200 34056 "" "Prometheus/2.33.4" 2023-10-09T08:55:45.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:45.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:45.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:45.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:45.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:45.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:45.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T08:55:46.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:46.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:46.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:46.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:46.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:46.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:46.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:46.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:46.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:46.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:46.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:46.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:46.432 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:46.454 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:46.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:46.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:46.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:46.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:46.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[75179]: cluster 2023-10-09T08:55:45.444673+0000 mgr.y (mgr.25421) 545 : cluster 0 pgmap v511: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:46.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[75179]: cluster 2023-10-09T08:55:45.469786+0000 mon.a (mon.0) 742 : cluster 0 osdmap e135: 8 total, 8 up, 7 in 2023-10-09T08:55:46.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[75179]: cluster 2023-10-09T08:55:45.470940+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T08:55:46.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[75179]: cluster 2023-10-09T08:55:45.472965+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T08:55:46.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[77892]: cluster 2023-10-09T08:55:45.444673+0000 mgr.y (mgr.25421) 545 : cluster 0 pgmap v511: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:46.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[77892]: cluster 2023-10-09T08:55:45.469786+0000 mon.a (mon.0) 742 : cluster 0 osdmap e135: 8 total, 8 up, 7 in 2023-10-09T08:55:46.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[77892]: cluster 2023-10-09T08:55:45.470940+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T08:55:46.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:46 smithi016 bash[77892]: cluster 2023-10-09T08:55:45.472965+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T08:55:46.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:46.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:46.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:46.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:46.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:46 smithi094 bash[59677]: cluster 2023-10-09T08:55:45.444673+0000 mgr.y (mgr.25421) 545 : cluster 0 pgmap v511: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:46.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:46 smithi094 bash[59677]: cluster 2023-10-09T08:55:45.469786+0000 mon.a (mon.0) 742 : cluster 0 osdmap e135: 8 total, 8 up, 7 in 2023-10-09T08:55:46.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:46 smithi094 bash[59677]: cluster 2023-10-09T08:55:45.470940+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T08:55:46.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:46 smithi094 bash[59677]: cluster 2023-10-09T08:55:45.472965+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T08:55:46.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:46.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:46.782 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:46.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:46.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:46.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:46.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:47.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:47.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:47.125 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:47.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:47.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:47.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:47.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:47.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:47.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:47.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:47.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:47.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:47.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:47.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:47.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:47.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:47.668 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[77892]: cluster 2023-10-09T08:55:46.474503+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.669 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[77892]: cluster 2023-10-09T08:55:46.474682+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.669 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[77892]: cluster 2023-10-09T08:55:46.477200+0000 mon.a (mon.0) 743 : cluster 0 osdmap e136: 8 total, 8 up, 7 in 2023-10-09T08:55:47.669 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[77892]: cluster 2023-10-09T08:55:46.785326+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.669 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[75179]: cluster 2023-10-09T08:55:46.474503+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.669 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[75179]: cluster 2023-10-09T08:55:46.474682+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.669 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[75179]: cluster 2023-10-09T08:55:46.477200+0000 mon.a (mon.0) 743 : cluster 0 osdmap e136: 8 total, 8 up, 7 in 2023-10-09T08:55:47.669 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:47 smithi016 bash[75179]: cluster 2023-10-09T08:55:46.785326+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:47.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:47 smithi094 bash[59677]: cluster 2023-10-09T08:55:46.474503+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:47 smithi094 bash[59677]: cluster 2023-10-09T08:55:46.474682+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:47 smithi094 bash[59677]: cluster 2023-10-09T08:55:46.477200+0000 mon.a (mon.0) 743 : cluster 0 osdmap e136: 8 total, 8 up, 7 in 2023-10-09T08:55:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:47 smithi094 bash[59677]: cluster 2023-10-09T08:55:46.785326+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T08:55:47.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:47.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:47.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:47.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:47.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T08:55:47.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:48.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:48.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:48.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:48.144 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:48.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:48.275 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:48.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:48.379 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:48.401 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:48.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:48.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:48.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:48.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:48.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:48.652 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:48 smithi094 bash[59677]: cluster 2023-10-09T08:55:47.445654+0000 mgr.y (mgr.25421) 546 : cluster 0 pgmap v514: 89 pgs: 3 creating+peering, 3 unknown, 83 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:48.673 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:48.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:48 smithi016 bash[75179]: cluster 2023-10-09T08:55:47.445654+0000 mgr.y (mgr.25421) 546 : cluster 0 pgmap v514: 89 pgs: 3 creating+peering, 3 unknown, 83 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:48 smithi016 bash[77892]: cluster 2023-10-09T08:55:47.445654+0000 mgr.y (mgr.25421) 546 : cluster 0 pgmap v514: 89 pgs: 3 creating+peering, 3 unknown, 83 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:48.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:48.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:48.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:48.912 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:48.940 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:48.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:49.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:49.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:49.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:49.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:49.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:49.206 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:49.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:49.310 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:49.340 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:49.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:49.443 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:49.474 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:49.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:49.581 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:49.607 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:49.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:49.667 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:55:49.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:49.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:49.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:49.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:49.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:49.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:49.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:49.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:50.012 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:50.116 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:50.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:50.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:50.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:50.275 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:50.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:50.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:50.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:50 smithi094 bash[59677]: audit 2023-10-09T08:55:49.220989+0000 mon.a (mon.0) 744 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:50.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:50 smithi094 bash[59677]: audit 2023-10-09T08:55:49.222336+0000 mon.a (mon.0) 745 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:50.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:50.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:50.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:50 smithi016 bash[75179]: audit 2023-10-09T08:55:49.220989+0000 mon.a (mon.0) 744 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:50.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:50 smithi016 bash[75179]: audit 2023-10-09T08:55:49.222336+0000 mon.a (mon.0) 745 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:50.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:50 smithi016 bash[77892]: audit 2023-10-09T08:55:49.220989+0000 mon.a (mon.0) 744 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:55:50.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:50 smithi016 bash[77892]: audit 2023-10-09T08:55:49.222336+0000 mon.a (mon.0) 745 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:55:50.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:50.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:50.598 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:55:50.625 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:55:50.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:50.649 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:55:50.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:50.674 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:55:50.700 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:55:50.723 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:55:50.748 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:55:50.769 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:55:50.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.772 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.773 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:55:50.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:50.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:50.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:50.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:50.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:51.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:51.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:51.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:51.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:51.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:51.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:51.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:51.431 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:51.455 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:51.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:51.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:51.688 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:51.696 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:51 smithi094 bash[59677]: cluster 2023-10-09T08:55:49.446448+0000 mgr.y (mgr.25421) 547 : cluster 0 pgmap v515: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:51.696 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:51 smithi094 bash[59677]: audit 2023-10-09T08:55:50.566407+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.16:0/3101729460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:51.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:51.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:51.746 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:51 smithi016 bash[75179]: cluster 2023-10-09T08:55:49.446448+0000 mgr.y (mgr.25421) 547 : cluster 0 pgmap v515: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:51.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:51 smithi016 bash[75179]: audit 2023-10-09T08:55:50.566407+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.16:0/3101729460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:51.747 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:51 smithi016 bash[77892]: cluster 2023-10-09T08:55:49.446448+0000 mgr.y (mgr.25421) 547 : cluster 0 pgmap v515: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:51.747 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:51 smithi016 bash[77892]: audit 2023-10-09T08:55:50.566407+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.16:0/3101729460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:55:51.800 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:55:51.801 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T08:55:51.801 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:55:51.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:51.836 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:51.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:51.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:51.966 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:51.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:51.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:52.069 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:52.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:52.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:52.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:52.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:52.228 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:52.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:52.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:52.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:52.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:52.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:52.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:52.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:52.497 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:52.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:52.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:52.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:52.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:52.730 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:52.751 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:52.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:52.854 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:52.877 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:52.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:52.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:52.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:53.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:53.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:53.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:53.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:53.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:53.243 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:55:53.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:53.273 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:53.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:53.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:53.511 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:53.519 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:53 smithi094 bash[59677]: cluster 2023-10-09T08:55:51.447411+0000 mgr.y (mgr.25421) 548 : cluster 0 pgmap v516: 89 pgs: 89 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:53.533 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:53.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:53.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:53 smithi016 bash[75179]: cluster 2023-10-09T08:55:51.447411+0000 mgr.y (mgr.25421) 548 : cluster 0 pgmap v516: 89 pgs: 89 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:53.647 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:53 smithi016 bash[77892]: cluster 2023-10-09T08:55:51.447411+0000 mgr.y (mgr.25421) 548 : cluster 0 pgmap v516: 89 pgs: 89 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:53.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:53.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:55:53.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:53.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:53.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:53.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:53.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:53.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:54.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:54.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:54.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:54.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:54.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:54.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:54.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:54.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:54.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:54.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:54.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:54.486 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:54.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:54.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:54.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:54.618 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:54.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:54.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:54.722 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:54.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:54.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:54.776 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T08:55:54.777 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:55:54.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:54.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:54.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:54.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:55.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:55.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:55.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:55.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:55.260 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:55.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:55.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:55.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:55.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:55 smithi016 bash[75179]: cluster 2023-10-09T08:55:53.448093+0000 mgr.y (mgr.25421) 549 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:55.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:55 smithi016 bash[77892]: cluster 2023-10-09T08:55:53.448093+0000 mgr.y (mgr.25421) 549 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:55.509 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:55 smithi094 bash[59677]: cluster 2023-10-09T08:55:53.448093+0000 mgr.y (mgr.25421) 549 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:55.526 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:55.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:55.629 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:55.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T08:55:55.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:55.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:55.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:55.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:55.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:55.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:55.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:56.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:55:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:55:55] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.33.4" 2023-10-09T08:55:56.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:56.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:56.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:56.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:56.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:56.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:56.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:56.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:56.385 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:56.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:56.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:56.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:56.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:56.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:56.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:56.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:56.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:56.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:56.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:56.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:56.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:56.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:56.888 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:56.918 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:55:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:56.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:55:57.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:57.053 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:57.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:57.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:57.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:55:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:57.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:55:57.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:57.322 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:57.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:57.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:57.438 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:57 smithi016 bash[75179]: cluster 2023-10-09T08:55:55.448962+0000 mgr.y (mgr.25421) 550 : cluster 0 pgmap v518: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:57.439 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:57 smithi016 bash[77892]: cluster 2023-10-09T08:55:55.448962+0000 mgr.y (mgr.25421) 550 : cluster 0 pgmap v518: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:57.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:57.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:57.562 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:57.573 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:57 smithi094 bash[59677]: cluster 2023-10-09T08:55:55.448962+0000 mgr.y (mgr.25421) 550 : cluster 0 pgmap v518: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:57.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:57.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:57.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:57.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:57.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:57.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:57.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:57.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:57.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:57.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:57.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:57.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:58.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:58.103 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:58.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:58.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:58.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:58.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:55:58.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:55:58.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:58.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:58.405 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:55:58.405 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:55:58.405 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T08:55:58.405 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T08:55:58.405 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T08:55:58.406 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T08:55:58.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:58.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:58.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:58.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:58.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:58.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:58.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:58.927 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:58.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:58.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:59.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:59.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:59.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:59.164 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:59.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:55:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:59.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:55:59.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:59.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:55:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:59.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:55:59.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:59.449 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:55:59 smithi094 bash[59677]: cluster 2023-10-09T08:55:57.449834+0000 mgr.y (mgr.25421) 551 : cluster 0 pgmap v519: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:59.470 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:55:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:59.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:55:59.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:59.587 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:55:59 smithi016 bash[75179]: cluster 2023-10-09T08:55:57.449834+0000 mgr.y (mgr.25421) 551 : cluster 0 pgmap v519: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:59.587 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:55:59 smithi016 bash[77892]: cluster 2023-10-09T08:55:57.449834+0000 mgr.y (mgr.25421) 551 : cluster 0 pgmap v519: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:55:59.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:59.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:59.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:59.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:55:59.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:59.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:55:59.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:59.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:55:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:55:59.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:55:59.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:00.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:00.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:00.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:00.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:00.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:00.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:00.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:00.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:00.455 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:56:00.479 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T08:56:00.479 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T08:56:00.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:00.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:00.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:00.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:00.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:00.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:00.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:00 smithi094 bash[59677]: cluster 2023-10-09T08:55:59.450376+0000 mgr.y (mgr.25421) 552 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:00.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:00 smithi094 bash[59677]: audit 2023-10-09T08:55:59.464752+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:00.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:00 smithi094 bash[59677]: audit 2023-10-09T08:55:59.695698+0000 mon.a (mon.0) 748 : audit 1 from='client.? 172.21.15.16:0/2168625407' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:56:00.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:00 smithi094 bash[59677]: audit 2023-10-09T08:56:00.050863+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.16:0/3364470615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:56:00.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:00 smithi094 bash[59677]: audit 2023-10-09T08:56:00.051694+0000 mon.a (mon.0) 749 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:56:00.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[75179]: cluster 2023-10-09T08:55:59.450376+0000 mgr.y (mgr.25421) 552 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:00.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[75179]: audit 2023-10-09T08:55:59.464752+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:00.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[75179]: audit 2023-10-09T08:55:59.695698+0000 mon.a (mon.0) 748 : audit 1 from='client.? 172.21.15.16:0/2168625407' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:56:00.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[75179]: audit 2023-10-09T08:56:00.050863+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.16:0/3364470615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:56:00.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[75179]: audit 2023-10-09T08:56:00.051694+0000 mon.a (mon.0) 749 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:56:00.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[77892]: cluster 2023-10-09T08:55:59.450376+0000 mgr.y (mgr.25421) 552 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:00.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[77892]: audit 2023-10-09T08:55:59.464752+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:00.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[77892]: audit 2023-10-09T08:55:59.695698+0000 mon.a (mon.0) 748 : audit 1 from='client.? 172.21.15.16:0/2168625407' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T08:56:00.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[77892]: audit 2023-10-09T08:56:00.050863+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.16:0/3364470615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:56:00.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:00 smithi016 bash[77892]: audit 2023-10-09T08:56:00.051694+0000 mon.a (mon.0) 749 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:56:00.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:00.823 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:00.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:00.963 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:00.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:01.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:01.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:01.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:01.198 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:01.225 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:01.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:01.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:01.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:01.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:01.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:01.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:01.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:01.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:01.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:01.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:01.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:01.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:01.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:01.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[75179]: cluster 2023-10-09T08:56:00.461858+0000 mon.a (mon.0) 750 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:01.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[75179]: cluster 2023-10-09T08:56:00.470606+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[75179]: cluster 2023-10-09T08:56:00.470898+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[75179]: audit 2023-10-09T08:56:00.473240+0000 mon.a (mon.0) 751 : audit 1 from='client.? 172.21.15.16:0/2168625407' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[75179]: audit 2023-10-09T08:56:00.473361+0000 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[75179]: cluster 2023-10-09T08:56:00.473443+0000 mon.a (mon.0) 753 : cluster 0 osdmap e137: 8 total, 8 up, 7 in 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[77892]: cluster 2023-10-09T08:56:00.461858+0000 mon.a (mon.0) 750 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[77892]: cluster 2023-10-09T08:56:00.470606+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[77892]: cluster 2023-10-09T08:56:00.470898+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[77892]: audit 2023-10-09T08:56:00.473240+0000 mon.a (mon.0) 751 : audit 1 from='client.? 172.21.15.16:0/2168625407' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:56:01.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[77892]: audit 2023-10-09T08:56:00.473361+0000 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:56:01.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:01 smithi016 bash[77892]: cluster 2023-10-09T08:56:00.473443+0000 mon.a (mon.0) 753 : cluster 0 osdmap e137: 8 total, 8 up, 7 in 2023-10-09T08:56:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:01 smithi094 bash[59677]: cluster 2023-10-09T08:56:00.461858+0000 mon.a (mon.0) 750 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:01 smithi094 bash[59677]: cluster 2023-10-09T08:56:00.470606+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T08:56:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:01 smithi094 bash[59677]: cluster 2023-10-09T08:56:00.470898+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T08:56:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:01 smithi094 bash[59677]: audit 2023-10-09T08:56:00.473240+0000 mon.a (mon.0) 751 : audit 1 from='client.? 172.21.15.16:0/2168625407' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T08:56:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:01 smithi094 bash[59677]: audit 2023-10-09T08:56:00.473361+0000 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:56:01.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:01 smithi094 bash[59677]: cluster 2023-10-09T08:56:00.473443+0000 mon.a (mon.0) 753 : cluster 0 osdmap e137: 8 total, 8 up, 7 in 2023-10-09T08:56:01.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:01.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:01.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:01.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:01.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:01.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:01.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:02.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:02.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:02.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:02.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:02.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:02.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:02.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:02.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:02.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:02.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:02.600 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:02.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:02.670 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:02 smithi016 bash[75179]: cluster 2023-10-09T08:56:01.450955+0000 mgr.y (mgr.25421) 553 : cluster 0 pgmap v522: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:02.671 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:02 smithi016 bash[77892]: cluster 2023-10-09T08:56:01.450955+0000 mgr.y (mgr.25421) 553 : cluster 0 pgmap v522: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:02.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:02.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:02 smithi094 bash[59677]: cluster 2023-10-09T08:56:01.450955+0000 mgr.y (mgr.25421) 553 : cluster 0 pgmap v522: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:02.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:02.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:02.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:02.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:02.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:02.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:03.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:03.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:03.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:03.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:03.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:03.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:03.233 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T08:56:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:03.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:03.235 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:56:03.277 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T08:56:03.277 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T08:56:03.323 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-10-09T08:56:03.323 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T08:56:03.324 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_128518 2023-10-09T08:56:03.324 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:56:03.324 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-10-09T08:56:03.336 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:03.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:03.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:03.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:03.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:03.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:03.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:03.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:03.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:03.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:03.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:03.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:03.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:03.878 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:03.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:03.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:03.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:04.009 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:04.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:04.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:04.137 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:04.162 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:04.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T08:56:04.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:04.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:04.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2312 2296 143.431 143.5 0.0677786 0.00653108 2023-10-09T08:56:04.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:04.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:04.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:04.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:04.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:04.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:04.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:04.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:04.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:04.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:04.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:04.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:04.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:04.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:05.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:05.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:05.144 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T08:56:05.145 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:56:05.184 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:05.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:05.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:05.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:05 smithi016 bash[75179]: cluster 2023-10-09T08:56:03.451567+0000 mgr.y (mgr.25421) 554 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:05.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:05.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 4820 4804 150.079 156.75 0.00482547 0.006644 2023-10-09T08:56:05.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:05 smithi094 bash[59677]: cluster 2023-10-09T08:56:03.451567+0000 mgr.y (mgr.25421) 554 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:05.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:05 smithi094 bash[59677]: audit 2023-10-09T08:56:04.220845+0000 mon.a (mon.0) 754 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:05.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:05 smithi094 bash[59677]: audit 2023-10-09T08:56:04.225635+0000 mon.a (mon.0) 755 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:05.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:05.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:05.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:05.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:05.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:05 smithi016 bash[75179]: audit 2023-10-09T08:56:04.220845+0000 mon.a (mon.0) 754 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:05.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:05 smithi016 bash[75179]: audit 2023-10-09T08:56:04.225635+0000 mon.a (mon.0) 755 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:05.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:05 smithi016 bash[77892]: cluster 2023-10-09T08:56:03.451567+0000 mgr.y (mgr.25421) 554 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T08:56:05.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:05 smithi016 bash[77892]: audit 2023-10-09T08:56:04.220845+0000 mon.a (mon.0) 754 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:05.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:05 smithi016 bash[77892]: audit 2023-10-09T08:56:04.225635+0000 mon.a (mon.0) 755 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:05.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:05.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:05.702 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:05.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:05.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:05.833 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:05.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:05.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:05.993 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:05.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:56:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:56:05] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-09T08:56:06.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:06.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:06.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:06.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:06.229 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:06.255 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:06.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:06.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:06.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7241 7225 150.485 151.312 0.00695683 0.00663127 2023-10-09T08:56:06.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:06.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:06.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:06.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:06.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:06.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:06.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:06.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:06.629 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:06.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:06.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:06.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:06.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:06.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:06.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:06.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:06.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:06.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:06.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:07.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:07.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:07.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:07.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:07.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:07.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:07.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T08:56:07.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:07.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:07.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9488 9472 147.97 140.438 0.00259802 0.00674866 2023-10-09T08:56:07.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:07.455 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:07.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:07 smithi094 bash[59677]: cluster 2023-10-09T08:56:05.452342+0000 mgr.y (mgr.25421) 555 : cluster 0 pgmap v524: 89 pgs: 89 active+clean; 76 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 7.5 MiB/s wr, 120 op/s 2023-10-09T08:56:07.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:07 smithi016 bash[75179]: cluster 2023-10-09T08:56:05.452342+0000 mgr.y (mgr.25421) 555 : cluster 0 pgmap v524: 89 pgs: 89 active+clean; 76 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 7.5 MiB/s wr, 120 op/s 2023-10-09T08:56:07.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:07 smithi016 bash[77892]: cluster 2023-10-09T08:56:05.452342+0000 mgr.y (mgr.25421) 555 : cluster 0 pgmap v524: 89 pgs: 89 active+clean; 76 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 7.5 MiB/s wr, 120 op/s 2023-10-09T08:56:07.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:07.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:07.692 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:07.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:07.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:07.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:07.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:07.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:07.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:07.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:08.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:08.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:08.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:08.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:08.225 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:08.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:08.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 11662 11646 145.549 135.875 0.00268103 0.00686273 2023-10-09T08:56:08.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:08.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:08.482 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.506 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:08.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:08.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:08.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:08.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:08.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:08.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:08.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:08.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:08.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:08.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:09.020 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:09.046 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:09.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:09.076 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:09.096 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:56:09.130 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:56:09.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:09.165 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:56:09.175 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:09.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:09.206 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:56:09.213 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:56:09.213 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T08:56:09.213 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:56:09.228 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:09 smithi016 bash[75179]: cluster 2023-10-09T08:56:07.453418+0000 mgr.y (mgr.25421) 556 : cluster 0 pgmap v525: 89 pgs: 89 active+clean; 367 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 37 MiB/s wr, 586 op/s 2023-10-09T08:56:09.228 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:09 smithi016 bash[75179]: audit 2023-10-09T08:56:09.033241+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.16:0/3275005618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:09.228 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:09 smithi016 bash[77892]: cluster 2023-10-09T08:56:07.453418+0000 mgr.y (mgr.25421) 556 : cluster 0 pgmap v525: 89 pgs: 89 active+clean; 367 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 37 MiB/s wr, 586 op/s 2023-10-09T08:56:09.228 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:09 smithi016 bash[77892]: audit 2023-10-09T08:56:09.033241+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.16:0/3275005618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:09.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:09 smithi094 bash[59677]: cluster 2023-10-09T08:56:07.453418+0000 mgr.y (mgr.25421) 556 : cluster 0 pgmap v525: 89 pgs: 89 active+clean; 367 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 37 MiB/s wr, 586 op/s 2023-10-09T08:56:09.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:09 smithi094 bash[59677]: audit 2023-10-09T08:56:09.033241+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.16:0/3275005618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:09.251 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:56:09.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:09.297 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:56:09.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:09.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 13477 13461 140.196 113.438 0.0132558 0.00712174 2023-10-09T08:56:09.339 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:56:09.378 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.384 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:56:09.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:09.437 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:09.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:09.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:09.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:09.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:09.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:09.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:09.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:09.812 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:09.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:09.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:09.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:09.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:10.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:10.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:10.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:10.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:10.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:10.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:10.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:10.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 15347 15331 136.863 116.875 0.010436 0.00729608 2023-10-09T08:56:10.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:10.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:10.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:10.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:10.493 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:10.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:10.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:10.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:10.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:10.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:10.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:10.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:10.858 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:10.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:10.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:10.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:11.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:11.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:11.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:11.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:11.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:11.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:11.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:11.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:11.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 16874 16858 131.684 95.4375 0.00987713 0.00758641 2023-10-09T08:56:11.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:11.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:11.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:11.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:11.653 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:11.675 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:11 smithi094 bash[59677]: cluster 2023-10-09T08:56:09.453960+0000 mgr.y (mgr.25421) 557 : cluster 0 pgmap v526: 89 pgs: 89 active+clean; 487 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s 2023-10-09T08:56:11.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:11 smithi016 bash[75179]: cluster 2023-10-09T08:56:09.453960+0000 mgr.y (mgr.25421) 557 : cluster 0 pgmap v526: 89 pgs: 89 active+clean; 487 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s 2023-10-09T08:56:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:11 smithi016 bash[77892]: cluster 2023-10-09T08:56:09.453960+0000 mgr.y (mgr.25421) 557 : cluster 0 pgmap v526: 89 pgs: 89 active+clean; 487 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s 2023-10-09T08:56:11.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:11.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:11.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:11.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:11.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:11.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:12.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:12.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:12.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:12.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:12.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:12.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:12.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:12.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:12.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 18563 18547 128.78 105.562 0.00273268 0.0077533 2023-10-09T08:56:12.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:12.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:12.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:12.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:12.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:12.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:12.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:12.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:12.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:12.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:12.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:12.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:12.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:12.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:12.860 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:12.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:12.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:12.958 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:56:12.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:13.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:13.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:13.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:13.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:13.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:13.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:13.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:13.325 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 20370 20354 127.195 112.938 0.0121639 0.00785285 2023-10-09T08:56:13.381 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:13.407 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:13 smithi016 bash[75179]: cluster 2023-10-09T08:56:11.454616+0000 mgr.y (mgr.25421) 558 : cluster 0 pgmap v527: 89 pgs: 89 active+clean; 767 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T08:56:13.408 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:13.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:13 smithi016 bash[77892]: cluster 2023-10-09T08:56:11.454616+0000 mgr.y (mgr.25421) 558 : cluster 0 pgmap v527: 89 pgs: 89 active+clean; 767 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T08:56:13.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:13.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:13.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:13.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:13.648 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:13 smithi094 bash[59677]: cluster 2023-10-09T08:56:11.454616+0000 mgr.y (mgr.25421) 558 : cluster 0 pgmap v527: 89 pgs: 89 active+clean; 767 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T08:56:13.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:13.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:13.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:13.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:13.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:13.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:13.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:13.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:13.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:14.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:14.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:14.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:14.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:14.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:14.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:14.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 22123 22107 125.591 109.562 0.00815017 0.00795233 2023-10-09T08:56:14.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:14.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:14.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:14.562 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:14.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:14.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:14.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:56:14.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:14.717 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:14.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:14.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:14.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:14.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:14.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:14.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:14.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:15.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:15.110 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:15.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:15.213 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:15.238 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:15.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:15.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 23706 23690 123.37 98.9375 0.00543408 0.00809916 2023-10-09T08:56:15.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:15.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:15.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:15.443 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:56:15.443 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:56:15.443 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T08:56:15.443 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.944017 2023-10-09T08:56:15.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0.944017431022 2023-10-09T08:56:15.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:15.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:15 smithi094 bash[59677]: cluster 2023-10-09T08:56:13.455307+0000 mgr.y (mgr.25421) 559 : cluster 0 pgmap v528: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T08:56:15.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:15.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:15.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:15.633 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:15 smithi016 bash[75179]: cluster 2023-10-09T08:56:13.455307+0000 mgr.y (mgr.25421) 559 : cluster 0 pgmap v528: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T08:56:15.633 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:15 smithi016 bash[77892]: cluster 2023-10-09T08:56:13.455307+0000 mgr.y (mgr.25421) 559 : cluster 0 pgmap v528: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T08:56:15.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:15.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:15.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:15.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:15.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:15.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:15.922 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:15.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:16.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:16.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.058 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:56:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:56:15] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-09T08:56:16.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:16.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:16.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:16.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:16.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:16.311 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:16.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:16.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 25172 25156 120.927 91.625 0.00574562 0.00823567 2023-10-09T08:56:16.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:16.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:16.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:16.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:16.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:16.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:16.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:16.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:16.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:16.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:16.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:16.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:17.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:17.111 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:17.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:17.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:17.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:17.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:17.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:17.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 26629 26613 118.793 91.0625 0.0165193 0.00841026 2023-10-09T08:56:17.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:17.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:17.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:17.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:17.416 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:17 smithi094 bash[59677]: cluster 2023-10-09T08:56:15.456021+0000 mgr.y (mgr.25421) 560 : cluster 0 pgmap v529: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T08:56:17.489 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:17.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:17 smithi016 bash[77892]: cluster 2023-10-09T08:56:15.456021+0000 mgr.y (mgr.25421) 560 : cluster 0 pgmap v529: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T08:56:17.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:17 smithi016 bash[75179]: cluster 2023-10-09T08:56:15.456021+0000 mgr.y (mgr.25421) 560 : cluster 0 pgmap v529: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T08:56:17.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:17.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:17.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:17.650 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:17.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:17.695 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:56:17.695 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T08:56:17.695 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T08:56:17.695 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T08:56:17.695 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:56:17.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:17.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:17.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:17.883 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:17.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:17.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:17.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:18.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:18.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:18.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:18.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:18.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:18.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:18.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:18.306 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:18.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 28121 28105 117.089 93.25 0.00600388 0.00853275 2023-10-09T08:56:18.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:18.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:18 smithi016 bash[75179]: audit 2023-10-09T08:56:17.527370+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.16:0/2500166205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:56:18.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:18 smithi016 bash[75179]: audit 2023-10-09T08:56:17.527878+0000 mon.a (mon.0) 757 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:56:18.418 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:18 smithi016 bash[77892]: audit 2023-10-09T08:56:17.527370+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.16:0/2500166205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:56:18.418 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:18 smithi016 bash[77892]: audit 2023-10-09T08:56:17.527878+0000 mon.a (mon.0) 757 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:56:18.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:18.438 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T08:56:18.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:18.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:18.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:18.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:18.690 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:18 smithi094 bash[59677]: audit 2023-10-09T08:56:17.527370+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.16:0/2500166205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:56:18.690 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:18 smithi094 bash[59677]: audit 2023-10-09T08:56:17.527878+0000 mon.a (mon.0) 757 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:56:18.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:18.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:18.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:18.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:18.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:18.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:18.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:18.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:19.067 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T08:56:19.069 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:19.101 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:19.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:19.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:19.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:19.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:19.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:19.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 29624 29608 115.642 93.9375 0.00766643 0.00863905 2023-10-09T08:56:19.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:19.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:19.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:19.469 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:19.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[75179]: cluster 2023-10-09T08:56:17.456776+0000 mgr.y (mgr.25421) 561 : cluster 0 pgmap v530: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T08:56:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[75179]: cluster 2023-10-09T08:56:18.414341+0000 mon.a (mon.0) 758 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[75179]: cluster 2023-10-09T08:56:18.425289+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T08:56:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[75179]: cluster 2023-10-09T08:56:18.426578+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T08:56:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[75179]: audit 2023-10-09T08:56:18.429234+0000 mon.a (mon.0) 759 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:56:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[75179]: cluster 2023-10-09T08:56:18.429401+0000 mon.a (mon.0) 760 : cluster 0 osdmap e138: 8 total, 8 up, 7 in 2023-10-09T08:56:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[75179]: audit 2023-10-09T08:56:19.214802+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[77892]: cluster 2023-10-09T08:56:17.456776+0000 mgr.y (mgr.25421) 561 : cluster 0 pgmap v530: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T08:56:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[77892]: cluster 2023-10-09T08:56:18.414341+0000 mon.a (mon.0) 758 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[77892]: cluster 2023-10-09T08:56:18.425289+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T08:56:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[77892]: cluster 2023-10-09T08:56:18.426578+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T08:56:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[77892]: audit 2023-10-09T08:56:18.429234+0000 mon.a (mon.0) 759 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:56:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[77892]: cluster 2023-10-09T08:56:18.429401+0000 mon.a (mon.0) 760 : cluster 0 osdmap e138: 8 total, 8 up, 7 in 2023-10-09T08:56:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:19 smithi016 bash[77892]: audit 2023-10-09T08:56:19.214802+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:19.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:19.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:19.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:19.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:19.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:19.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:19.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:19 smithi094 bash[59677]: cluster 2023-10-09T08:56:17.456776+0000 mgr.y (mgr.25421) 561 : cluster 0 pgmap v530: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T08:56:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:19 smithi094 bash[59677]: cluster 2023-10-09T08:56:18.414341+0000 mon.a (mon.0) 758 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:19 smithi094 bash[59677]: cluster 2023-10-09T08:56:18.425289+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T08:56:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:19 smithi094 bash[59677]: cluster 2023-10-09T08:56:18.426578+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T08:56:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:19 smithi094 bash[59677]: audit 2023-10-09T08:56:18.429234+0000 mon.a (mon.0) 759 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:56:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:19 smithi094 bash[59677]: cluster 2023-10-09T08:56:18.429401+0000 mon.a (mon.0) 760 : cluster 0 osdmap e138: 8 total, 8 up, 7 in 2023-10-09T08:56:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:19 smithi094 bash[59677]: audit 2023-10-09T08:56:19.214802+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:19.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:19.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:19.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:19.876 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:19.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:19.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:19.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:20.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:20.036 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:20.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:20.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:20.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:20.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:20.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:20.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:20.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:20.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 31144 31128 114.427 95 0.00608535 0.00873173 2023-10-09T08:56:20.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:20.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:20.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:20.541 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:20.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:20.673 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:20.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:20.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:20.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:20.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:20.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:20.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:20.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:20.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:21.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:21.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:21.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:21.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:21.182 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:21.182 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:21.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:21.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:21.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:21.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 32616 32600 113.181 92 0.00728676 0.0088282 2023-10-09T08:56:21.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:21.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:21.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:21.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:21.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:21.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:21 smithi094 bash[59677]: cluster 2023-10-09T08:56:19.457286+0000 mgr.y (mgr.25421) 562 : cluster 0 pgmap v532: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T08:56:21.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:21 smithi016 bash[75179]: cluster 2023-10-09T08:56:19.457286+0000 mgr.y (mgr.25421) 562 : cluster 0 pgmap v532: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T08:56:21.497 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:21 smithi016 bash[77892]: cluster 2023-10-09T08:56:19.457286+0000 mgr.y (mgr.25421) 562 : cluster 0 pgmap v532: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T08:56:21.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:21.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:21.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:21.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:21.648 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:21.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:21.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:21.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:21.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:21.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:21.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:21.914 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:21.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:21.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:22.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:22.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:22.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:22.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:22.285 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:22.312 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:22.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 33887 33871 111.404 79.4375 0.00625872 0.0089689 2023-10-09T08:56:22.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:22.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:22.546 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:22.572 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:22.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:22.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T08:56:22.703 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:22.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:22.806 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:22.830 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:22.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:22.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:22.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:22.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:23.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:23.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:23.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:23.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:23.209 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:23.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:23.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:23.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:23.326 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:56:23.326053+0000 min lat: 0.00139597 max lat: 0.219692 avg lat: 0.00907587 2023-10-09T08:56:23.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:56:23.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 35250 35234 110.093 85.1875 0.00740221 0.00907587 2023-10-09T08:56:23.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:23.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:23.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:23.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:23 smithi094 bash[59677]: cluster 2023-10-09T08:56:21.457965+0000 mgr.y (mgr.25421) 563 : cluster 0 pgmap v533: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T08:56:23.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:23.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:23.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:23.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:23.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:23.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:23.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:23 smithi016 bash[75179]: cluster 2023-10-09T08:56:21.457965+0000 mgr.y (mgr.25421) 563 : cluster 0 pgmap v533: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T08:56:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:23 smithi016 bash[77892]: cluster 2023-10-09T08:56:21.457965+0000 mgr.y (mgr.25421) 563 : cluster 0 pgmap v533: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T08:56:23.758 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:23.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:23.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:23.885 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T08:56:23.894 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:23.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:23.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:23.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:24.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:24.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:24.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:24.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:24.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:24.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:24.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 36682 36666 109.112 89.5 0.0151604 0.00915932 2023-10-09T08:56:24.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:24.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:24.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:24.549 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:24.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:24.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:24.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:24.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:24.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:24.918 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:24.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:24.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:25.047 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:25.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:25.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:25.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:25.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:25.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:25.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:25.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:25.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 38078 38062 108.118 87.25 0.00560005 0.00924242 2023-10-09T08:56:25.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:25.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:25.410 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:25 smithi016 bash[75179]: cluster 2023-10-09T08:56:23.458788+0000 mgr.y (mgr.25421) 564 : cluster 0 pgmap v534: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.54k op/s 2023-10-09T08:56:25.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:25.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:25.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:25.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:25.579 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:25 smithi094 bash[59677]: cluster 2023-10-09T08:56:23.458788+0000 mgr.y (mgr.25421) 564 : cluster 0 pgmap v534: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.54k op/s 2023-10-09T08:56:25.594 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:25.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:25.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:25.707 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:25 smithi016 bash[77892]: cluster 2023-10-09T08:56:23.458788+0000 mgr.y (mgr.25421) 564 : cluster 0 pgmap v534: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.54k op/s 2023-10-09T08:56:25.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:25.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:25.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:25.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:25.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:25.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:25.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:25.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:26.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:56:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:56:25] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T08:56:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:26.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:26.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:26.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:26.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:26.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:26.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 39307 39291 106.757 76.8125 0.00376843 0.00936083 2023-10-09T08:56:26.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:26.411 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:26.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:26 smithi016 bash[75179]: audit 2023-10-09T08:56:25.416141+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:26.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:26 smithi016 bash[75179]: audit 2023-10-09T08:56:25.893993+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:26.419 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:26 smithi016 bash[75179]: audit 2023-10-09T08:56:25.903325+0000 mon.a (mon.0) 764 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:26.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:26.541 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:26.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:26.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:26.654 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:26 smithi094 bash[59677]: audit 2023-10-09T08:56:25.416141+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:26.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:26 smithi094 bash[59677]: audit 2023-10-09T08:56:25.893993+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:26.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:26 smithi094 bash[59677]: audit 2023-10-09T08:56:25.903325+0000 mon.a (mon.0) 764 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:26.670 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:26.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:26 smithi016 bash[77892]: audit 2023-10-09T08:56:25.416141+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:26 smithi016 bash[77892]: audit 2023-10-09T08:56:25.893993+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:26 smithi016 bash[77892]: audit 2023-10-09T08:56:25.903325+0000 mon.a (mon.0) 764 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:26.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:26.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:26.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:26.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:26.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:26.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:27.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:27.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:27.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:27.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:27.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:27.308 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:27.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 40669 40653 105.855 85.125 0.00389685 0.00943618 2023-10-09T08:56:27.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:27.422 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:27 smithi016 bash[75179]: cluster 2023-10-09T08:56:25.461477+0000 mgr.y (mgr.25421) 565 : cluster 0 pgmap v535: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T08:56:27.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:27.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:27.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:27.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:27.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:27.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:27.688 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:27 smithi016 bash[77892]: cluster 2023-10-09T08:56:25.461477+0000 mgr.y (mgr.25421) 565 : cluster 0 pgmap v535: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T08:56:27.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:27.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:27.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:27 smithi094 bash[59677]: cluster 2023-10-09T08:56:25.461477+0000 mgr.y (mgr.25421) 565 : cluster 0 pgmap v535: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T08:56:27.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:27.837 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:27.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:27.969 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:27.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:28.073 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:28.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:28.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:28.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:28.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:28.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:28.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:28.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:28.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 41940 41924 104.798 79.4375 0.0108444 0.009534 2023-10-09T08:56:28.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:28.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:28.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:28.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:28.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:28.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:28.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:28.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:28.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:28.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:28.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:28.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:28.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:28.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:28.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:28.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:28.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:28.942 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:16759 6.5 6:ac876bf5:::benchmark_data_smithi016_105291_object16758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:50:52.641331+0000", 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 336.28823542399999, 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14528342899999999, 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.943 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 16759 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641331+0000", 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.944 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641331+0000", 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641330+0000", 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000108515 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.945 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641438+0000", 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5497000000000001e-05 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641454+0000", 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.807e-06 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641461+0000", 2023-10-09T08:56:28.946 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1499999999999998e-05 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641513+0000", 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013452900000000001 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641647+0000", 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019266300000000001 2023-10-09T08:56:28.947 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,6", 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.641840+0000", 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001979013 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.643819+0000", 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000112773 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.948 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.643932+0000", 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14238791100000001 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.786320+0000", 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000181977 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:28.949 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.786502+0000", 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00011318499999999999 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:52.786615+0000", 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14528342899999999 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.950 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:27326 6.5 6:ac6f3e07:::benchmark_data_smithi016_105291_object27325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:50:58.360910+0000", 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 330.56865706500002, 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14003676800000001, 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 27326 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.951 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.360910+0000", 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.360910+0000", 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.952 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.360909+0000", 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000103735 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.361012+0000", 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2500000000000001e-05 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.361025+0000", 2023-10-09T08:56:28.953 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.849e-06 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.361032+0000", 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.109e-05 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.361083+0000", 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000122031 2023-10-09T08:56:28.954 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.361205+0000", 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015779699999999999 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,6", 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.361363+0000", 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0014868010000000001 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.955 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.362849+0000", 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001484097 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364334+0000", 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.136504193 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.500838+0000", 2023-10-09T08:56:28.956 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4580999999999998e-05 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.500892+0000", 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4249999999999997e-05 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.500947+0000", 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14003676800000001 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.957 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:27333 6.5 6:ac29c39b:::benchmark_data_smithi016_105291_object27332:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:50:58.364697+0000", 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 330.56486961799999, 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.13670115499999999, 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.958 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 27333 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364697+0000", 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.959 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364697+0000", 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364696+0000", 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4030999999999999e-05 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364740+0000", 2023-10-09T08:56:28.960 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1691e-05 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364752+0000", 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3109999999999996e-06 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364757+0000", 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3114000000000002e-05 2023-10-09T08:56:28.961 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364790+0000", 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8311000000000003e-05 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.364868+0000", 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001433705 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,6", 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.366302+0000", 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001293757 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.367596+0000", 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00072797300000000003 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.368324+0000", 2023-10-09T08:56:28.963 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.13277154999999999 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.501095+0000", 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000167088 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.501262+0000", 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000136081 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.964 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.501398+0000", 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.13670115499999999 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:27342 6.8 6:1473de54:::benchmark_data_smithi016_105291_object27341:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:50:58.369299+0000", 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 330.56026746499998, 2023-10-09T08:56:28.965 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.137055171, 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 27342 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369299+0000", 2023-10-09T08:56:28.966 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369299+0000", 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369298+0000", 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.8461000000000003e-05 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.967 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369336+0000", 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1297e-05 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369348+0000", 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4920000000000001e-06 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.968 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369353+0000", 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6294000000000002e-05 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369390+0000", 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4641000000000005e-05 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369474+0000", 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00011157 2023-10-09T08:56:28.969 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.369586+0000", 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001267759 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.370853+0000", 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0043073419999999996 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.970 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.375161+0000", 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.13096587200000001 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.506127+0000", 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013183100000000001 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.506259+0000", 2023-10-09T08:56:28.971 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.6058999999999996e-05 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.506355+0000", 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.137055171 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:27346 6.5 6:a29bc092:::benchmark_data_smithi016_105291_object27345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.972 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:50:58.374387+0000", 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 330.55517985400002, 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.12923585100000001, 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 27346 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.973 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374387+0000", 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374387+0000", 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374385+0000", 2023-10-09T08:56:28.974 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6560000000000001e-05 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374432+0000", 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2288e-05 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374444+0000", 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5489999999999997e-06 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.975 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374450+0000", 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6208000000000003e-05 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374486+0000", 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.5063999999999996e-05 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.976 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374581+0000", 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000120674 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,6", 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.374702+0000", 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001096189 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.375798+0000", 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00064383500000000005 2023-10-09T08:56:28.977 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.376442+0000", 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.12688189 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.503324+0000", 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018494799999999999 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.978 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.503509+0000", 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000114221 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:50:58.503623+0000", 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.12923585100000001 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.979 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:31829 6.8 6:1567288e:::benchmark_data_smithi016_105291_object31828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:01.142508+0000", 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 327.78705854600003, 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.119098224, 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.980 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 31829 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.142508+0000", 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.142508+0000", 2023-10-09T08:56:28.981 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.142507+0000", 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.4563000000000001e-05 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.142542+0000", 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0142999999999999e-05 2023-10-09T08:56:28.982 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.142552+0000", 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.7610000000000007e-06 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.142560+0000", 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.0918999999999999e-05 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.983 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.142601+0000", 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001425253 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.144026+0000", 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000155608 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.144182+0000", 2023-10-09T08:56:28.984 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001655852 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.145837+0000", 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00031759600000000001 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146155+0000", 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.115285103 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.985 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.261440+0000", 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.5822999999999994e-05 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.261536+0000", 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0605000000000006e-05 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.986 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.261607+0000", 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.119098224 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:31833 6.5 6:a0945dff:::benchmark_data_smithi016_105291_object31832:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:01.146462+0000", 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 327.78310443599997, 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.116781998, 2023-10-09T08:56:28.987 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 31833 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146462+0000", 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.988 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146462+0000", 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146461+0000", 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.7448000000000003e-05 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.989 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146518+0000", 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3695e-05 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146532+0000", 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.4630000000000004e-06 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.990 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146539+0000", 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8060000000000006e-05 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146637+0000", 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000153905 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146791+0000", 2023-10-09T08:56:28.991 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000183575 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,6", 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146975+0000", 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0030144220000000001 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.149989+0000", 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017095399999999999 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.992 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.150160+0000", 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.112277844 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.262438+0000", 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00060223700000000004 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:28.993 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.263040+0000", 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000204045 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.263244+0000", 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.116781998 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.994 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:31838 6.8 6:12d021d0:::benchmark_data_smithi016_105291_object31837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:01.146921+0000", 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 327.78264632000003, 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.11874942199999999, 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 31838 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.995 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146921+0000", 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146921+0000", 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.996 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146919+0000", 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4806999999999999e-05 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146974+0000", 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1888e-05 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146986+0000", 2023-10-09T08:56:28.997 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.9039999999999992e-06 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.146994+0000", 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00052340500000000001 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.147517+0000", 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014735700000000001 2023-10-09T08:56:28.998 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.147665+0000", 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000182494 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.147847+0000", 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020028609999999999 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:28.999 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.149850+0000", 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013241500000000001 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.149982+0000", 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.115486134 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.000 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.265469+0000", 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000143511 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.265612+0000", 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.7907e-05 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:01.265670+0000", 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.11874942199999999 2023-10-09T08:56:29.001 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:33238 6.3 6:c9c198f8:::benchmark_data_smithi016_105291_object33237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:02.083752+0000", 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 326.845815141, 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.157091545, 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.002 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 33238 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.083752+0000", 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.003 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.083752+0000", 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.083750+0000", 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.9829e-05 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.083800+0000", 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.4570999999999999e-05 2023-10-09T08:56:29.004 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.083825+0000", 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1270000000000003e-06 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.083833+0000", 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000133208 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.005 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.083966+0000", 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000138796 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.084105+0000", 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000203542 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.084308+0000", 2023-10-09T08:56:29.006 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029831940000000002 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087292+0000", 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00039193000000000002 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087683+0000", 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15292575899999999 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.007 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.240609+0000", 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012877799999999999 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.240738+0000", 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000105276 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.008 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.240843+0000", 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.157091545 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:33242 6.8 6:1211d225:::benchmark_data_smithi016_105291_object33241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:02.087774+0000", 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 326.84179332600002, 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15288173199999999, 2023-10-09T08:56:29.009 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 33242 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087774+0000", 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.010 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087774+0000", 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087772+0000", 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5592999999999994e-05 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.011 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.012 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.012 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087838+0000", 2023-10-09T08:56:29.012 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2721000000000001e-05 2023-10-09T08:56:29.012 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.012 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.012 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.012 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087851+0000", 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9569999999999996e-06 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087858+0000", 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3826999999999998e-05 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.013 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.087911+0000", 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00023454699999999999 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.088146+0000", 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000217319 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:56:29.014 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.088363+0000", 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0016190969999999999 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.089982+0000", 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00040380599999999999 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.090386+0000", 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15010827400000001 2023-10-09T08:56:29.015 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.240494+0000", 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7514000000000003e-05 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.240582+0000", 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3300999999999995e-05 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.016 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.240655+0000", 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15288173199999999 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:33250 6.3 6:cacb0d34:::benchmark_data_smithi016_105291_object33249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:02.092853+0000", 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 326.836713476, 2023-10-09T08:56:29.017 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.148495135, 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 33250 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.092853+0000", 2023-10-09T08:56:29.018 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.092853+0000", 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.092851+0000", 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3427000000000001e-05 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.019 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.092915+0000", 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8763e-05 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.092934+0000", 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.7710000000000001e-06 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.020 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.092940+0000", 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001089097 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.094029+0000", 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000155954 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.094185+0000", 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012389599999999999 2023-10-09T08:56:29.021 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.094309+0000", 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001129637 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.095439+0000", 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0017653199999999999 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.022 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.097204+0000", 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.143876114 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.241080+0000", 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000147138 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.241228+0000", 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012102700000000001 2023-10-09T08:56:29.023 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.241349+0000", 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.148495135 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:33264 6.8 6:1156e47a:::benchmark_data_smithi016_105291_object33263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:29.024 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:02.243562+0000", 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 326.68600486700001, 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.111816833, 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 33264 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.025 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.243562+0000", 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.243562+0000", 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.243560+0000", 2023-10-09T08:56:29.026 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0075999999999999e-05 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.243620+0000", 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9009e-05 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.243639+0000", 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.9779999999999994e-06 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.027 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.243648+0000", 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020187040000000001 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.245667+0000", 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000194458 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.028 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.245861+0000", 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00030831000000000001 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.246170+0000", 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0050779709999999997 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.251248+0000", 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001064108 2023-10-09T08:56:29.029 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.252312+0000", 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.102903089 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.355215+0000", 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.0185000000000005e-05 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.030 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.355305+0000", 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3721999999999997e-05 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.355379+0000", 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.111816833 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.031 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:33266 6.3 6:c636fe8d:::benchmark_data_smithi016_105291_object33265:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:02.246326+0000", 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 326.683241291, 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.107072574, 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 33266 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.032 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.246326+0000", 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.246326+0000", 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.033 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.246324+0000", 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001555114 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247879+0000", 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7944e-05 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247897+0000", 2023-10-09T08:56:29.034 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1080000000000001e-05 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247908+0000", 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.3988999999999998e-05 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247992+0000", 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00020337099999999999 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.035 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.248195+0000", 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00021395400000000001 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.248409+0000", 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001566259 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.249976+0000", 2023-10-09T08:56:29.036 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0026544089999999999 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.252630+0000", 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.100564402 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.353194+0000", 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.1718999999999994e-05 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.037 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.353286+0000", 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000112058 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.353398+0000", 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.107072574 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.038 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:33267 6.8 6:18a491a1:::benchmark_data_smithi016_105291_object33266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:02.247943+0000", 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 326.68162339700001, 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.107765931, 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 33267 2023-10-09T08:56:29.039 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247943+0000", 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247943+0000", 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.040 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247942+0000", 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3047000000000003e-05 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247985+0000", 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3862e-05 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.041 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.247999+0000", 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7690000000000002e-06 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.248007+0000", 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0017129059999999999 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.249720+0000", 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.132e-05 2023-10-09T08:56:29.042 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.249812+0000", 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00024951599999999997 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.250061+0000", 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0014607789999999999 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.043 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.251522+0000", 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.104003391 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.355525+0000", 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9549999999999997e-05 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.355585+0000", 2023-10-09T08:56:29.044 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0568999999999995e-05 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.355656+0000", 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3925000000000003e-05 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.355709+0000", 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.107765931 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.045 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:33279 6.5 6:ac99918a:::benchmark_data_smithi016_105291_object33278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:51:02.254351+0000", 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 326.67521568500001, 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.11021563099999999, 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.046 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 33279 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254351+0000", 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.047 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254351+0000", 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254349+0000", 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.8356999999999998e-05 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254397+0000", 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8301e-05 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.048 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254416+0000", 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8460000000000008e-06 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254424+0000", 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3421000000000001e-05 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.049 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254497+0000", 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000122067 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254619+0000", 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000110312 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,6", 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.254729+0000", 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001496495 2023-10-09T08:56:29.050 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.256226+0000", 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00025206499999999999 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.256478+0000", 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.10793306699999999 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.051 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.364411+0000", 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000104424 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.364515+0000", 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1425000000000003e-05 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:51:02.364567+0000", 2023-10-09T08:56:29.052 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.11021563099999999 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.44690.0:112265 6.c e110/108 6:3410ccb9:::benchmark_data_smithi016_105291_object11138:head v 110'6904, mlcod=0'0)", 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:52:33.013236+0000", 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 235.91633050799999, 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.095153519000000006, 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.053 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013236+0000", 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013236+0000", 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.054 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013236+0000", 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.1280000000000002e-06 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013243+0000", 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.9189999999999994e-06 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013253+0000", 2023-10-09T08:56:29.055 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.392e-06 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013259+0000", 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2259000000000002e-05 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013311+0000", 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6420000000000001e-05 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.056 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.013338+0000", 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.095016323999999999 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.108354+0000", 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.5725000000000001e-05 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.057 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:33.108390+0000", 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.095153519000000006 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:133182 6.3 6:caf9e5df:::benchmark_data_smithi016_105291_object32055:head [delete] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:53:06.930301+0000", 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 201.99926597800001, 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.093536538000000002, 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.058 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 133182 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.930301+0000", 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.059 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.930301+0000", 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.930299+0000", 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.4139999999999997e-06 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.060 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.930307+0000", 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2737e-05 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.930319+0000", 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3810000000000001e-06 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.930327+0000", 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.060110809000000001 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.061 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.990438+0000", 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020776840000000002 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.992515+0000", 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013029400000000001 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T08:56:29.062 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:06.992646+0000", 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.012134582999999999 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:07.004780+0000", 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010521700000000001 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:07.004885+0000", 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.018766682 2023-10-09T08:56:29.063 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:07.023652+0000", 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013959700000000001 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:07.023792+0000", 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.5769000000000002e-05 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.064 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:07.023837+0000", 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.093536538000000002 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.44690.0:140434 6.1 e113/108 6:80296409:::benchmark_data_smithi016_105291_object39307:head v 113'8786, mlcod=0'0)", 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:53:18.600411+0000", 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 190.32915563700001, 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.10413757, 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.065 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600411+0000", 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600411+0000", 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T08:56:29.066 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600411+0000", 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3869999999999997e-06 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600418+0000", 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.9070000000000002e-06 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.067 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600428+0000", 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.242e-06 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600433+0000", 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00040923799999999997 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600843+0000", 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6665e-05 2023-10-09T08:56:29.068 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.600899+0000", 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.103613365 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.704513+0000", 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6225e-05 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.069 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.704549+0000", 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.10413757 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.44690.0:140435 6.1 e113/108 6:80f28076:::benchmark_data_smithi016_105291_object39308:head v 113'8787, mlcod=0'0)", 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:53:18.610347+0000", 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 190.31922034600001, 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.094224666999999998, 2023-10-09T08:56:29.070 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.610347+0000", 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.610347+0000", 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T08:56:29.071 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.610345+0000", 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.9310000000000003e-06 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.610355+0000", 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6053999999999999e-05 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.072 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.610371+0000", 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4239999999999998e-06 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.610378+0000", 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0032336399999999999 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.613611+0000", 2023-10-09T08:56:29.073 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6631e-05 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.613668+0000", 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.090884798000000003 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.704553+0000", 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8397999999999999e-05 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.074 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:18.704571+0000", 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.094224666999999998 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.44690.0:141482 6.1 e113/108 6:846b9dab:::benchmark_data_smithi016_105291_object40355:head v 113'8856, mlcod=0'0)", 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:53:20.365592+0000", 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 188.563974637, 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.097155385999999996, 2023-10-09T08:56:29.075 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.365592+0000", 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.365592+0000", 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T08:56:29.076 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.365591+0000", 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1790000000000001e-05 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.365603+0000", 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6932999999999999e-05 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.077 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.365620+0000", 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9469999999999997e-06 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.365627+0000", 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0061220290000000002 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.371749+0000", 2023-10-09T08:56:29.078 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6110000000000003e-05 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.371805+0000", 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.090920751999999994 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.462726+0000", 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1801e-05 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T08:56:29.079 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:53:20.462748+0000", 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.097155385999999996 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T08:56:29.080 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T08:56:29.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:29.106 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:29.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:29.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:29.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:29.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:29.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:29.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 43251 43235 103.919 81.9375 0.0115133 0.00961597 2023-10-09T08:56:29.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:29.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:29.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:29.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:29.428 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:29 smithi094 bash[59677]: cluster 2023-10-09T08:56:27.462133+0000 mgr.y (mgr.25421) 566 : cluster 0 pgmap v536: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T08:56:29.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:29.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:29.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:29 smithi016 bash[75179]: cluster 2023-10-09T08:56:27.462133+0000 mgr.y (mgr.25421) 566 : cluster 0 pgmap v536: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T08:56:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:29 smithi016 bash[77892]: cluster 2023-10-09T08:56:27.462133+0000 mgr.y (mgr.25421) 566 : cluster 0 pgmap v536: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T08:56:29.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:29.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:29.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:29.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:29.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:29.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:29.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:29.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:29.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:29.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:29.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T08:56:30.000 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:30.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:30.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:30.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:30.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:30.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:30.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:30.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 44480 44464 102.914 76.8125 0.00845276 0.0097099 2023-10-09T08:56:30.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:30.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:30.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:30.521 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:30.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:30.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:30.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:30.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:30.754 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:56:30.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:30.802 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:56:30.813 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:30.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:30.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:30.861 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:56:30.904 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:56:30.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:30.950 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:56:30.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:30.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:30.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:31.013 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:56:31.057 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:56:31.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:31.101 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:31.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:31.116 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:56:31.124 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.124 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.124 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.124 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.125 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.125 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.125 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.125 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:56:31.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.232 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:31.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:31.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 45444 45428 101.39 60.25 0.0152013 0.00985531 2023-10-09T08:56:31.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:31.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:31.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:31.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:31 smithi094 bash[59677]: cluster 2023-10-09T08:56:29.462670+0000 mgr.y (mgr.25421) 567 : cluster 0 pgmap v537: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T08:56:31.451 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:31 smithi094 bash[59677]: audit 2023-10-09T08:56:30.682561+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.16:0/3235974126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:31.451 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:31 smithi094 bash[59677]: audit 2023-10-09T08:56:31.159546+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.16:0/324585657' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]: dispatch 2023-10-09T08:56:31.451 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:31 smithi094 bash[59677]: audit 2023-10-09T08:56:31.161215+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]: dispatch 2023-10-09T08:56:31.458 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.5 primary-affinity to 0.944017 (8618672) 2023-10-09T08:56:31.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:31.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:31.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:31.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[75179]: cluster 2023-10-09T08:56:29.462670+0000 mgr.y (mgr.25421) 567 : cluster 0 pgmap v537: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T08:56:31.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[75179]: audit 2023-10-09T08:56:30.682561+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.16:0/3235974126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:31.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[75179]: audit 2023-10-09T08:56:31.159546+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.16:0/324585657' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]: dispatch 2023-10-09T08:56:31.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[75179]: audit 2023-10-09T08:56:31.161215+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]: dispatch 2023-10-09T08:56:31.648 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[77892]: cluster 2023-10-09T08:56:29.462670+0000 mgr.y (mgr.25421) 567 : cluster 0 pgmap v537: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T08:56:31.648 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[77892]: audit 2023-10-09T08:56:30.682561+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.16:0/3235974126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:31.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[77892]: audit 2023-10-09T08:56:31.159546+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.16:0/324585657' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]: dispatch 2023-10-09T08:56:31.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:31 smithi016 bash[77892]: audit 2023-10-09T08:56:31.161215+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]: dispatch 2023-10-09T08:56:31.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:31.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:31.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:31.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.799 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:31.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:31.902 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.935 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:31.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:31.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:32.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:32.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:32.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:32.164 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:32.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:32.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:32.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:32.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:32.327 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 45845 45829 98.7577 25.0625 0.00311761 0.00988416 2023-10-09T08:56:32.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:32.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:32.430 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:32.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:32.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:32.532 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: cluster 2023-10-09T08:56:31.437507+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: cluster 2023-10-09T08:56:31.437651+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: cluster 2023-10-09T08:56:31.440655+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: audit 2023-10-09T08:56:31.442129+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]': finished 2023-10-09T08:56:32.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: cluster 2023-10-09T08:56:31.442684+0000 mon.a (mon.0) 767 : cluster 0 osdmap e139: 8 total, 8 up, 7 in 2023-10-09T08:56:32.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: cluster 2023-10-09T08:56:31.463114+0000 mgr.y (mgr.25421) 568 : cluster 0 pgmap v539: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T08:56:32.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: audit 2023-10-09T08:56:32.143003+0000 mon.a (mon.0) 768 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:32.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[75179]: audit 2023-10-09T08:56:32.148218+0000 mon.a (mon.0) 769 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:32.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: cluster 2023-10-09T08:56:31.437507+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: cluster 2023-10-09T08:56:31.437651+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: cluster 2023-10-09T08:56:31.440655+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: audit 2023-10-09T08:56:31.442129+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]': finished 2023-10-09T08:56:32.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: cluster 2023-10-09T08:56:31.442684+0000 mon.a (mon.0) 767 : cluster 0 osdmap e139: 8 total, 8 up, 7 in 2023-10-09T08:56:32.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: cluster 2023-10-09T08:56:31.463114+0000 mgr.y (mgr.25421) 568 : cluster 0 pgmap v539: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T08:56:32.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: audit 2023-10-09T08:56:32.143003+0000 mon.a (mon.0) 768 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:32.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:32 smithi016 bash[77892]: audit 2023-10-09T08:56:32.148218+0000 mon.a (mon.0) 769 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:32.591 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:32.618 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:32.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:32.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: cluster 2023-10-09T08:56:31.437507+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: cluster 2023-10-09T08:56:31.437651+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: cluster 2023-10-09T08:56:31.440655+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T08:56:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: audit 2023-10-09T08:56:31.442129+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.944017431022}]': finished 2023-10-09T08:56:32.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: cluster 2023-10-09T08:56:31.442684+0000 mon.a (mon.0) 767 : cluster 0 osdmap e139: 8 total, 8 up, 7 in 2023-10-09T08:56:32.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: cluster 2023-10-09T08:56:31.463114+0000 mgr.y (mgr.25421) 568 : cluster 0 pgmap v539: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T08:56:32.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: audit 2023-10-09T08:56:32.143003+0000 mon.a (mon.0) 768 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:32.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:32 smithi094 bash[59677]: audit 2023-10-09T08:56:32.148218+0000 mon.a (mon.0) 769 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:32.754 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:32.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:32.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:32.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:32.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:32.916 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:32.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:33.019 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:33.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:33.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:33.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:33.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:33.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:33.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:33.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:33.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:33.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:33.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:33.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:33.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 46837 46821 97.5323 62 0.0212352 0.0102458 2023-10-09T08:56:33.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:33.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:33.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:33.461 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: cluster 2023-10-09T08:56:32.454785+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T08:56:33.461 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: cluster 2023-10-09T08:56:32.457259+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T08:56:33.461 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: cluster 2023-10-09T08:56:32.457633+0000 mon.a (mon.0) 770 : cluster 0 osdmap e140: 8 total, 8 up, 7 in 2023-10-09T08:56:33.461 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: audit 2023-10-09T08:56:32.815488+0000 mon.a (mon.0) 771 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: audit 2023-10-09T08:56:32.826214+0000 mon.a (mon.0) 772 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: audit 2023-10-09T08:56:33.405993+0000 mon.a (mon.0) 773 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:33.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: audit 2023-10-09T08:56:33.407308+0000 mon.a (mon.0) 774 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:33.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[75179]: audit 2023-10-09T08:56:33.415156+0000 mon.a (mon.0) 775 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.463 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: cluster 2023-10-09T08:56:32.454785+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T08:56:33.463 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: cluster 2023-10-09T08:56:32.457259+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T08:56:33.464 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: cluster 2023-10-09T08:56:32.457633+0000 mon.a (mon.0) 770 : cluster 0 osdmap e140: 8 total, 8 up, 7 in 2023-10-09T08:56:33.464 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: audit 2023-10-09T08:56:32.815488+0000 mon.a (mon.0) 771 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.464 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: audit 2023-10-09T08:56:32.826214+0000 mon.a (mon.0) 772 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.464 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: audit 2023-10-09T08:56:33.405993+0000 mon.a (mon.0) 773 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:33.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: audit 2023-10-09T08:56:33.407308+0000 mon.a (mon.0) 774 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:33.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:33 smithi016 bash[77892]: audit 2023-10-09T08:56:33.415156+0000 mon.a (mon.0) 775 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:33.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:33.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:33.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:33.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:33.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:33.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: cluster 2023-10-09T08:56:32.454785+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T08:56:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: cluster 2023-10-09T08:56:32.457259+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T08:56:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: cluster 2023-10-09T08:56:32.457633+0000 mon.a (mon.0) 770 : cluster 0 osdmap e140: 8 total, 8 up, 7 in 2023-10-09T08:56:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: audit 2023-10-09T08:56:32.815488+0000 mon.a (mon.0) 771 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: audit 2023-10-09T08:56:32.826214+0000 mon.a (mon.0) 772 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: audit 2023-10-09T08:56:33.405993+0000 mon.a (mon.0) 773 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: audit 2023-10-09T08:56:33.407308+0000 mon.a (mon.0) 774 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:33.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:33 smithi094 bash[59677]: audit 2023-10-09T08:56:33.415156+0000 mon.a (mon.0) 775 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:56:33.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:33.869 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:33.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:33.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:33.972 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:33.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:34.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:34.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:34.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:34.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:34.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:34.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:34.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:34.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 48038 48022 96.8072 75.0625 0.0325536 0.0103218 2023-10-09T08:56:34.362 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:34.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:34.472 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:34 smithi016 bash[75179]: cluster 2023-10-09T08:56:33.463586+0000 mgr.y (mgr.25421) 569 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T08:56:34.473 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:34 smithi016 bash[75179]: audit 2023-10-09T08:56:34.217281+0000 mon.a (mon.0) 776 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:34.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:34.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:34.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:34.632 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:34 smithi094 bash[59677]: cluster 2023-10-09T08:56:33.463586+0000 mgr.y (mgr.25421) 569 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T08:56:34.632 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:34 smithi094 bash[59677]: audit 2023-10-09T08:56:34.217281+0000 mon.a (mon.0) 776 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:34.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:34.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:34 smithi016 bash[77892]: cluster 2023-10-09T08:56:33.463586+0000 mgr.y (mgr.25421) 569 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T08:56:34.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:34 smithi016 bash[77892]: audit 2023-10-09T08:56:34.217281+0000 mon.a (mon.0) 776 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:34.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:34.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:34.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:34.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:34.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:34.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:34.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:35.019 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:35.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:35.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:35.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:35.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:35.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:35.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:35.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:35.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:35.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 49196 49180 96.0436 72.375 0.0150772 0.0104066 2023-10-09T08:56:35.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:35.437 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:35.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:35.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:35.684 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:35.712 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:35.815 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:35.817 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:56:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:56:35] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.33.4" 2023-10-09T08:56:35.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:35.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:56:35.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:35.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:35.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:35.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:36.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:36.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:36.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:36.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:36.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:36.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:36.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:36.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 50301 50285 95.2258 69.0625 0.0136302 0.0104942 2023-10-09T08:56:36.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:36.385 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:36.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:36.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:36.388 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:36.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:36.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:36.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:36.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:36.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:36.631 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:36 smithi094 bash[59677]: cluster 2023-10-09T08:56:35.464298+0000 mgr.y (mgr.25421) 570 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:36.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:36.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:36.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:36.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:36.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:36 smithi016 bash[75179]: cluster 2023-10-09T08:56:35.464298+0000 mgr.y (mgr.25421) 570 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:36.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:36 smithi016 bash[77892]: cluster 2023-10-09T08:56:35.464298+0000 mgr.y (mgr.25421) 570 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:36.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:36.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:36.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:36.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:36.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:37.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:37.040 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:37.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:37.171 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:37.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:37.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:37.300 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:37.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:37.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 51516 51500 94.6583 75.9375 0.00997631 0.010558 2023-10-09T08:56:37.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:37.430 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:37.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:37.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:37.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:37.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:37.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:37.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:37.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:37.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:37.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:37.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:37.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:37.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:37.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:38.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:38.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:38.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:38.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:38.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:38.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:38.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:38.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:38.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 52669 52653 94.0125 72.0625 0.0113005 0.0106306 2023-10-09T08:56:38.355 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:38.381 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:38.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:38.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:38.479 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:38 smithi016 bash[75179]: cluster 2023-10-09T08:56:37.465547+0000 mgr.y (mgr.25421) 571 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:38.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:38.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:38.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:38.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:38.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:38.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:38.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:38.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:38 smithi094 bash[59677]: cluster 2023-10-09T08:56:37.465547+0000 mgr.y (mgr.25421) 571 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:38.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:38.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:38 smithi016 bash[77892]: cluster 2023-10-09T08:56:37.465547+0000 mgr.y (mgr.25421) 571 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:38.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:38.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:38.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:38.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:38.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:38.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:38.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:38.918 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:56:38.918 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:56:38.918 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T08:56:38.918 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T08:56:38.919 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T08:56:38.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T08:56:39.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:39.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:39.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:39.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:39.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:39.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:39.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 53640 53624 93.0867 60.6875 0.0251792 0.0107359 2023-10-09T08:56:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:39.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:39.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:39.497 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T08:56:39.561 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:39.571 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:39 smithi094 bash[59677]: audit 2023-10-09T08:56:39.271852+0000 mon.a (mon.0) 777 : audit 1 from='client.? 172.21.15.16:0/1201766992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:56:39.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:39.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:39.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:39.717 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:39.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:39 smithi016 bash[75179]: audit 2023-10-09T08:56:39.271852+0000 mon.a (mon.0) 777 : audit 1 from='client.? 172.21.15.16:0/1201766992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:56:39.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:39 smithi016 bash[77892]: audit 2023-10-09T08:56:39.271852+0000 mon.a (mon.0) 777 : audit 1 from='client.? 172.21.15.16:0/1201766992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:56:39.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:39.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:39.954 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:39.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:39.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:40.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:40.117 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:40.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:40.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:40.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:40.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:40.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 54755 54739 92.4541 69.6875 0.00729779 0.0108105 2023-10-09T08:56:40.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:40.378 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:40.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:40.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:40.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[77892]: cluster 2023-10-09T08:56:39.466081+0000 mgr.y (mgr.25421) 572 : cluster 0 pgmap v544: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:40.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[77892]: cluster 2023-10-09T08:56:39.476571+0000 mon.a (mon.0) 778 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:40.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[77892]: cluster 2023-10-09T08:56:39.487611+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T08:56:40.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[77892]: cluster 2023-10-09T08:56:39.487710+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T08:56:40.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[77892]: audit 2023-10-09T08:56:39.490865+0000 mon.a (mon.0) 779 : audit 1 from='client.? 172.21.15.16:0/1201766992' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:56:40.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[77892]: cluster 2023-10-09T08:56:39.490970+0000 mon.a (mon.0) 780 : cluster 0 osdmap e141: 8 total, 8 up, 7 in 2023-10-09T08:56:40.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:40.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:40.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:40.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:40.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:40.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:40 smithi094 bash[59677]: cluster 2023-10-09T08:56:39.466081+0000 mgr.y (mgr.25421) 572 : cluster 0 pgmap v544: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:40 smithi094 bash[59677]: cluster 2023-10-09T08:56:39.476571+0000 mon.a (mon.0) 778 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:40 smithi094 bash[59677]: cluster 2023-10-09T08:56:39.487611+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T08:56:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:40 smithi094 bash[59677]: cluster 2023-10-09T08:56:39.487710+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T08:56:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:40 smithi094 bash[59677]: audit 2023-10-09T08:56:39.490865+0000 mon.a (mon.0) 779 : audit 1 from='client.? 172.21.15.16:0/1201766992' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:56:40.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:40 smithi094 bash[59677]: cluster 2023-10-09T08:56:39.490970+0000 mon.a (mon.0) 780 : cluster 0 osdmap e141: 8 total, 8 up, 7 in 2023-10-09T08:56:40.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:40.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:40.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:40.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:40.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[75179]: cluster 2023-10-09T08:56:39.466081+0000 mgr.y (mgr.25421) 572 : cluster 0 pgmap v544: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T08:56:40.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[75179]: cluster 2023-10-09T08:56:39.476571+0000 mon.a (mon.0) 778 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:56:40.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[75179]: cluster 2023-10-09T08:56:39.487611+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T08:56:40.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[75179]: cluster 2023-10-09T08:56:39.487710+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T08:56:40.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[75179]: audit 2023-10-09T08:56:39.490865+0000 mon.a (mon.0) 779 : audit 1 from='client.? 172.21.15.16:0/1201766992' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:56:40.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:40 smithi016 bash[75179]: cluster 2023-10-09T08:56:39.490970+0000 mon.a (mon.0) 780 : cluster 0 osdmap e141: 8 total, 8 up, 7 in 2023-10-09T08:56:40.879 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:40.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:40.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:41.007 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:41.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:41.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:41.166 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:41.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:41.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 55872 55856 91.8581 69.8125 0.0117233 0.0108797 2023-10-09T08:56:41.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:41.430 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:41.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:41.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:41.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:41.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:41.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:41.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:41.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:42.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:42.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:42.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:42.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:42.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:42.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:42.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:42.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 57041 57025 91.376 73.0625 0.0214688 0.0109373 2023-10-09T08:56:42.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:42.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:42.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:42.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:42.469 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:42.485 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:42 smithi016 bash[75179]: cluster 2023-10-09T08:56:41.466901+0000 mgr.y (mgr.25421) 573 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T08:56:42.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:42.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:42.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:42.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:42.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:42.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:42.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:42.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:42.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:42 smithi016 bash[77892]: cluster 2023-10-09T08:56:41.466901+0000 mgr.y (mgr.25421) 573 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T08:56:42.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:42 smithi094 bash[59677]: cluster 2023-10-09T08:56:41.466901+0000 mgr.y (mgr.25421) 573 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T08:56:42.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:42.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:42.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:42.896 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:42.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:42.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:42.943 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:43.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:43.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:43.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:43.089 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:56:43.089 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:56:43.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:43.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:43.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:43.311 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:43.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:43.328 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:56:43.328170+0000 min lat: 0.00139597 max lat: 1.0974 avg lat: 0.0110056 2023-10-09T08:56:43.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:56:43.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 57982 57966 90.5617 58.8125 0.00318392 0.0110056 2023-10-09T08:56:43.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:43.443 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:43.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:43.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:43.688 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:43.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:43.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:43.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:43.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:43.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:43.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:43.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:44.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:44.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:44.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:44.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:44.232 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:44.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:44.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 58923 58907 89.7872 58.8125 0.0159503 0.0111314 2023-10-09T08:56:44.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:44.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:44.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:44.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T08:56:44.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:44.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:44.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:44.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:44.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:44.677 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:44.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:44.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:44.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:44.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:44.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:44.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:44.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:44.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:44.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:45.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:45.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:45.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:45.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:45.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:45.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:45.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:45.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:45.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 59981 59965 89.2236 66.125 0.0248915 0.0112016 2023-10-09T08:56:45.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:45.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:45.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T08:56:45.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:45.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:45.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:45.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:45.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:45 smithi016 bash[75179]: cluster 2023-10-09T08:56:43.467528+0000 mgr.y (mgr.25421) 574 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T08:56:45.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:45 smithi016 bash[77892]: cluster 2023-10-09T08:56:43.467528+0000 mgr.y (mgr.25421) 574 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T08:56:45.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:45.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:45.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:45.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:45.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:45 smithi094 bash[59677]: cluster 2023-10-09T08:56:43.467528+0000 mgr.y (mgr.25421) 574 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T08:56:45.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:45.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:45.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:45.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:45.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:45.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:45.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:45.941 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:56:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:56:45] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.33.4" 2023-10-09T08:56:46.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:46.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:46.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:46.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:46.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:46.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:46.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:46.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:46.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 60915 60899 88.5061 58.375 0.00907701 0.0112911 2023-10-09T08:56:46.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:46.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:46.560 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:46.586 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:46.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:46.716 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:46.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:46.844 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:46.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:46.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:46.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:47.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:47.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:47.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:47.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:47.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:47.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:47.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 61931 61915 87.9377 63.5 0.0163509 0.0113649 2023-10-09T08:56:47.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:47.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:47.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:47.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:47 smithi016 bash[75179]: cluster 2023-10-09T08:56:45.470021+0000 mgr.y (mgr.25421) 575 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T08:56:47.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:47 smithi016 bash[77892]: cluster 2023-10-09T08:56:45.470021+0000 mgr.y (mgr.25421) 575 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T08:56:47.506 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:47.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:47.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:47.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:47.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:47.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:47 smithi094 bash[59677]: cluster 2023-10-09T08:56:45.470021+0000 mgr.y (mgr.25421) 575 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T08:56:47.785 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:47.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:47.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:47.900 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:47.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:47.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:47.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:47.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:48.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:48.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:48.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:48.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:48.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:48.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:48.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:48.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:48.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:48.329 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 62992 62976 87.4559 66.3125 0.0258243 0.0114283 2023-10-09T08:56:48.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:48.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:48.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:48.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:48.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:48.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:48.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:48.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:48.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:48.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:48.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:48.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:48.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:48.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:48.877 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:48.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:48.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:49.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:49.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:49.144 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:49.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:49.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:49.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:49.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 63941 63925 86.844 59.3125 0.010651 0.0115091 2023-10-09T08:56:49.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:49.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:49.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:49.551 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:49 smithi016 bash[75179]: cluster 2023-10-09T08:56:47.470786+0000 mgr.y (mgr.25421) 576 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T08:56:49.551 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:49 smithi016 bash[75179]: audit 2023-10-09T08:56:49.215866+0000 mon.a (mon.0) 781 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:49.552 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:49 smithi016 bash[77892]: cluster 2023-10-09T08:56:47.470786+0000 mgr.y (mgr.25421) 576 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T08:56:49.552 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:49 smithi016 bash[77892]: audit 2023-10-09T08:56:49.215866+0000 mon.a (mon.0) 781 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:49.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:49.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:49.687 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:49 smithi094 bash[59677]: cluster 2023-10-09T08:56:47.470786+0000 mgr.y (mgr.25421) 576 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T08:56:49.687 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:49 smithi094 bash[59677]: audit 2023-10-09T08:56:49.215866+0000 mon.a (mon.0) 781 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:56:49.702 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:49.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:49.808 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:49.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:49.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:49.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:49.944 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:49.945 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:56:49.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:49.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:49.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:50.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:50.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:50.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:50.117 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:56:50.163 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:56:50.202 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:56:50.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:50.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:50.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:50.261 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:56:50.301 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:56:50.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 64805 64789 86.1451 54 0.0133214 0.011602 2023-10-09T08:56:50.340 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:50.342 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:56:50.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:50.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:50.381 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:56:50.418 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:56:50.421 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.421 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.422 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.422 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.422 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.422 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.422 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.422 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:56:50.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:50.507 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:50.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:50.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:50.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:50.622 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:50 smithi094 bash[59677]: audit 2023-10-09T08:56:50.071823+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.16:0/1962257586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:50.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:50.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:50.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:50.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:50.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:50 smithi016 bash[75179]: audit 2023-10-09T08:56:50.071823+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.16:0/1962257586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:50.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:50 smithi016 bash[77892]: audit 2023-10-09T08:56:50.071823+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.16:0/1962257586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:56:50.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:50.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:50.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:50.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:50.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:50.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:51.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:51.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:51.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:51.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:51.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:51.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:51.301 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:51.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:51.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 65657 65641 85.4596 53.25 0.003722 0.0116689 2023-10-09T08:56:51.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:51.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:51.547 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:51.577 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:51.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:51.686 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:51.700 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:51 smithi094 bash[59677]: cluster 2023-10-09T08:56:49.670802+0000 mgr.y (mgr.25421) 577 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T08:56:51.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:51.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:51.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:51 smithi016 bash[75179]: cluster 2023-10-09T08:56:49.670802+0000 mgr.y (mgr.25421) 577 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T08:56:51.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:51 smithi016 bash[77892]: cluster 2023-10-09T08:56:49.670802+0000 mgr.y (mgr.25421) 577 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T08:56:51.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:51.854 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:51.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:51.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:51.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:52.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:52.117 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:52.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:52.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:52.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:52.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:52.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:52.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 66620 66604 84.9438 60.1875 0.009704 0.0117663 2023-10-09T08:56:52.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:52.380 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:52.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:52.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:52.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:52.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:52.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:52.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:52.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:52.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:52.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:52.780 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:52.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:52.883 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:52.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:52.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:53.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:53.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:53.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:53.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:53.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:53.290 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:53.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:53.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:53.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 67439 67423 84.2686 51.1875 0.00848353 0.0118612 2023-10-09T08:56:53.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:53.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:53.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:53.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:53 smithi094 bash[59677]: cluster 2023-10-09T08:56:51.671849+0000 mgr.y (mgr.25421) 578 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T08:56:53.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:53.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:53.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:53 smithi016 bash[75179]: cluster 2023-10-09T08:56:51.671849+0000 mgr.y (mgr.25421) 578 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T08:56:53.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:53 smithi016 bash[77892]: cluster 2023-10-09T08:56:51.671849+0000 mgr.y (mgr.25421) 578 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T08:56:53.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:53.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:53.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:53.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:53.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:53.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:53.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:54.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:54.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:54.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:54.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:54.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:54.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:54.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:54.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 68382 68366 83.7718 58.9375 0.0109752 0.0119311 2023-10-09T08:56:54.362 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:54.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:54.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:54.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:54.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:54.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:54.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:54.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:54.673 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:54.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:54.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:54.812 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:54.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:54.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:54.938 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:54.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:54.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:55.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:55.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:55.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:55.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:55.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:55.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:55.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:55.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 69355 69339 83.3301 60.8125 0.00532758 0.0119945 2023-10-09T08:56:55.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:55.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:55 smithi016 bash[75179]: cluster 2023-10-09T08:56:53.672854+0000 mgr.y (mgr.25421) 579 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T08:56:55.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:55 smithi016 bash[77892]: cluster 2023-10-09T08:56:53.672854+0000 mgr.y (mgr.25421) 579 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T08:56:55.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:55.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:55.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:55.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:55.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:55.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:56:55.693 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:55.701 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:55 smithi094 bash[59677]: cluster 2023-10-09T08:56:53.672854+0000 mgr.y (mgr.25421) 579 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T08:56:55.716 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:55.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:55.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:55.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:55.956 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:56:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:56:55] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-09T08:56:55.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:55.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:55.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:55.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:56.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:56.124 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:56.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:56.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:56.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:56.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:56.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 70185 70169 82.7366 51.875 0.0174603 0.01208 2023-10-09T08:56:56.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:56.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:56.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:56.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:56.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:56.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:56.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:56.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:56.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:56.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:56.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:56.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:56.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:56.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:56.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:56.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:56.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:56.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:57.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:57.041 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:57.146 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:57.168 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:57.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:57.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:57.301 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:57.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:57.318 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:56:57.318 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T08:56:57.318 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:56:57.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 71030 71014 82.1823 52.8125 0.0259777 0.0121615 2023-10-09T08:56:57.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:57.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:57.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:57.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:57.547 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:57 smithi016 bash[75179]: cluster 2023-10-09T08:56:55.673908+0000 mgr.y (mgr.25421) 580 : cluster 0 pgmap v553: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T08:56:57.548 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:57 smithi016 bash[77892]: cluster 2023-10-09T08:56:55.673908+0000 mgr.y (mgr.25421) 580 : cluster 0 pgmap v553: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T08:56:57.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:57.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:57.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:57.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:57 smithi094 bash[59677]: cluster 2023-10-09T08:56:55.673908+0000 mgr.y (mgr.25421) 580 : cluster 0 pgmap v553: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T08:56:57.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:57.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:57.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:57.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:57.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:57.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:57.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:58.061 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:58.088 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:56:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:58.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:56:58.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:58.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:58.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:58.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:58.330 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 71956 71940 81.7403 57.875 0.0152566 0.0122276 2023-10-09T08:56:58.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:58.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:58.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:58.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:58.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:58.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:58.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:58.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:56:58.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:58.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:56:58.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:58.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:56:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:58.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:56:58.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:58.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:58.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:58.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:59.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:59.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:56:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:59.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:56:59.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:59.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:59.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:59.286 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:59.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:59.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:59.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 72906 72890 81.3408 59.375 0.0143047 0.012288 2023-10-09T08:56:59.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:59.419 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:56:59 smithi016 bash[75179]: cluster 2023-10-09T08:56:57.674541+0000 mgr.y (mgr.25421) 581 : cluster 0 pgmap v554: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T08:56:59.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:56:59.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:59.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:56:59.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:59.570 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:56:59 smithi094 bash[59677]: cluster 2023-10-09T08:56:57.674541+0000 mgr.y (mgr.25421) 581 : cluster 0 pgmap v554: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T08:56:59.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:56:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:59.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:56:59.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:59.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:56:59 smithi016 bash[77892]: cluster 2023-10-09T08:56:57.674541+0000 mgr.y (mgr.25421) 581 : cluster 0 pgmap v554: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T08:56:59.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:56:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:59.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:56:59.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:59.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:56:59.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:56:59.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:56:59.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:00.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:00.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:00.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:00.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:00.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:00.242 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:00.273 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:00.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 73778 73762 80.8698 54.5 0.0125552 0.0123598 2023-10-09T08:57:00.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:00.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:00.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:00.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:00.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:00.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:00.671 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:00.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:00.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:00.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:00.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:00.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:00.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:00.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:01.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:01.067 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:01.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:01.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:01.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:01.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:01.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 74717 74701 80.4873 58.6875 0.00965948 0.0124189 2023-10-09T08:57:01.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:01.425 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:01 smithi016 bash[75179]: cluster 2023-10-09T08:56:59.674936+0000 mgr.y (mgr.25421) 582 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T08:57:01.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:01.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.574 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:01 smithi094 bash[59677]: cluster 2023-10-09T08:56:59.674936+0000 mgr.y (mgr.25421) 582 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T08:57:01.589 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:01.692 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:01.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:01 smithi016 bash[77892]: cluster 2023-10-09T08:56:59.674936+0000 mgr.y (mgr.25421) 582 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T08:57:01.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:01.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:01.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:01.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:01.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:02.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:02.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:02.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:02.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:02.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:02.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:02.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 75665 75649 80.1273 59.25 0.0150972 0.0124734 2023-10-09T08:57:02.340 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:02.373 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:02.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:02.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:02.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:02.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:02.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:02.610 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:02.634 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:02.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:02.740 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:02.770 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:02.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:02.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:02.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:02.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:02.957 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:57:02.957 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T08:57:02.957 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T08:57:02.958 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T08:57:02.958 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:57:03.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:03.045 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:03.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:03.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:03.173 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:03.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:03.308 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:03.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:57:03.330695+0000 min lat: 0.00139597 max lat: 1.0974 avg lat: 0.0125492 2023-10-09T08:57:03.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:57:03.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 76482 76466 79.6428 51.0625 0.0192412 0.0125492 2023-10-09T08:57:03.414 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:03.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:03.456 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T08:57:03.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:03 smithi016 bash[75179]: cluster 2023-10-09T08:57:01.675617+0000 mgr.y (mgr.25421) 583 : cluster 0 pgmap v556: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-09T08:57:03.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:03 smithi016 bash[75179]: audit 2023-10-09T08:57:03.400151+0000 mon.a (mon.0) 782 : audit 1 from='client.? 172.21.15.16:0/3356982497' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:03.496 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:03 smithi016 bash[77892]: cluster 2023-10-09T08:57:01.675617+0000 mgr.y (mgr.25421) 583 : cluster 0 pgmap v556: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-09T08:57:03.496 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:03 smithi016 bash[77892]: audit 2023-10-09T08:57:03.400151+0000 mon.a (mon.0) 782 : audit 1 from='client.? 172.21.15.16:0/3356982497' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:03.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:03.553 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:03 smithi094 bash[59677]: cluster 2023-10-09T08:57:01.675617+0000 mgr.y (mgr.25421) 583 : cluster 0 pgmap v556: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-09T08:57:03.554 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:03 smithi094 bash[59677]: audit 2023-10-09T08:57:03.400151+0000 mon.a (mon.0) 782 : audit 1 from='client.? 172.21.15.16:0/3356982497' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:03.571 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:03.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:03.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:03.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:03.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:03.828 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:03.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:03.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:03.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:04.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:04.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:04.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:04.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:04.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:04.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:04.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:04.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:04.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 77331 77315 79.2069 53.0625 0.0113271 0.01262 2023-10-09T08:57:04.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:04.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:04.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: cluster 2023-10-09T08:57:03.424863+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: cluster 2023-10-09T08:57:03.424920+0000 mon.a (mon.0) 784 : cluster 1 Cluster is now healthy 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: cluster 2023-10-09T08:57:03.433691+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: audit 2023-10-09T08:57:03.437518+0000 mon.a (mon.0) 785 : audit 1 from='client.? 172.21.15.16:0/3356982497' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: cluster 2023-10-09T08:57:03.437596+0000 mon.a (mon.0) 786 : cluster 0 osdmap e142: 8 total, 8 up, 7 in 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: cluster 2023-10-09T08:57:03.438824+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: cluster 2023-10-09T08:57:03.443413+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: audit 2023-10-09T08:57:04.220235+0000 mon.a (mon.0) 787 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:04.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[75179]: audit 2023-10-09T08:57:04.222129+0000 mon.a (mon.0) 788 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:04.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: cluster 2023-10-09T08:57:03.424863+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:57:04.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: cluster 2023-10-09T08:57:03.424920+0000 mon.a (mon.0) 784 : cluster 1 Cluster is now healthy 2023-10-09T08:57:04.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: cluster 2023-10-09T08:57:03.433691+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: audit 2023-10-09T08:57:03.437518+0000 mon.a (mon.0) 785 : audit 1 from='client.? 172.21.15.16:0/3356982497' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:57:04.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: cluster 2023-10-09T08:57:03.437596+0000 mon.a (mon.0) 786 : cluster 0 osdmap e142: 8 total, 8 up, 7 in 2023-10-09T08:57:04.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: cluster 2023-10-09T08:57:03.438824+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: cluster 2023-10-09T08:57:03.443413+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: audit 2023-10-09T08:57:04.220235+0000 mon.a (mon.0) 787 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:04.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:04 smithi016 bash[77892]: audit 2023-10-09T08:57:04.222129+0000 mon.a (mon.0) 788 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:04.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:04.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:04.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:04.591 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:04.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:04.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:04.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:04.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T08:57:04.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: cluster 2023-10-09T08:57:03.424863+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: cluster 2023-10-09T08:57:03.424920+0000 mon.a (mon.0) 784 : cluster 1 Cluster is now healthy 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: cluster 2023-10-09T08:57:03.433691+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: audit 2023-10-09T08:57:03.437518+0000 mon.a (mon.0) 785 : audit 1 from='client.? 172.21.15.16:0/3356982497' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: cluster 2023-10-09T08:57:03.437596+0000 mon.a (mon.0) 786 : cluster 0 osdmap e142: 8 total, 8 up, 7 in 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: cluster 2023-10-09T08:57:03.438824+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: cluster 2023-10-09T08:57:03.443413+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: audit 2023-10-09T08:57:04.220235+0000 mon.a (mon.0) 787 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:04.737 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:04 smithi094 bash[59677]: audit 2023-10-09T08:57:04.222129+0000 mon.a (mon.0) 788 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:04.752 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:04.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:04.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:04.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:04.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:04.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:04.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:05.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:05.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:05.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:05.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:05.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:05.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:05.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:05.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 78213 78197 78.8185 55.125 0.00702705 0.0126803 2023-10-09T08:57:05.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:05.412 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:05.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:05.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:05.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:05.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:05.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:05 smithi094 bash[59677]: cluster 2023-10-09T08:57:03.676802+0000 mgr.y (mgr.25421) 584 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 904 op/s 2023-10-09T08:57:05.677 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:05.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:05.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:05 smithi016 bash[75179]: cluster 2023-10-09T08:57:03.676802+0000 mgr.y (mgr.25421) 584 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 904 op/s 2023-10-09T08:57:05.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:05 smithi016 bash[77892]: cluster 2023-10-09T08:57:03.676802+0000 mgr.y (mgr.25421) 584 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 904 op/s 2023-10-09T08:57:05.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:05.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:05.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:05.936 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:05.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:57:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:57:05] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.33.4" 2023-10-09T08:57:06.001 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:06.039 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:06.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:06.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:06.151 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:57:06.152 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:57:06.152 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-10-09T08:57:06.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T08:57:06.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:06.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:06.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:06.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:06.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:06.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:06.331 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 79097 79081 78.4443 55.25 0.0392935 0.0127411 2023-10-09T08:57:06.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:06.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:06.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:06.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:06.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:06.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:06.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:06.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:06.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:06.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:06.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:06.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:06.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:06.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:06.972 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:07.004 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:07.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:07.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:07.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:07.242 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:07.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:07.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:07.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 79980 79964 78.0808 55.1875 0.0117895 0.012801 2023-10-09T08:57:07.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:07.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:07.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:07.504 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:07.515 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:07 smithi094 bash[59677]: cluster 2023-10-09T08:57:05.677400+0000 mgr.y (mgr.25421) 585 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T08:57:07.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:07.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T08:57:07.635 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:07.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:07 smithi016 bash[75179]: cluster 2023-10-09T08:57:05.677400+0000 mgr.y (mgr.25421) 585 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T08:57:07.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:07 smithi016 bash[77892]: cluster 2023-10-09T08:57:05.677400+0000 mgr.y (mgr.25421) 585 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T08:57:07.661 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:07.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:07.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:07.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:07.933 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:07.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:08.039 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:08.065 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:08.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:08.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:08.194 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:08.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:08.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:08.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:08.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:08.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 80790 80774 77.6583 50.625 0.0250538 0.0128697 2023-10-09T08:57:08.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:08.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:08.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:08.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:08.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:08.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:08.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:08.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:08.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:08.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:08.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:08.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:08.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:08.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:08.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:09.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:09.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:09.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:09.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:09.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:09.288 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:09.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:09.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 81628 81612 77.2752 52.375 0.0148912 0.0129347 2023-10-09T08:57:09.392 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:09.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:09.447 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:09 smithi016 bash[75179]: cluster 2023-10-09T08:57:07.678064+0000 mgr.y (mgr.25421) 586 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T08:57:09.448 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:09 smithi016 bash[77892]: cluster 2023-10-09T08:57:07.678064+0000 mgr.y (mgr.25421) 586 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T08:57:09.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:09.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:09.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:09.666 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:09 smithi094 bash[59677]: cluster 2023-10-09T08:57:07.678064+0000 mgr.y (mgr.25421) 586 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T08:57:09.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:09.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:09.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:09.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:09.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:09.956 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:09.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:09.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:10.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:10.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:10.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:10.155 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:10.162 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:10.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:10.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:10.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:10.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 82453 82437 76.8913 51.5625 0.011357 0.0129988 2023-10-09T08:57:10.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:10.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:10.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:10.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:10.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:10.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:10.534 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:57:10.573 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:57:10.616 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:10.624 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:57:10.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:10.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:10.664 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:57:10.708 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:57:10.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:10.755 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:57:10.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:10.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:10.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:10.796 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:57:10.836 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:57:10.840 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.840 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.840 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.841 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.841 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.841 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.841 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.841 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:57:10.874 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:10.899 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:10.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:11.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:11.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:11.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:11.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.156 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:11.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:11.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:11.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 83246 83230 76.4893 49.5625 0.0134147 0.0130678 2023-10-09T08:57:11.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:11.426 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:11.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:11.539 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:11 smithi016 bash[75179]: cluster 2023-10-09T08:57:09.678545+0000 mgr.y (mgr.25421) 587 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T08:57:11.539 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:11 smithi016 bash[75179]: audit 2023-10-09T08:57:10.479075+0000 mon.a (mon.0) 789 : audit 0 from='client.? 172.21.15.16:0/1168846755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:11.540 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:11 smithi016 bash[77892]: cluster 2023-10-09T08:57:09.678545+0000 mgr.y (mgr.25421) 587 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T08:57:11.540 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:11 smithi016 bash[77892]: audit 2023-10-09T08:57:10.479075+0000 mon.a (mon.0) 789 : audit 0 from='client.? 172.21.15.16:0/1168846755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:11.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:11.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.687 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:11 smithi094 bash[59677]: cluster 2023-10-09T08:57:09.678545+0000 mgr.y (mgr.25421) 587 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T08:57:11.687 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:11 smithi094 bash[59677]: audit 2023-10-09T08:57:10.479075+0000 mon.a (mon.0) 789 : audit 0 from='client.? 172.21.15.16:0/1168846755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:11.711 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:11.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:11.946 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.971 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:11.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:11.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:12.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:12.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:12.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:12.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:12.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:12.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:12.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:12.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 84007 83991 76.07 47.5625 0.0134669 0.0131385 2023-10-09T08:57:12.357 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:12.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:12.460 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:12.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:12.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:12.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:12.602 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:12 smithi094 bash[59677]: cluster 2023-10-09T08:57:11.679038+0000 mgr.y (mgr.25421) 588 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T08:57:12.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:12.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:12.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:12.727 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:12.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:12 smithi016 bash[77892]: cluster 2023-10-09T08:57:11.679038+0000 mgr.y (mgr.25421) 588 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T08:57:12.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:12 smithi016 bash[75179]: cluster 2023-10-09T08:57:11.679038+0000 mgr.y (mgr.25421) 588 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T08:57:12.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:12.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:12.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:12.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:12.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:12.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:13.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:13.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:13.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:13.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:13.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:13.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:13.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:13.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:13.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 84923 84907 75.8011 57.25 0.0193689 0.0131863 2023-10-09T08:57:13.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:13.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:13.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:13.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:13.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:13.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:13.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:13.596 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:13.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:13.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:13.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:13.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:13.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:13.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:13.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:13.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:13.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:13.916 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:13.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:14.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:14.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:14.126 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:14.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:14.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:14.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:14.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:14.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 85710 85694 75.4262 49.1875 0.0151535 0.0132522 2023-10-09T08:57:14.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:14.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:14.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:14.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:14.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:14.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:14.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:14.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:14.802 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:14.826 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:14.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:14.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:14.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:14.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:14.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:15.068 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:15.094 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:15.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:15.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:15.197 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:15.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:15.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:15.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:15.332 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 86454 86438 75.0244 46.5 0.0197399 0.0133228 2023-10-09T08:57:15.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:15.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:15.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:15.420 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:15 smithi094 bash[59677]: cluster 2023-10-09T08:57:13.679634+0000 mgr.y (mgr.25421) 589 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T08:57:15.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:57:15.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:15.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:15.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:15.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:15.640 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:15 smithi016 bash[75179]: cluster 2023-10-09T08:57:13.679634+0000 mgr.y (mgr.25421) 589 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T08:57:15.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:15 smithi016 bash[77892]: cluster 2023-10-09T08:57:13.679634+0000 mgr.y (mgr.25421) 589 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T08:57:15.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:15.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:15.769 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:15.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:15.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:15.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:15.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:15.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:15.937 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:57:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:57:15] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.33.4" 2023-10-09T08:57:16.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:16.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:16.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:16.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:16.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:16.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:16.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:16.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:16.338 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 87212 87196 74.6455 47.375 0.0277633 0.0133906 2023-10-09T08:57:16.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:16.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:16.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:16.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:16.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:16.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:16.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:16.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:16.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:16.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:16.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:16.751 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:16.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:16.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:16.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:16.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:16.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:16.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:17.019 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:17.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:17.149 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:17.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:17.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:17.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:17.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 88026 88010 74.3243 50.875 0.0180613 0.0134481 2023-10-09T08:57:17.387 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:17.419 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:17.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:17.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:17.553 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:17.656 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:17.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:17 smithi094 bash[59677]: cluster 2023-10-09T08:57:15.683676+0000 mgr.y (mgr.25421) 590 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T08:57:17.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:17.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:17 smithi016 bash[75179]: cluster 2023-10-09T08:57:15.683676+0000 mgr.y (mgr.25421) 590 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T08:57:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:17 smithi016 bash[77892]: cluster 2023-10-09T08:57:15.683676+0000 mgr.y (mgr.25421) 590 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T08:57:17.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:17.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:17.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:17.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:17.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:17.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:17.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:18.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:18.107 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:18.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:18.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:18.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:18.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:18.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:18.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 88908 88892 74.0682 55.125 0.0195158 0.0134953 2023-10-09T08:57:18.359 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:18.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:18.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:18.500 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:18.529 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:18.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:18.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:18.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:18.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:18.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:18.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:18.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:18.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:18.896 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:18.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:18.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:19.028 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:19.055 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:19.158 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:19.189 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:19.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:19.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:19.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 89759 89743 73.7934 53.1875 0.0111829 0.013545 2023-10-09T08:57:19.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:19.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:19.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:19.579 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:19 smithi016 bash[75179]: cluster 2023-10-09T08:57:17.684315+0000 mgr.y (mgr.25421) 591 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-09T08:57:19.579 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:19 smithi016 bash[75179]: audit 2023-10-09T08:57:19.215998+0000 mon.a (mon.0) 790 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:19.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:19 smithi016 bash[77892]: cluster 2023-10-09T08:57:17.684315+0000 mgr.y (mgr.25421) 591 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-09T08:57:19.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:19 smithi016 bash[77892]: audit 2023-10-09T08:57:19.215998+0000 mon.a (mon.0) 790 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:19.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:19.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:19.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:19.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:19 smithi094 bash[59677]: cluster 2023-10-09T08:57:17.684315+0000 mgr.y (mgr.25421) 591 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-09T08:57:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:19 smithi094 bash[59677]: audit 2023-10-09T08:57:19.215998+0000 mon.a (mon.0) 790 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:19.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:19.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:19.970 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:19.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:19.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:19.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:20.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:20.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:20.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:20.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:20.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:20.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:20.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:20.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 90444 90428 73.391 42.8125 0.0107543 0.0136193 2023-10-09T08:57:20.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:20.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:20.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:20.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:20.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:20.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:20.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:20.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:20.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:20.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:20.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:20.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:20.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:20.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:20.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:21.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:21.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:21.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:21.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:21.165 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:21.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:21.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:21.301 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:21.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:21.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 91280 91264 73.1199 52.25 0.0182816 0.0136703 2023-10-09T08:57:21.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:21.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:21.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:21.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:21.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:21.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:21.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:21.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:21.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:21.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:21.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:21.723 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:21 smithi094 bash[59677]: cluster 2023-10-09T08:57:19.684743+0000 mgr.y (mgr.25421) 592 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-09T08:57:21.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:21.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:21 smithi016 bash[75179]: cluster 2023-10-09T08:57:19.684743+0000 mgr.y (mgr.25421) 592 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-09T08:57:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:21 smithi016 bash[77892]: cluster 2023-10-09T08:57:19.684743+0000 mgr.y (mgr.25421) 592 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-09T08:57:21.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:21.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:21.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:21.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:21.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:22.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:22.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:22.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:22.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:22.253 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:22.279 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:22.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:22.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 92110 92094 72.8509 51.875 0.00898102 0.0137208 2023-10-09T08:57:22.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:22.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:22.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:22.438 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:22.508 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:22.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:22.641 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:22.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:22.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:22.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:22.904 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:22.931 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:22.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:23.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:23.065 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:23.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:23.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:23.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:23.196 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:23.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:23.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:23.300 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:23.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:23.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:23.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T08:57:23.332707+0000 min lat: 0.00139597 max lat: 1.0974 avg lat: 0.0137685 2023-10-09T08:57:23.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T08:57:23.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 92949 92933 72.5957 52.4375 0.0306264 0.0137685 2023-10-09T08:57:23.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:23.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:23.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:23.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:23.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:23.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:23 smithi016 bash[75179]: cluster 2023-10-09T08:57:21.685380+0000 mgr.y (mgr.25421) 593 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-10-09T08:57:23.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:23 smithi016 bash[77892]: cluster 2023-10-09T08:57:21.685380+0000 mgr.y (mgr.25421) 593 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-10-09T08:57:23.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:23.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:23.710 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:23.718 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:23 smithi094 bash[59677]: cluster 2023-10-09T08:57:21.685380+0000 mgr.y (mgr.25421) 593 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-10-09T08:57:23.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:23.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:23.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:23.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:23.870 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:23.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:23.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:23.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:23.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:24.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:24.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:24.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:24.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:24.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 93794 93778 72.3514 52.8125 0.029323 0.0138155 2023-10-09T08:57:24.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:24.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:24.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:24.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:24.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:24.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:24.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:24.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:24.799 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:24.854 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:57:24.854 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:57:24.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:24.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:24.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:24.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:25.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:25.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:25.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:25.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:25.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:25.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:25.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:25.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:25.318 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:25.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 94455 94439 71.9729 41.3125 0.0296447 0.0138884 2023-10-09T08:57:25.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:25.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:25.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:25.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:25.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:25.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:25.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:25.557 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T08:57:25.568 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T08:57:25.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:25.609 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:25.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:25.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:25.721 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:25 smithi094 bash[59677]: cluster 2023-10-09T08:57:23.685964+0000 mgr.y (mgr.25421) 594 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T08:57:25.740 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:57:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:57:25] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-09T08:57:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:25 smithi016 bash[75179]: cluster 2023-10-09T08:57:23.685964+0000 mgr.y (mgr.25421) 594 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T08:57:25.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:25 smithi016 bash[77892]: cluster 2023-10-09T08:57:23.685964+0000 mgr.y (mgr.25421) 594 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T08:57:25.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:25.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:25.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:25.849 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:25.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:25.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:25.978 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:26.005 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:26.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:26.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:26.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:26.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:26.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:26.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:26.333 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 95256 95240 71.7088 50.0625 0.0127246 0.0139394 2023-10-09T08:57:26.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:26.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:26.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:26.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:26 smithi016 bash[75179]: audit 2023-10-09T08:57:25.534139+0000 mgr.y (mgr.25421) 595 : audit 0 from='client.35076 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:57:26.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:26 smithi016 bash[75179]: cluster 2023-10-09T08:57:25.687050+0000 mgr.y (mgr.25421) 596 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T08:57:26.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:26 smithi016 bash[77892]: audit 2023-10-09T08:57:25.534139+0000 mgr.y (mgr.25421) 595 : audit 0 from='client.35076 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:57:26.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:26 smithi016 bash[77892]: cluster 2023-10-09T08:57:25.687050+0000 mgr.y (mgr.25421) 596 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T08:57:26.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:26.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:26.640 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:26.650 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:26 smithi094 bash[59677]: audit 2023-10-09T08:57:25.534139+0000 mgr.y (mgr.25421) 595 : audit 0 from='client.35076 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:57:26.650 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:26 smithi094 bash[59677]: cluster 2023-10-09T08:57:25.687050+0000 mgr.y (mgr.25421) 596 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T08:57:26.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:26.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:26.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:26.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:26.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:26.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:26.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:27.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:27.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:27.165 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:27.196 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:27.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:27.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:27.335 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 96064 96048 71.4546 50.5 0.0196302 0.0139881 2023-10-09T08:57:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:27.437 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:27.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:27.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:27.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:27.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T08:57:27.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:27.731 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:27.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:27.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:27.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:27.972 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:27.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:27.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:27.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:28.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:28.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:28.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:28.237 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:28.261 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:28.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:28.335 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 96786 96770 71.1448 45.125 0.0151867 0.0140504 2023-10-09T08:57:28.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:28.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:28.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:28.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:28.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:28.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:28.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:28.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:28.606 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":568,"stamp":"2023-10-09T08:57:23.685578+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5995501472,"num_objects":91499,"num_object_clones":0,"num_object_copies":274497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91499,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":91735,"num_write_kb":5856845,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":726874,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91669,"ondisk_log_size":91669,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19676340,"kb_used_data":17983932,"kb_used_omap":126,"kb_used_meta":1692225,"kb_avail":636539724,"statfs":{"total":671965249536,"available":651816677376,"internally_reserved":0,"allocated":18415546368,"data_stored":18411704512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1732839035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":658898944,"num_objects":10054,"num_object_clones":0,"num_object_copies":30162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10054,"num_write_kb":643456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006867"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":125,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506082+0000","last_change":"2023-10-09T08:46:49.233483+0000","last_active":"2023-10-09T08:56:39.506082+0000","last_peered":"2023-10-09T08:56:39.506082+0000","last_clean":"2023-10-09T08:56:39.506082+0000","last_became_active":"2023-10-09T08:46:49.233033+0000","last_became_peered":"2023-10-09T08:46:49.233033+0000","last_unstale":"2023-10-09T08:56:39.506082+0000","last_undegraded":"2023-10-09T08:56:39.506082+0000","last_fullsized":"2023-10-09T08:56:39.506082+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":114,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:03.444649+0000","last_change":"2023-10-09T08:47:16.965051+0000","last_active":"2023-10-09T08:57:03.444649+0000","last_peered":"2023-10-09T08:57:03.444649+0000","last_clean":"2023-10-09T08:57:03.444649+0000","last_became_active":"2023-10-09T08:47:16.964664+0000","last_became_peered":"2023-10-09T08:47:16.964664+0000","last_unstale":"2023-10-09T08:57:03.444649+0000","last_undegraded":"2023-10-09T08:57:03.444649+0000","last_fullsized":"2023-10-09T08:57:03.444649+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.377202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":99,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.510699+0000","last_change":"2023-10-09T08:47:16.968800+0000","last_active":"2023-10-09T08:56:39.510699+0000","last_peered":"2023-10-09T08:56:39.510699+0000","last_clean":"2023-10-09T08:56:39.510699+0000","last_became_active":"2023-10-09T08:47:16.968430+0000","last_became_peered":"2023-10-09T08:47:16.968430+0000","last_unstale":"2023-10-09T08:56:39.510699+0000","last_undegraded":"2023-10-09T08:56:39.510699+0000","last_fullsized":"2023-10-09T08:56:39.510699+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:08.011498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508721+0000","last_change":"2023-10-09T08:54:16.096082+0000","last_active":"2023-10-09T08:56:39.508721+0000","last_peered":"2023-10-09T08:56:39.508721+0000","last_clean":"2023-10-09T08:56:39.508721+0000","last_became_active":"2023-10-09T08:54:16.095715+0000","last_became_peered":"2023-10-09T08:54:16.095715+0000","last_unstale":"2023-10-09T08:56:39.508721+0000","last_undegraded":"2023-10-09T08:56:39.508721+0000","last_fullsized":"2023-10-09T08:56:39.508721+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:40.399356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":130,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:55.450820+0000","last_change":"2023-10-09T08:47:16.965460+0000","last_active":"2023-10-09T08:56:55.450820+0000","last_peered":"2023-10-09T08:56:55.450820+0000","last_clean":"2023-10-09T08:56:55.450820+0000","last_became_active":"2023-10-09T08:47:16.965040+0000","last_became_peered":"2023-10-09T08:47:16.965040+0000","last_unstale":"2023-10-09T08:56:55.450820+0000","last_undegraded":"2023-10-09T08:56:55.450820+0000","last_fullsized":"2023-10-09T08:56:55.450820+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.025735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508806+0000","last_change":"2023-10-09T08:46:18.680365+0000","last_active":"2023-10-09T08:56:39.508806+0000","last_peered":"2023-10-09T08:56:39.508806+0000","last_clean":"2023-10-09T08:56:39.508806+0000","last_became_active":"2023-10-09T08:46:18.679875+0000","last_became_peered":"2023-10-09T08:46:18.679875+0000","last_unstale":"2023-10-09T08:56:39.508806+0000","last_undegraded":"2023-10-09T08:56:39.508806+0000","last_fullsized":"2023-10-09T08:56:39.508806+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":103,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.507303+0000","last_change":"2023-10-09T08:47:16.971965+0000","last_active":"2023-10-09T08:56:39.507303+0000","last_peered":"2023-10-09T08:56:39.507303+0000","last_clean":"2023-10-09T08:56:39.507303+0000","last_became_active":"2023-10-09T08:47:16.971651+0000","last_became_peered":"2023-10-09T08:47:16.971651+0000","last_unstale":"2023-10-09T08:56:39.507303+0000","last_undegraded":"2023-10-09T08:56:39.507303+0000","last_fullsized":"2023-10-09T08:56:39.507303+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:04.259603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.511224+0000","last_change":"2023-10-09T08:47:16.969948+0000","last_active":"2023-10-09T08:56:39.511224+0000","last_peered":"2023-10-09T08:56:39.511224+0000","last_clean":"2023-10-09T08:56:39.511224+0000","last_became_active":"2023-10-09T08:47:16.969666+0000","last_became_peered":"2023-10-09T08:47:16.969666+0000","last_unstale":"2023-10-09T08:56:39.511224+0000","last_undegraded":"2023-10-09T08:56:39.511224+0000","last_fullsized":"2023-10-09T08:56:39.511224+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:25.089633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.495479+0000","last_change":"2023-10-09T08:56:32.476917+0000","last_active":"2023-10-09T08:56:39.495479+0000","last_peered":"2023-10-09T08:56:39.495479+0000","last_clean":"2023-10-09T08:56:39.495479+0000","last_became_active":"2023-10-09T08:56:32.476030+0000","last_became_peered":"2023-10-09T08:56:32.476030+0000","last_unstale":"2023-10-09T08:56:39.495479+0000","last_undegraded":"2023-10-09T08:56:39.495479+0000","last_fullsized":"2023-10-09T08:56:39.495479+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.510795+0000","last_change":"2023-10-09T08:54:16.090726+0000","last_active":"2023-10-09T08:56:39.510795+0000","last_peered":"2023-10-09T08:56:39.510795+0000","last_clean":"2023-10-09T08:56:39.510795+0000","last_became_active":"2023-10-09T08:54:16.090390+0000","last_became_peered":"2023-10-09T08:54:16.090390+0000","last_unstale":"2023-10-09T08:56:39.510795+0000","last_undegraded":"2023-10-09T08:56:39.510795+0000","last_fullsized":"2023-10-09T08:56:39.510795+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:52.968772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":51,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:40.428620+0000","last_change":"2023-10-09T08:54:16.100639+0000","last_active":"2023-10-09T08:56:40.428620+0000","last_peered":"2023-10-09T08:56:40.428620+0000","last_clean":"2023-10-09T08:56:40.428620+0000","last_became_active":"2023-10-09T08:54:16.085401+0000","last_became_peered":"2023-10-09T08:54:16.085401+0000","last_unstale":"2023-10-09T08:56:40.428620+0000","last_undegraded":"2023-10-09T08:56:40.428620+0000","last_fullsized":"2023-10-09T08:56:40.428620+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.238266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":99,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.510941+0000","last_change":"2023-10-09T08:46:49.231880+0000","last_active":"2023-10-09T08:56:39.510941+0000","last_peered":"2023-10-09T08:56:39.510941+0000","last_clean":"2023-10-09T08:56:39.510941+0000","last_became_active":"2023-10-09T08:46:49.231471+0000","last_became_peered":"2023-10-09T08:46:49.231471+0000","last_unstale":"2023-10-09T08:56:39.510941+0000","last_undegraded":"2023-10-09T08:56:39.510941+0000","last_fullsized":"2023-10-09T08:56:39.510941+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:31.742745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":135,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506543+0000","last_change":"2023-10-09T08:54:16.167494+0000","last_active":"2023-10-09T08:56:39.506543+0000","last_peered":"2023-10-09T08:56:39.506543+0000","last_clean":"2023-10-09T08:56:39.506543+0000","last_became_active":"2023-10-09T08:54:16.098095+0000","last_became_peered":"2023-10-09T08:54:16.098095+0000","last_unstale":"2023-10-09T08:56:39.506543+0000","last_undegraded":"2023-10-09T08:56:39.506543+0000","last_fullsized":"2023-10-09T08:56:39.506543+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506960+0000","last_change":"2023-10-09T08:46:18.676902+0000","last_active":"2023-10-09T08:56:39.506960+0000","last_peered":"2023-10-09T08:56:39.506960+0000","last_clean":"2023-10-09T08:56:39.506960+0000","last_became_active":"2023-10-09T08:46:18.676560+0000","last_became_peered":"2023-10-09T08:46:18.676560+0000","last_unstale":"2023-10-09T08:56:39.506960+0000","last_undegraded":"2023-10-09T08:56:39.506960+0000","last_fullsized":"2023-10-09T08:56:39.506960+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":50,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506995+0000","last_change":"2023-10-09T08:54:16.254300+0000","last_active":"2023-10-09T08:56:39.506995+0000","last_peered":"2023-10-09T08:56:39.506995+0000","last_clean":"2023-10-09T08:56:39.506995+0000","last_became_active":"2023-10-09T08:54:16.098089+0000","last_became_peered":"2023-10-09T08:54:16.098089+0000","last_unstale":"2023-10-09T08:56:39.506995+0000","last_undegraded":"2023-10-09T08:56:39.506995+0000","last_fullsized":"2023-10-09T08:56:39.506995+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":123,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.510989+0000","last_change":"2023-10-09T08:54:16.087143+0000","last_active":"2023-10-09T08:56:39.510989+0000","last_peered":"2023-10-09T08:56:39.510989+0000","last_clean":"2023-10-09T08:56:39.510989+0000","last_became_active":"2023-10-09T08:54:16.086741+0000","last_became_peered":"2023-10-09T08:54:16.086741+0000","last_unstale":"2023-10-09T08:56:39.510989+0000","last_undegraded":"2023-10-09T08:56:39.510989+0000","last_fullsized":"2023-10-09T08:56:39.510989+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:53.402285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":121,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.504190+0000","last_change":"2023-10-09T08:54:16.088635+0000","last_active":"2023-10-09T08:56:39.504190+0000","last_peered":"2023-10-09T08:56:39.504190+0000","last_clean":"2023-10-09T08:56:39.504190+0000","last_became_active":"2023-10-09T08:54:16.088295+0000","last_became_peered":"2023-10-09T08:54:16.088295+0000","last_unstale":"2023-10-09T08:56:39.504190+0000","last_undegraded":"2023-10-09T08:56:39.504190+0000","last_fullsized":"2023-10-09T08:56:39.504190+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:44.023596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.495300+0000","last_change":"2023-10-09T08:56:32.471273+0000","last_active":"2023-10-09T08:56:39.495300+0000","last_peered":"2023-10-09T08:56:39.495300+0000","last_clean":"2023-10-09T08:56:39.495300+0000","last_became_active":"2023-10-09T08:56:32.470840+0000","last_became_peered":"2023-10-09T08:56:32.470840+0000","last_unstale":"2023-10-09T08:56:39.495300+0000","last_undegraded":"2023-10-09T08:56:39.495300+0000","last_fullsized":"2023-10-09T08:56:39.495300+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":33,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.509941+0000","last_change":"2023-10-09T08:54:16.092507+0000","last_active":"2023-10-09T08:56:39.509941+0000","last_peered":"2023-10-09T08:56:39.509941+0000","last_clean":"2023-10-09T08:56:39.509941+0000","last_became_active":"2023-10-09T08:54:16.092206+0000","last_became_peered":"2023-10-09T08:54:16.092206+0000","last_unstale":"2023-10-09T08:56:39.509941+0000","last_undegraded":"2023-10-09T08:56:39.509941+0000","last_fullsized":"2023-10-09T08:56:39.509941+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":112,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508892+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T08:56:39.508892+0000","last_peered":"2023-10-09T08:56:39.508892+0000","last_clean":"2023-10-09T08:56:39.508892+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T08:56:39.508892+0000","last_undegraded":"2023-10-09T08:56:39.508892+0000","last_fullsized":"2023-10-09T08:56:39.508892+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":155,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.507706+0000","last_change":"2023-10-09T08:51:53.462296+0000","last_active":"2023-10-09T08:56:39.507706+0000","last_peered":"2023-10-09T08:56:39.507706+0000","last_clean":"2023-10-09T08:56:39.507706+0000","last_became_active":"2023-10-09T08:51:53.454854+0000","last_became_peered":"2023-10-09T08:51:53.454854+0000","last_unstale":"2023-10-09T08:56:39.507706+0000","last_undegraded":"2023-10-09T08:56:39.507706+0000","last_fullsized":"2023-10-09T08:56:39.507706+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:42.210201+0000","last_change":"2023-10-09T08:54:16.094203+0000","last_active":"2023-10-09T08:56:42.210201+0000","last_peered":"2023-10-09T08:56:42.210201+0000","last_clean":"2023-10-09T08:56:42.210201+0000","last_became_active":"2023-10-09T08:54:16.090060+0000","last_became_peered":"2023-10-09T08:54:16.090060+0000","last_unstale":"2023-10-09T08:56:42.210201+0000","last_undegraded":"2023-10-09T08:56:42.210201+0000","last_fullsized":"2023-10-09T08:56:42.210201+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:17.599914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"142'5787","reported_seq":5804,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.266711+0000","last_change":"2023-10-09T08:55:45.516647+0000","last_active":"2023-10-09T08:57:22.266711+0000","last_peered":"2023-10-09T08:57:22.266711+0000","last_clean":"2023-10-09T08:57:22.266711+0000","last_became_active":"2023-10-09T08:55:45.516040+0000","last_became_peered":"2023-10-09T08:55:45.516040+0000","last_unstale":"2023-10-09T08:57:22.266711+0000","last_undegraded":"2023-10-09T08:57:22.266711+0000","last_fullsized":"2023-10-09T08:57:22.266711+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:37.447434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379256832,"num_objects":5787,"num_object_clones":0,"num_object_copies":17361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5787,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":48,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506715+0000","last_change":"2023-10-09T08:54:16.151974+0000","last_active":"2023-10-09T08:56:39.506715+0000","last_peered":"2023-10-09T08:56:39.506715+0000","last_clean":"2023-10-09T08:56:39.506715+0000","last_became_active":"2023-10-09T08:54:16.091029+0000","last_became_peered":"2023-10-09T08:54:16.091029+0000","last_unstale":"2023-10-09T08:56:39.506715+0000","last_undegraded":"2023-10-09T08:56:39.506715+0000","last_fullsized":"2023-10-09T08:56:39.506715+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.493881+0000","last_change":"2023-10-09T08:56:32.479320+0000","last_active":"2023-10-09T08:56:39.493881+0000","last_peered":"2023-10-09T08:56:39.493881+0000","last_clean":"2023-10-09T08:56:39.493881+0000","last_became_active":"2023-10-09T08:56:32.478751+0000","last_became_peered":"2023-10-09T08:56:32.478751+0000","last_unstale":"2023-10-09T08:56:39.493881+0000","last_undegraded":"2023-10-09T08:56:39.493881+0000","last_fullsized":"2023-10-09T08:56:39.493881+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"142'5727","reported_seq":5744,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:20.455286+0000","last_change":"2023-10-09T08:55:45.515343+0000","last_active":"2023-10-09T08:57:20.455286+0000","last_peered":"2023-10-09T08:57:20.455286+0000","last_clean":"2023-10-09T08:57:20.455286+0000","last_became_active":"2023-10-09T08:55:45.514812+0000","last_became_peered":"2023-10-09T08:55:45.514812+0000","last_unstale":"2023-10-09T08:57:20.455286+0000","last_undegraded":"2023-10-09T08:57:20.455286+0000","last_fullsized":"2023-10-09T08:57:20.455286+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:27.107329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375324672,"num_objects":5727,"num_object_clones":0,"num_object_copies":17181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5727,"num_write_kb":366528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":48,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506435+0000","last_change":"2023-10-09T08:54:16.158765+0000","last_active":"2023-10-09T08:56:39.506435+0000","last_peered":"2023-10-09T08:56:39.506435+0000","last_clean":"2023-10-09T08:56:39.506435+0000","last_became_active":"2023-10-09T08:54:16.096769+0000","last_became_peered":"2023-10-09T08:54:16.096769+0000","last_unstale":"2023-10-09T08:56:39.506435+0000","last_undegraded":"2023-10-09T08:56:39.506435+0000","last_fullsized":"2023-10-09T08:56:39.506435+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":141,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.509927+0000","last_change":"2023-10-09T08:51:33.439973+0000","last_active":"2023-10-09T08:56:39.509927+0000","last_peered":"2023-10-09T08:56:39.509927+0000","last_clean":"2023-10-09T08:56:39.509927+0000","last_became_active":"2023-10-09T08:51:33.439461+0000","last_became_peered":"2023-10-09T08:51:33.439461+0000","last_unstale":"2023-10-09T08:56:39.509927+0000","last_undegraded":"2023-10-09T08:56:39.509927+0000","last_fullsized":"2023-10-09T08:56:39.509927+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"142'5611","reported_seq":5628,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.546432+0000","last_change":"2023-10-09T08:55:45.482995+0000","last_active":"2023-10-09T08:57:22.546432+0000","last_peered":"2023-10-09T08:57:22.546432+0000","last_clean":"2023-10-09T08:57:22.546432+0000","last_became_active":"2023-10-09T08:55:45.482663+0000","last_became_peered":"2023-10-09T08:55:45.482663+0000","last_unstale":"2023-10-09T08:57:22.546432+0000","last_undegraded":"2023-10-09T08:57:22.546432+0000","last_fullsized":"2023-10-09T08:57:22.546432+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:20.365601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367722496,"num_objects":5611,"num_object_clones":0,"num_object_copies":16833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5611,"num_write_kb":359104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":25,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.495203+0000","last_change":"2023-10-09T08:56:32.473052+0000","last_active":"2023-10-09T08:56:39.495203+0000","last_peered":"2023-10-09T08:56:39.495203+0000","last_clean":"2023-10-09T08:56:39.495203+0000","last_became_active":"2023-10-09T08:56:32.472493+0000","last_became_peered":"2023-10-09T08:56:32.472493+0000","last_unstale":"2023-10-09T08:56:39.495203+0000","last_undegraded":"2023-10-09T08:56:39.495203+0000","last_fullsized":"2023-10-09T08:56:39.495203+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":126,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508098+0000","last_change":"2023-10-09T08:54:16.092901+0000","last_active":"2023-10-09T08:56:39.508098+0000","last_peered":"2023-10-09T08:56:39.508098+0000","last_clean":"2023-10-09T08:56:39.508098+0000","last_became_active":"2023-10-09T08:54:16.090513+0000","last_became_peered":"2023-10-09T08:54:16.090513+0000","last_unstale":"2023-10-09T08:56:39.508098+0000","last_undegraded":"2023-10-09T08:56:39.508098+0000","last_fullsized":"2023-10-09T08:56:39.508098+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"142'5794","reported_seq":5811,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.623993+0000","last_change":"2023-10-09T08:55:45.515192+0000","last_active":"2023-10-09T08:57:22.623993+0000","last_peered":"2023-10-09T08:57:22.623993+0000","last_clean":"2023-10-09T08:57:22.623993+0000","last_became_active":"2023-10-09T08:55:45.514868+0000","last_became_peered":"2023-10-09T08:55:45.514868+0000","last_unstale":"2023-10-09T08:57:22.623993+0000","last_undegraded":"2023-10-09T08:57:22.623993+0000","last_fullsized":"2023-10-09T08:57:22.623993+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5794,"log_dups_size":0,"ondisk_log_size":5794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:49.374940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379715584,"num_objects":5794,"num_object_clones":0,"num_object_copies":17382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5794,"num_write_kb":370816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.495202+0000","last_change":"2023-10-09T08:56:32.471256+0000","last_active":"2023-10-09T08:56:39.495202+0000","last_peered":"2023-10-09T08:56:39.495202+0000","last_clean":"2023-10-09T08:56:39.495202+0000","last_became_active":"2023-10-09T08:56:32.470904+0000","last_became_peered":"2023-10-09T08:56:32.470904+0000","last_unstale":"2023-10-09T08:56:39.495202+0000","last_undegraded":"2023-10-09T08:56:39.495202+0000","last_fullsized":"2023-10-09T08:56:39.495202+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":104,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:57:02.498455+0000","last_change":"2023-10-09T08:46:18.683057+0000","last_active":"2023-10-09T08:57:02.498455+0000","last_peered":"2023-10-09T08:57:02.498455+0000","last_clean":"2023-10-09T08:57:02.498455+0000","last_became_active":"2023-10-09T08:46:18.682672+0000","last_became_peered":"2023-10-09T08:46:18.682672+0000","last_unstale":"2023-10-09T08:57:02.498455+0000","last_undegraded":"2023-10-09T08:57:02.498455+0000","last_fullsized":"2023-10-09T08:57:02.498455+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:42.975534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"142'5746","reported_seq":5763,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.499024+0000","last_change":"2023-10-09T08:55:45.482309+0000","last_active":"2023-10-09T08:57:22.499024+0000","last_peered":"2023-10-09T08:57:22.499024+0000","last_clean":"2023-10-09T08:57:22.499024+0000","last_became_active":"2023-10-09T08:55:45.481954+0000","last_became_peered":"2023-10-09T08:55:45.481954+0000","last_unstale":"2023-10-09T08:57:22.499024+0000","last_undegraded":"2023-10-09T08:57:22.499024+0000","last_fullsized":"2023-10-09T08:57:22.499024+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:47.249071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376569856,"num_objects":5746,"num_object_clones":0,"num_object_copies":17238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5746,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":113,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.511412+0000","last_change":"2023-10-09T08:47:16.965045+0000","last_active":"2023-10-09T08:56:39.511412+0000","last_peered":"2023-10-09T08:56:39.511412+0000","last_clean":"2023-10-09T08:56:39.511412+0000","last_became_active":"2023-10-09T08:47:16.964658+0000","last_became_peered":"2023-10-09T08:47:16.964658+0000","last_unstale":"2023-10-09T08:56:39.511412+0000","last_undegraded":"2023-10-09T08:56:39.511412+0000","last_fullsized":"2023-10-09T08:56:39.511412+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:49.126847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.507523+0000","last_change":"2023-10-09T08:47:16.972109+0000","last_active":"2023-10-09T08:56:39.507523+0000","last_peered":"2023-10-09T08:56:39.507523+0000","last_clean":"2023-10-09T08:56:39.507523+0000","last_became_active":"2023-10-09T08:47:16.971716+0000","last_became_peered":"2023-10-09T08:47:16.971716+0000","last_unstale":"2023-10-09T08:56:39.507523+0000","last_undegraded":"2023-10-09T08:56:39.507523+0000","last_fullsized":"2023-10-09T08:56:39.507523+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"142'5756","reported_seq":5773,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.607577+0000","last_change":"2023-10-09T08:55:45.517336+0000","last_active":"2023-10-09T08:57:22.607577+0000","last_peered":"2023-10-09T08:57:22.607577+0000","last_clean":"2023-10-09T08:57:22.607577+0000","last_became_active":"2023-10-09T08:55:45.516782+0000","last_became_peered":"2023-10-09T08:55:45.516782+0000","last_unstale":"2023-10-09T08:57:22.607577+0000","last_undegraded":"2023-10-09T08:57:22.607577+0000","last_fullsized":"2023-10-09T08:57:22.607577+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:06:24.700012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377225216,"num_objects":5756,"num_object_clones":0,"num_object_copies":17268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5756,"num_write_kb":368384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":109,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.507933+0000","last_change":"2023-10-09T08:46:18.681222+0000","last_active":"2023-10-09T08:56:39.507933+0000","last_peered":"2023-10-09T08:56:39.507933+0000","last_clean":"2023-10-09T08:56:39.507933+0000","last_became_active":"2023-10-09T08:46:18.680710+0000","last_became_peered":"2023-10-09T08:46:18.680710+0000","last_unstale":"2023-10-09T08:56:39.507933+0000","last_undegraded":"2023-10-09T08:56:39.507933+0000","last_fullsized":"2023-10-09T08:56:39.507933+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":123,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506872+0000","last_change":"2023-10-09T08:46:49.234714+0000","last_active":"2023-10-09T08:56:39.506872+0000","last_peered":"2023-10-09T08:56:39.506872+0000","last_clean":"2023-10-09T08:56:39.506872+0000","last_became_active":"2023-10-09T08:46:49.234307+0000","last_became_peered":"2023-10-09T08:46:49.234307+0000","last_unstale":"2023-10-09T08:56:39.506872+0000","last_undegraded":"2023-10-09T08:56:39.506872+0000","last_fullsized":"2023-10-09T08:56:39.506872+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"142'5651","reported_seq":5667,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:20.713731+0000","last_change":"2023-10-09T08:55:45.486075+0000","last_active":"2023-10-09T08:57:20.713731+0000","last_peered":"2023-10-09T08:57:20.713731+0000","last_clean":"2023-10-09T08:57:20.713731+0000","last_became_active":"2023-10-09T08:55:45.485477+0000","last_became_peered":"2023-10-09T08:55:45.485477+0000","last_unstale":"2023-10-09T08:57:20.713731+0000","last_undegraded":"2023-10-09T08:57:20.713731+0000","last_fullsized":"2023-10-09T08:57:20.713731+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5651,"log_dups_size":0,"ondisk_log_size":5651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370343936,"num_objects":5651,"num_object_clones":0,"num_object_copies":16953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5651,"num_write_kb":361664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.509733+0000","last_change":"2023-10-09T08:54:16.087335+0000","last_active":"2023-10-09T08:56:39.509733+0000","last_peered":"2023-10-09T08:56:39.509733+0000","last_clean":"2023-10-09T08:56:39.509733+0000","last_became_active":"2023-10-09T08:54:16.086195+0000","last_became_peered":"2023-10-09T08:54:16.086195+0000","last_unstale":"2023-10-09T08:56:39.509733+0000","last_undegraded":"2023-10-09T08:56:39.509733+0000","last_fullsized":"2023-10-09T08:56:39.509733+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506399+0000","last_change":"2023-10-09T08:46:18.676843+0000","last_active":"2023-10-09T08:56:39.506399+0000","last_peered":"2023-10-09T08:56:39.506399+0000","last_clean":"2023-10-09T08:56:39.506399+0000","last_became_active":"2023-10-09T08:46:18.676468+0000","last_became_peered":"2023-10-09T08:46:18.676468+0000","last_unstale":"2023-10-09T08:56:39.506399+0000","last_undegraded":"2023-10-09T08:56:39.506399+0000","last_fullsized":"2023-10-09T08:56:39.506399+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"142'5576","reported_seq":5592,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:20.752329+0000","last_change":"2023-10-09T08:55:45.486129+0000","last_active":"2023-10-09T08:57:20.752329+0000","last_peered":"2023-10-09T08:57:20.752329+0000","last_clean":"2023-10-09T08:57:20.752329+0000","last_became_active":"2023-10-09T08:55:45.485666+0000","last_became_peered":"2023-10-09T08:55:45.485666+0000","last_unstale":"2023-10-09T08:57:20.752329+0000","last_undegraded":"2023-10-09T08:57:20.752329+0000","last_fullsized":"2023-10-09T08:57:20.752329+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365428736,"num_objects":5576,"num_object_clones":0,"num_object_copies":16728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5576,"num_write_kb":356864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508393+0000","last_change":"2023-10-09T08:54:16.088414+0000","last_active":"2023-10-09T08:56:39.508393+0000","last_peered":"2023-10-09T08:56:39.508393+0000","last_clean":"2023-10-09T08:56:39.508393+0000","last_became_active":"2023-10-09T08:54:16.088020+0000","last_became_peered":"2023-10-09T08:54:16.088020+0000","last_unstale":"2023-10-09T08:56:39.508393+0000","last_undegraded":"2023-10-09T08:56:39.508393+0000","last_fullsized":"2023-10-09T08:56:39.508393+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:42.977336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":99,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.511627+0000","last_change":"2023-10-09T08:47:16.965479+0000","last_active":"2023-10-09T08:56:39.511627+0000","last_peered":"2023-10-09T08:56:39.511627+0000","last_clean":"2023-10-09T08:56:39.511627+0000","last_became_active":"2023-10-09T08:47:16.965141+0000","last_became_peered":"2023-10-09T08:47:16.965141+0000","last_unstale":"2023-10-09T08:56:39.511627+0000","last_undegraded":"2023-10-09T08:56:39.511627+0000","last_fullsized":"2023-10-09T08:56:39.511627+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:50.145649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"142'5574","reported_seq":5590,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:19.864263+0000","last_change":"2023-10-09T08:55:45.485829+0000","last_active":"2023-10-09T08:57:19.864263+0000","last_peered":"2023-10-09T08:57:19.864263+0000","last_clean":"2023-10-09T08:57:19.864263+0000","last_became_active":"2023-10-09T08:55:45.485440+0000","last_became_peered":"2023-10-09T08:55:45.485440+0000","last_unstale":"2023-10-09T08:57:19.864263+0000","last_undegraded":"2023-10-09T08:57:19.864263+0000","last_fullsized":"2023-10-09T08:57:19.864263+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5574,"log_dups_size":0,"ondisk_log_size":5574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365297664,"num_objects":5574,"num_object_clones":0,"num_object_copies":16722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5574,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":134,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:42.210142+0000","last_change":"2023-10-09T08:54:16.095002+0000","last_active":"2023-10-09T08:56:42.210142+0000","last_peered":"2023-10-09T08:56:42.210142+0000","last_clean":"2023-10-09T08:56:42.210142+0000","last_became_active":"2023-10-09T08:54:16.093450+0000","last_became_peered":"2023-10-09T08:54:16.093450+0000","last_unstale":"2023-10-09T08:56:42.210142+0000","last_undegraded":"2023-10-09T08:56:42.210142+0000","last_fullsized":"2023-10-09T08:56:42.210142+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:19.952788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.495399+0000","last_change":"2023-10-09T08:56:32.472787+0000","last_active":"2023-10-09T08:56:39.495399+0000","last_peered":"2023-10-09T08:56:39.495399+0000","last_clean":"2023-10-09T08:56:39.495399+0000","last_became_active":"2023-10-09T08:56:32.472379+0000","last_became_peered":"2023-10-09T08:56:32.472379+0000","last_unstale":"2023-10-09T08:56:39.495399+0000","last_undegraded":"2023-10-09T08:56:39.495399+0000","last_fullsized":"2023-10-09T08:56:39.495399+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:42.204465+0000","last_change":"2023-10-09T08:46:18.684501+0000","last_active":"2023-10-09T08:56:42.204465+0000","last_peered":"2023-10-09T08:56:42.204465+0000","last_clean":"2023-10-09T08:56:42.204465+0000","last_became_active":"2023-10-09T08:46:18.684159+0000","last_became_peered":"2023-10-09T08:46:18.684159+0000","last_unstale":"2023-10-09T08:56:42.204465+0000","last_undegraded":"2023-10-09T08:56:42.204465+0000","last_fullsized":"2023-10-09T08:56:42.204465+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:21.270259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"142'5762","reported_seq":5776,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:20.852542+0000","last_change":"2023-10-09T08:55:45.482390+0000","last_active":"2023-10-09T08:57:20.852542+0000","last_peered":"2023-10-09T08:57:20.852542+0000","last_clean":"2023-10-09T08:57:20.852542+0000","last_became_active":"2023-10-09T08:55:45.482069+0000","last_became_peered":"2023-10-09T08:55:45.482069+0000","last_unstale":"2023-10-09T08:57:20.852542+0000","last_undegraded":"2023-10-09T08:57:20.852542+0000","last_fullsized":"2023-10-09T08:57:20.852542+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5762,"log_dups_size":0,"ondisk_log_size":5762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377618432,"num_objects":5762,"num_object_clones":0,"num_object_copies":17286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5762,"num_write_kb":368768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":98,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.494011+0000","last_change":"2023-10-09T08:56:32.475934+0000","last_active":"2023-10-09T08:56:39.494011+0000","last_peered":"2023-10-09T08:56:39.494011+0000","last_clean":"2023-10-09T08:56:39.494011+0000","last_became_active":"2023-10-09T08:56:32.474729+0000","last_became_peered":"2023-10-09T08:56:32.474729+0000","last_unstale":"2023-10-09T08:56:39.494011+0000","last_undegraded":"2023-10-09T08:56:39.494011+0000","last_fullsized":"2023-10-09T08:56:39.494011+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506243+0000","last_change":"2023-10-09T08:45:49.184088+0000","last_active":"2023-10-09T08:56:39.506243+0000","last_peered":"2023-10-09T08:56:39.506243+0000","last_clean":"2023-10-09T08:56:39.506243+0000","last_became_active":"2023-10-09T08:45:49.183674+0000","last_became_peered":"2023-10-09T08:45:49.183674+0000","last_unstale":"2023-10-09T08:56:39.506243+0000","last_undegraded":"2023-10-09T08:56:39.506243+0000","last_fullsized":"2023-10-09T08:56:39.506243+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":114,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.494010+0000","last_change":"2023-10-09T08:56:32.471336+0000","last_active":"2023-10-09T08:56:39.494010+0000","last_peered":"2023-10-09T08:56:39.494010+0000","last_clean":"2023-10-09T08:56:39.494010+0000","last_became_active":"2023-10-09T08:56:32.470933+0000","last_became_peered":"2023-10-09T08:56:32.470933+0000","last_unstale":"2023-10-09T08:56:39.494010+0000","last_undegraded":"2023-10-09T08:56:39.494010+0000","last_fullsized":"2023-10-09T08:56:39.494010+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"142'5699","reported_seq":5715,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.551295+0000","last_change":"2023-10-09T08:55:45.483446+0000","last_active":"2023-10-09T08:57:22.551295+0000","last_peered":"2023-10-09T08:57:22.551295+0000","last_clean":"2023-10-09T08:57:22.551295+0000","last_became_active":"2023-10-09T08:55:45.483104+0000","last_became_peered":"2023-10-09T08:55:45.483104+0000","last_unstale":"2023-10-09T08:57:22.551295+0000","last_undegraded":"2023-10-09T08:57:22.551295+0000","last_fullsized":"2023-10-09T08:57:22.551295+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:35.565912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373489664,"num_objects":5699,"num_object_clones":0,"num_object_copies":17097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5699,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":145,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:03.438812+0000","last_change":"2023-10-09T08:51:33.431789+0000","last_active":"2023-10-09T08:57:03.438812+0000","last_peered":"2023-10-09T08:57:03.438812+0000","last_clean":"2023-10-09T08:57:03.438812+0000","last_became_active":"2023-10-09T08:51:33.431448+0000","last_became_peered":"2023-10-09T08:51:33.431448+0000","last_unstale":"2023-10-09T08:57:03.438812+0000","last_undegraded":"2023-10-09T08:57:03.438812+0000","last_fullsized":"2023-10-09T08:57:03.438812+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:42.314482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":100,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508472+0000","last_change":"2023-10-09T08:46:49.231894+0000","last_active":"2023-10-09T08:56:39.508472+0000","last_peered":"2023-10-09T08:56:39.508472+0000","last_clean":"2023-10-09T08:56:39.508472+0000","last_became_active":"2023-10-09T08:46:49.231531+0000","last_became_peered":"2023-10-09T08:46:49.231531+0000","last_unstale":"2023-10-09T08:56:39.508472+0000","last_undegraded":"2023-10-09T08:56:39.508472+0000","last_fullsized":"2023-10-09T08:56:39.508472+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:50.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:03.453014+0000","last_change":"2023-10-09T08:46:18.681375+0000","last_active":"2023-10-09T08:57:03.453014+0000","last_peered":"2023-10-09T08:57:03.453014+0000","last_clean":"2023-10-09T08:57:03.453014+0000","last_became_active":"2023-10-09T08:46:18.680713+0000","last_became_peered":"2023-10-09T08:46:18.680713+0000","last_unstale":"2023-10-09T08:57:03.453014+0000","last_undegraded":"2023-10-09T08:57:03.453014+0000","last_fullsized":"2023-10-09T08:57:03.453014+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:59.671965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"142'5850","reported_seq":5867,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.532008+0000","last_change":"2023-10-09T08:55:45.515662+0000","last_active":"2023-10-09T08:57:22.532008+0000","last_peered":"2023-10-09T08:57:22.532008+0000","last_clean":"2023-10-09T08:57:22.532008+0000","last_became_active":"2023-10-09T08:55:45.515059+0000","last_became_peered":"2023-10-09T08:55:45.515059+0000","last_unstale":"2023-10-09T08:57:22.532008+0000","last_undegraded":"2023-10-09T08:57:22.532008+0000","last_fullsized":"2023-10-09T08:57:22.532008+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:44.335086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383385600,"num_objects":5850,"num_object_clones":0,"num_object_copies":17550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5850,"num_write_kb":374400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.493777+0000","last_change":"2023-10-09T08:56:32.472950+0000","last_active":"2023-10-09T08:56:39.493777+0000","last_peered":"2023-10-09T08:56:39.493777+0000","last_clean":"2023-10-09T08:56:39.493777+0000","last_became_active":"2023-10-09T08:56:32.472539+0000","last_became_peered":"2023-10-09T08:56:32.472539+0000","last_unstale":"2023-10-09T08:56:39.493777+0000","last_undegraded":"2023-10-09T08:56:39.493777+0000","last_fullsized":"2023-10-09T08:56:39.493777+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":308,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.503518+0000","last_change":"2023-10-09T08:55:31.002285+0000","last_active":"2023-10-09T08:56:39.503518+0000","last_peered":"2023-10-09T08:56:39.503518+0000","last_clean":"2023-10-09T08:56:39.503518+0000","last_became_active":"2023-10-09T08:55:31.001638+0000","last_became_peered":"2023-10-09T08:55:31.001638+0000","last_unstale":"2023-10-09T08:56:39.503518+0000","last_undegraded":"2023-10-09T08:56:39.503518+0000","last_fullsized":"2023-10-09T08:56:39.503518+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:32.135259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":47,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.493890+0000","last_change":"2023-10-09T08:56:32.473338+0000","last_active":"2023-10-09T08:56:39.493890+0000","last_peered":"2023-10-09T08:56:39.493890+0000","last_clean":"2023-10-09T08:56:39.493890+0000","last_became_active":"2023-10-09T08:56:32.472963+0000","last_became_peered":"2023-10-09T08:56:32.472963+0000","last_unstale":"2023-10-09T08:56:39.493890+0000","last_undegraded":"2023-10-09T08:56:39.493890+0000","last_fullsized":"2023-10-09T08:56:39.493890+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508624+0000","last_change":"2023-10-09T08:54:16.097597+0000","last_active":"2023-10-09T08:56:39.508624+0000","last_peered":"2023-10-09T08:56:39.508624+0000","last_clean":"2023-10-09T08:56:39.508624+0000","last_became_active":"2023-10-09T08:54:16.097205+0000","last_became_peered":"2023-10-09T08:54:16.097205+0000","last_unstale":"2023-10-09T08:56:39.508624+0000","last_undegraded":"2023-10-09T08:56:39.508624+0000","last_fullsized":"2023-10-09T08:56:39.508624+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"142'5888","reported_seq":5904,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:22.602600+0000","last_change":"2023-10-09T08:55:45.480038+0000","last_active":"2023-10-09T08:57:22.602600+0000","last_peered":"2023-10-09T08:57:22.602600+0000","last_clean":"2023-10-09T08:57:22.602600+0000","last_became_active":"2023-10-09T08:55:45.479662+0000","last_became_peered":"2023-10-09T08:55:45.479662+0000","last_unstale":"2023-10-09T08:57:22.602600+0000","last_undegraded":"2023-10-09T08:57:22.602600+0000","last_fullsized":"2023-10-09T08:57:22.602600+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:06.666686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385875968,"num_objects":5888,"num_object_clones":0,"num_object_copies":17664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5888,"num_write_kb":376832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":94,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508544+0000","last_change":"2023-10-09T08:54:16.095375+0000","last_active":"2023-10-09T08:56:39.508544+0000","last_peered":"2023-10-09T08:56:39.508544+0000","last_clean":"2023-10-09T08:56:39.508544+0000","last_became_active":"2023-10-09T08:54:16.095020+0000","last_became_peered":"2023-10-09T08:54:16.095020+0000","last_unstale":"2023-10-09T08:56:39.508544+0000","last_undegraded":"2023-10-09T08:56:39.508544+0000","last_fullsized":"2023-10-09T08:56:39.508544+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:28.039593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":122,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.504269+0000","last_change":"2023-10-09T08:51:33.441477+0000","last_active":"2023-10-09T08:56:39.504269+0000","last_peered":"2023-10-09T08:56:39.504269+0000","last_clean":"2023-10-09T08:56:39.504269+0000","last_became_active":"2023-10-09T08:51:33.441144+0000","last_became_peered":"2023-10-09T08:51:33.441144+0000","last_unstale":"2023-10-09T08:56:39.504269+0000","last_undegraded":"2023-10-09T08:56:39.504269+0000","last_fullsized":"2023-10-09T08:56:39.504269+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:54.567772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":119,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508435+0000","last_change":"2023-10-09T08:46:49.232279+0000","last_active":"2023-10-09T08:56:39.508435+0000","last_peered":"2023-10-09T08:56:39.508435+0000","last_clean":"2023-10-09T08:56:39.508435+0000","last_became_active":"2023-10-09T08:46:49.231911+0000","last_became_peered":"2023-10-09T08:46:49.231911+0000","last_unstale":"2023-10-09T08:56:39.508435+0000","last_undegraded":"2023-10-09T08:56:39.508435+0000","last_fullsized":"2023-10-09T08:56:39.508435+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:38.497218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"142'5736","reported_seq":5769,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:20.087403+0000","last_change":"2023-10-09T08:56:32.539724+0000","last_active":"2023-10-09T08:57:20.087403+0000","last_peered":"2023-10-09T08:57:20.087403+0000","last_clean":"2023-10-09T08:57:20.087403+0000","last_became_active":"2023-10-09T08:56:32.484067+0000","last_became_peered":"2023-10-09T08:56:32.484067+0000","last_unstale":"2023-10-09T08:57:20.087403+0000","last_undegraded":"2023-10-09T08:57:20.087403+0000","last_fullsized":"2023-10-09T08:57:20.087403+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375914496,"num_objects":5736,"num_object_clones":0,"num_object_copies":17208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5736,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":75,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.507049+0000","last_change":"2023-10-09T08:54:16.094534+0000","last_active":"2023-10-09T08:56:39.507049+0000","last_peered":"2023-10-09T08:56:39.507049+0000","last_clean":"2023-10-09T08:56:39.507049+0000","last_became_active":"2023-10-09T08:54:16.094095+0000","last_became_peered":"2023-10-09T08:54:16.094095+0000","last_unstale":"2023-10-09T08:56:39.507049+0000","last_undegraded":"2023-10-09T08:56:39.507049+0000","last_fullsized":"2023-10-09T08:56:39.507049+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:42.851572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":100,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:42.210173+0000","last_change":"2023-10-09T08:46:49.233112+0000","last_active":"2023-10-09T08:56:42.210173+0000","last_peered":"2023-10-09T08:56:42.210173+0000","last_clean":"2023-10-09T08:56:42.210173+0000","last_became_active":"2023-10-09T08:46:49.232724+0000","last_became_peered":"2023-10-09T08:46:49.232724+0000","last_unstale":"2023-10-09T08:56:42.210173+0000","last_undegraded":"2023-10-09T08:56:42.210173+0000","last_fullsized":"2023-10-09T08:56:42.210173+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.285159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":98,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.504298+0000","last_change":"2023-10-09T08:54:16.091388+0000","last_active":"2023-10-09T08:56:39.504298+0000","last_peered":"2023-10-09T08:56:39.504298+0000","last_clean":"2023-10-09T08:56:39.504298+0000","last_became_active":"2023-10-09T08:54:16.088188+0000","last_became_peered":"2023-10-09T08:54:16.088188+0000","last_unstale":"2023-10-09T08:56:39.504298+0000","last_undegraded":"2023-10-09T08:56:39.504298+0000","last_fullsized":"2023-10-09T08:56:39.504298+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:50.065824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"142'5529","reported_seq":5546,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:20.451216+0000","last_change":"2023-10-09T08:55:45.484342+0000","last_active":"2023-10-09T08:57:20.451216+0000","last_peered":"2023-10-09T08:57:20.451216+0000","last_clean":"2023-10-09T08:57:20.451216+0000","last_became_active":"2023-10-09T08:55:45.483601+0000","last_became_peered":"2023-10-09T08:55:45.483601+0000","last_unstale":"2023-10-09T08:57:20.451216+0000","last_undegraded":"2023-10-09T08:57:20.451216+0000","last_fullsized":"2023-10-09T08:57:20.451216+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:25.586122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362348544,"num_objects":5529,"num_object_clones":0,"num_object_copies":16587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5529,"num_write_kb":353856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":126,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.512892+0000","last_change":"2023-10-09T08:54:16.155171+0000","last_active":"2023-10-09T08:56:39.512892+0000","last_peered":"2023-10-09T08:56:39.512892+0000","last_clean":"2023-10-09T08:56:39.512892+0000","last_became_active":"2023-10-09T08:54:16.090138+0000","last_became_peered":"2023-10-09T08:54:16.090138+0000","last_unstale":"2023-10-09T08:56:39.512892+0000","last_undegraded":"2023-10-09T08:56:39.512892+0000","last_fullsized":"2023-10-09T08:56:39.512892+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:44.395778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:03.452930+0000","last_change":"2023-10-09T08:46:18.678094+0000","last_active":"2023-10-09T08:57:03.452930+0000","last_peered":"2023-10-09T08:57:03.452930+0000","last_clean":"2023-10-09T08:57:03.452930+0000","last_became_active":"2023-10-09T08:46:18.677705+0000","last_became_peered":"2023-10-09T08:46:18.677705+0000","last_unstale":"2023-10-09T08:57:03.452930+0000","last_undegraded":"2023-10-09T08:57:03.452930+0000","last_fullsized":"2023-10-09T08:57:03.452930+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:31.647828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.503583+0000","last_change":"2023-10-09T08:54:16.091706+0000","last_active":"2023-10-09T08:56:39.503583+0000","last_peered":"2023-10-09T08:56:39.503583+0000","last_clean":"2023-10-09T08:56:39.503583+0000","last_became_active":"2023-10-09T08:54:16.091344+0000","last_became_peered":"2023-10-09T08:54:16.091344+0000","last_unstale":"2023-10-09T08:56:39.503583+0000","last_undegraded":"2023-10-09T08:56:39.503583+0000","last_fullsized":"2023-10-09T08:56:39.503583+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:11.816910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.2","version":"142'5789","reported_seq":5806,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:20.415421+0000","last_change":"2023-10-09T08:55:45.479441+0000","last_active":"2023-10-09T08:57:20.415421+0000","last_peered":"2023-10-09T08:57:20.415421+0000","last_clean":"2023-10-09T08:57:20.415421+0000","last_became_active":"2023-10-09T08:55:45.479049+0000","last_became_peered":"2023-10-09T08:55:45.479049+0000","last_unstale":"2023-10-09T08:57:20.415421+0000","last_undegraded":"2023-10-09T08:57:20.415421+0000","last_fullsized":"2023-10-09T08:57:20.415421+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":134,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:55:44.451704+0000","last_clean_scrub_stamp":"2023-10-09T08:55:44.451704+0000","objects_scrubbed":0,"log_size":5789,"log_dups_size":0,"ondisk_log_size":5789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:54.089120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379387904,"num_objects":5789,"num_object_clones":0,"num_object_copies":17367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5789,"num_write_kb":370496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":122,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508710+0000","last_change":"2023-10-09T08:50:37.369204+0000","last_active":"2023-10-09T08:56:39.508710+0000","last_peered":"2023-10-09T08:56:39.508710+0000","last_clean":"2023-10-09T08:56:39.508710+0000","last_became_active":"2023-10-09T08:50:37.368197+0000","last_became_peered":"2023-10-09T08:50:37.368197+0000","last_unstale":"2023-10-09T08:56:39.508710+0000","last_undegraded":"2023-10-09T08:56:39.508710+0000","last_fullsized":"2023-10-09T08:56:39.508710+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.508547+0000","last_change":"2023-10-09T08:46:18.683082+0000","last_active":"2023-10-09T08:56:39.508547+0000","last_peered":"2023-10-09T08:56:39.508547+0000","last_clean":"2023-10-09T08:56:39.508547+0000","last_became_active":"2023-10-09T08:46:18.682657+0000","last_became_peered":"2023-10-09T08:46:18.682657+0000","last_unstale":"2023-10-09T08:56:39.508547+0000","last_undegraded":"2023-10-09T08:56:39.508547+0000","last_fullsized":"2023-10-09T08:56:39.508547+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:45.531585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.509730+0000","last_change":"2023-10-09T08:34:27.749854+0000","last_active":"2023-10-09T08:56:39.509730+0000","last_peered":"2023-10-09T08:56:39.509730+0000","last_clean":"2023-10-09T08:56:39.509730+0000","last_became_active":"2023-10-09T08:34:27.749445+0000","last_became_peered":"2023-10-09T08:34:27.749445+0000","last_unstale":"2023-10-09T08:56:39.509730+0000","last_undegraded":"2023-10-09T08:56:39.509730+0000","last_fullsized":"2023-10-09T08:56:39.509730+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":101,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.504431+0000","last_change":"2023-10-09T08:54:16.091839+0000","last_active":"2023-10-09T08:56:39.504431+0000","last_peered":"2023-10-09T08:56:39.504431+0000","last_clean":"2023-10-09T08:56:39.504431+0000","last_became_active":"2023-10-09T08:54:16.091292+0000","last_became_peered":"2023-10-09T08:54:16.091292+0000","last_unstale":"2023-10-09T08:56:39.504431+0000","last_undegraded":"2023-10-09T08:56:39.504431+0000","last_fullsized":"2023-10-09T08:56:39.504431+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:04.866003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":133,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.511989+0000","last_change":"2023-10-09T08:51:53.456605+0000","last_active":"2023-10-09T08:56:39.511989+0000","last_peered":"2023-10-09T08:56:39.511989+0000","last_clean":"2023-10-09T08:56:39.511989+0000","last_became_active":"2023-10-09T08:51:53.456254+0000","last_became_peered":"2023-10-09T08:51:53.456254+0000","last_unstale":"2023-10-09T08:56:39.511989+0000","last_undegraded":"2023-10-09T08:56:39.511989+0000","last_fullsized":"2023-10-09T08:56:39.511989+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:22.564027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":96,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.495285+0000","last_change":"2023-10-09T08:56:32.475206+0000","last_active":"2023-10-09T08:56:39.495285+0000","last_peered":"2023-10-09T08:56:39.495285+0000","last_clean":"2023-10-09T08:56:39.495285+0000","last_became_active":"2023-10-09T08:56:32.474621+0000","last_became_peered":"2023-10-09T08:56:32.474621+0000","last_unstale":"2023-10-09T08:56:39.495285+0000","last_undegraded":"2023-10-09T08:56:39.495285+0000","last_fullsized":"2023-10-09T08:56:39.495285+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506537+0000","last_change":"2023-10-09T08:47:16.968742+0000","last_active":"2023-10-09T08:56:39.506537+0000","last_peered":"2023-10-09T08:56:39.506537+0000","last_clean":"2023-10-09T08:56:39.506537+0000","last_became_active":"2023-10-09T08:47:16.968317+0000","last_became_peered":"2023-10-09T08:47:16.968317+0000","last_unstale":"2023-10-09T08:56:39.506537+0000","last_undegraded":"2023-10-09T08:56:39.506537+0000","last_fullsized":"2023-10-09T08:56:39.506537+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":81,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.495430+0000","last_change":"2023-10-09T08:56:32.474675+0000","last_active":"2023-10-09T08:56:39.495430+0000","last_peered":"2023-10-09T08:56:39.495430+0000","last_clean":"2023-10-09T08:56:39.495430+0000","last_became_active":"2023-10-09T08:56:32.474212+0000","last_became_peered":"2023-10-09T08:56:32.474212+0000","last_unstale":"2023-10-09T08:56:39.495430+0000","last_undegraded":"2023-10-09T08:56:39.495430+0000","last_fullsized":"2023-10-09T08:56:39.495430+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.506031+0000","last_change":"2023-10-09T08:51:33.443239+0000","last_active":"2023-10-09T08:56:39.506031+0000","last_peered":"2023-10-09T08:56:39.506031+0000","last_clean":"2023-10-09T08:56:39.506031+0000","last_became_active":"2023-10-09T08:51:33.442063+0000","last_became_peered":"2023-10-09T08:51:33.442063+0000","last_unstale":"2023-10-09T08:56:39.506031+0000","last_undegraded":"2023-10-09T08:56:39.506031+0000","last_fullsized":"2023-10-09T08:56:39.506031+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":121,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T08:57:03.448498+0000","last_change":"2023-10-09T08:47:16.970132+0000","last_active":"2023-10-09T08:57:03.448498+0000","last_peered":"2023-10-09T08:57:03.448498+0000","last_clean":"2023-10-09T08:57:03.448498+0000","last_became_active":"2023-10-09T08:47:16.969760+0000","last_became_peered":"2023-10-09T08:47:16.969760+0000","last_unstale":"2023-10-09T08:57:03.448498+0000","last_undegraded":"2023-10-09T08:57:03.448498+0000","last_fullsized":"2023-10-09T08:57:03.448498+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.801830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":120,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:47.551201+0000","last_change":"2023-10-09T08:51:53.455827+0000","last_active":"2023-10-09T08:56:47.551201+0000","last_peered":"2023-10-09T08:56:47.551201+0000","last_clean":"2023-10-09T08:56:47.551201+0000","last_became_active":"2023-10-09T08:51:53.455328+0000","last_became_peered":"2023-10-09T08:51:53.455328+0000","last_unstale":"2023-10-09T08:56:47.551201+0000","last_undegraded":"2023-10-09T08:56:47.551201+0000","last_fullsized":"2023-10-09T08:56:47.551201+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:46.784739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":123,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.507310+0000","last_change":"2023-10-09T08:51:33.445775+0000","last_active":"2023-10-09T08:56:39.507310+0000","last_peered":"2023-10-09T08:56:39.507310+0000","last_clean":"2023-10-09T08:56:39.507310+0000","last_became_active":"2023-10-09T08:51:33.440135+0000","last_became_peered":"2023-10-09T08:51:33.440135+0000","last_unstale":"2023-10-09T08:56:39.507310+0000","last_undegraded":"2023-10-09T08:56:39.507310+0000","last_fullsized":"2023-10-09T08:56:39.507310+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:02.419868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T08:56:39.505953+0000","last_change":"2023-10-09T08:46:18.676602+0000","last_active":"2023-10-09T08:56:39.505953+0000","last_peered":"2023-10-09T08:56:39.505953+0000","last_clean":"2023-10-09T08:56:39.505953+0000","last_became_active":"2023-10-09T08:46:18.676122+0000","last_became_peered":"2023-10-09T08:46:18.676122+0000","last_unstale":"2023-10-09T08:56:39.505953+0000","last_undegraded":"2023-10-09T08:56:39.505953+0000","last_fullsized":"2023-10-09T08:56:39.505953+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5994905600,"num_objects":91475,"num_object_clones":0,"num_object_copies":274425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":91475,"num_write_kb":5854400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17974296576,"data_stored":17974296576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91475,"ondisk_log_size":91475,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5434,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430522,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626755,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2105464,"kb_used_data":1804144,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91639688,"statfs":{"total":95995035648,"available":93839040512,"internally_reserved":0,"allocated":1847443456,"data_stored":1846975766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:57:12 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":2.1190000000000002,"15min":1.361},"min":{"1min":0.29699999999999999,"5min":0.251,"15min":0.249},"max":{"1min":39.991,"5min":39.991,"15min":39.991},"last":1.7230000000000001},{"interface":"front","average":{"1min":3.3660000000000001,"5min":1.849,"15min":1.3200000000000001},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":39.951000000000001,"5min":39.951000000000001,"15min":39.951000000000001},"last":1.7849999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":2.1760000000000002,"15min":1.3680000000000001},"min":{"1min":0.31,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.6150000000000002,"5min":62.219000000000001,"15min":62.219000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.81899999999999995,"5min":2.2959999999999998,"15min":1.4059999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":2.593,"5min":57.021000000000001,"15min":57.021000000000001},"last":0.376}]},{"osd":2,"last update":"Mon Oct 9 08:57:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.8660000000000001,"15min":1.292},"min":{"1min":0.30199999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4700000000000002,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":1.806},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.77,"15min":1.25},"min":{"1min":0.436,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.391,"5min":10.747,"15min":10.747},"last":0.42899999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":2.4910000000000001,"15min":1.6930000000000001},"min":{"1min":0.34799999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.9099999999999999,"5min":55.713000000000001,"15min":57.636000000000003},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.9259999999999999,"15min":1.46},"min":{"1min":0.45500000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.9770000000000001,"5min":56.064,"15min":58.393000000000001},"last":5.6630000000000003}]},{"osd":4,"last update":"Mon Oct 9 08:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":2.3719999999999999,"15min":1.4019999999999999},"min":{"1min":0.371,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.446,"5min":12.462999999999999,"15min":12.462999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.76900000000000002,"5min":1.5209999999999999,"15min":1.135},"min":{"1min":0.40400000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.2200000000000002,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Oct 9 08:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.9370000000000001,"15min":1.3069999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.546,"5min":12.404999999999999,"15min":12.404999999999999},"last":0.498},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.5249999999999999,"15min":1.2130000000000001},"min":{"1min":0.41099999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.3939999999999999,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.56599999999999995}]},{"osd":7,"last update":"Mon Oct 9 08:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":2.1200000000000001,"15min":1.4419999999999999},"min":{"1min":0.46600000000000003,"5min":0.44,"15min":0.40500000000000003},"max":{"1min":2.706,"5min":11.382,"15min":11.382},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.755,"5min":1.835,"15min":1.349},"min":{"1min":0.43099999999999999,"5min":0.39000000000000001,"15min":0.377},"max":{"1min":1.8540000000000001,"5min":9.4800000000000004,"15min":9.4800000000000004},"last":0.73899999999999999}]}]},{"osd":1,"up_from":83,"seq":356482285703,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3859616,"kb_used_data":3669720,"kb_used_omap":9,"kb_used_meta":189878,"kb_avail":89885536,"statfs":{"total":95995035648,"available":92042788864,"internally_reserved":0,"allocated":3757793280,"data_stored":3757172535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":194436073},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:56:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.77,"15min":1.0509999999999999},"min":{"1min":0.219,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":9.6590000000000007,"5min":43.058999999999997,"15min":43.058999999999997},"last":0.24099999999999999},{"interface":"front","average":{"1min":2.806,"5min":2.0750000000000002,"15min":1.1699999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.0530000000000008,"5min":41.850000000000001,"15min":41.850000000000001},"last":2.7080000000000002}]},{"osd":2,"last update":"Mon Oct 9 08:57:07 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.3979999999999999,"15min":1.038},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.252},"max":{"1min":10.789999999999999,"5min":10.789999999999999,"15min":10.789999999999999},"last":1.778},{"interface":"front","average":{"1min":2.3100000000000001,"5min":1.46,"15min":1.026},"min":{"1min":0.42799999999999999,"5min":0.29199999999999998,"15min":0.23999999999999999},"max":{"1min":7.8449999999999998,"5min":7.8449999999999998,"15min":7.8449999999999998},"last":2.3260000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:56:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.6359999999999999,"15min":1.1559999999999999},"min":{"1min":0.58799999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3310000000000004,"5min":13.01,"15min":13.01},"last":2.2989999999999999},{"interface":"front","average":{"1min":1.996,"5min":1.502,"15min":1.115},"min":{"1min":0.433,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.5250000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":2.1680000000000001}]},{"osd":4,"last update":"Mon Oct 9 08:56:40 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":1.603,"15min":1.1799999999999999},"min":{"1min":0.39000000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.532,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":5.8200000000000003},{"interface":"front","average":{"1min":1.46,"5min":1.851,"15min":1.2829999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":6.7519999999999998,"5min":12.045999999999999,"15min":12.045999999999999},"last":3.831}]},{"osd":5,"last update":"Mon Oct 9 08:56:40 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":2.0470000000000002,"15min":1.413},"min":{"1min":0.40400000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":5.21,"5min":12.673,"15min":12.673},"last":1.53},{"interface":"front","average":{"1min":2.0739999999999998,"5min":2.0529999999999999,"15min":1.391},"min":{"1min":0.48799999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.9610000000000003,"5min":14.439,"15min":14.439},"last":5.9859999999999998}]},{"osd":6,"last update":"Mon Oct 9 08:56:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":2.2509999999999999,"15min":1.486},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":6.8150000000000004,"5min":12.619999999999999,"15min":12.619999999999999},"last":2.1970000000000001},{"interface":"front","average":{"1min":1.806,"5min":2.5619999999999998,"15min":1.611},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":5.2809999999999997,"5min":12.718,"15min":12.718},"last":1.7470000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:56:40 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.0409999999999999,"15min":1.385},"min":{"1min":0.55900000000000005,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.4009999999999998,"5min":10.855,"15min":10.855},"last":5.7750000000000004},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.6819999999999999,"15min":1.3160000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":6.9859999999999998,"5min":8.8390000000000004,"15min":8.8390000000000004},"last":2.5310000000000001}]}]},{"osd":0,"up_from":78,"seq":335007449229,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3452900,"kb_used_data":3304796,"kb_used_omap":28,"kb_used_meta":148067,"kb_avail":90292252,"statfs":{"total":95995035648,"available":92459266048,"internally_reserved":0,"allocated":3384111104,"data_stored":3383486938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":151621456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 08:56:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.242,"15min":1.0389999999999999},"min":{"1min":0.29899999999999999,"5min":0.23499999999999999,"15min":0.21199999999999999},"max":{"1min":6.7539999999999996,"5min":8.2579999999999991,"15min":37.380000000000003},"last":0.70299999999999996},{"interface":"front","average":{"1min":2.004,"5min":1.363,"15min":1.105},"min":{"1min":0.41699999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.4729999999999999,"5min":12.615,"15min":38.012999999999998},"last":0.47199999999999998}]},{"osd":2,"last update":"Mon Oct 9 08:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.1040000000000001,"15min":0.96999999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.8390000000000004,"5min":6.032,"15min":6.032},"last":0.73599999999999999},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.349,"15min":1.1080000000000001},"min":{"1min":0.626,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.2720000000000002,"5min":7.5940000000000003,"15min":7.5940000000000003},"last":0.75600000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:56:27 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.4830000000000001,"15min":1.694},"min":{"1min":0.39500000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.7729999999999997,"5min":110.988,"15min":110.988},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.167,"5min":2.3029999999999999,"15min":1.6499999999999999},"min":{"1min":0.54800000000000004,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.4210000000000003,"5min":107.66500000000001,"15min":107.66500000000001},"last":0.36499999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.6240000000000001,"15min":1.222},"min":{"1min":0.375,"5min":0.375,"15min":0.35199999999999998},"max":{"1min":5.5309999999999997,"5min":8.2840000000000007,"15min":8.2840000000000007},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.768,"5min":1.7150000000000001,"15min":1.238},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.9290000000000003,"5min":6.4610000000000003,"15min":6.5700000000000003},"last":0.33700000000000002}]},{"osd":5,"last update":"Mon Oct 9 08:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.1879999999999999,"15min":1.0820000000000001},"min":{"1min":0.37,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.8680000000000003,"5min":7.9710000000000001,"15min":7.9710000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.877,"5min":1.5940000000000001,"15min":1.2789999999999999},"min":{"1min":0.45800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":5.4290000000000003,"5min":6.1550000000000002,"15min":6.1550000000000002},"last":0.65900000000000003}]},{"osd":6,"last update":"Mon Oct 9 08:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.5629999999999999,"15min":1.2969999999999999},"min":{"1min":0.40500000000000003,"5min":0.376,"15min":0.376},"max":{"1min":4.8129999999999997,"5min":7.9359999999999999,"15min":7.9359999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.772,"15min":1.3879999999999999},"min":{"1min":0.50900000000000001,"5min":0.45300000000000001,"15min":0.42099999999999999},"max":{"1min":5.5,"5min":7.952,"15min":7.952},"last":0.69299999999999995}]},{"osd":7,"last update":"Mon Oct 9 08:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.833,"15min":1.4179999999999999},"min":{"1min":0.48399999999999999,"5min":0.42599999999999999,"15min":0.39400000000000002},"max":{"1min":5.4720000000000004,"5min":7.774,"15min":7.774},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.839,"5min":1.861,"15min":1.415},"min":{"1min":0.55200000000000005,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":4.8010000000000002,"5min":9.0250000000000004,"15min":9.0250000000000004},"last":0.434}]}]},{"osd":2,"up_from":88,"seq":377957122177,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2971160,"kb_used_data":2740944,"kb_used_omap":81,"kb_used_meta":230126,"kb_avail":90773992,"statfs":{"total":95995035648,"available":92952567808,"internally_reserved":0,"allocated":2806726656,"data_stored":2806115932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":235649242},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:57:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4489999999999998,"5min":1.52,"15min":1.014},"min":{"1min":0.23200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":8.7409999999999997,"5min":9.9049999999999994,"15min":9.9049999999999994},"last":1.3839999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.80200000000000005,"15min":0.84399999999999997},"min":{"1min":0.26600000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.7869999999999999,"5min":7.1429999999999998,"15min":7.2450000000000001},"last":0.64700000000000002}]},{"osd":1,"last update":"Mon Oct 9 08:57:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":1.712,"15min":1.165},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4850000000000003,"5min":9.9570000000000007,"15min":11.728},"last":2.4710000000000001},{"interface":"front","average":{"1min":1.6839999999999999,"5min":1.167,"15min":1.03},"min":{"1min":0.30099999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.944,"5min":5.944,"15min":8.266},"last":3.5049999999999999}]},{"osd":3,"last update":"Mon Oct 9 08:56:35 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.248,"15min":0.99099999999999999},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.96,"5min":7.3019999999999996,"15min":7.3019999999999996},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.637,"5min":1.9850000000000001,"15min":1.2809999999999999},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.7039999999999997,"5min":45.512999999999998,"15min":45.512999999999998},"last":1.345}]},{"osd":4,"last update":"Mon Oct 9 08:57:14 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":2.0830000000000002,"15min":1.4119999999999999},"min":{"1min":0.46700000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.9939999999999998,"5min":12.191000000000001,"15min":12.191000000000001},"last":0.95999999999999996},{"interface":"front","average":{"1min":2.508,"5min":2.2679999999999998,"15min":1.4790000000000001},"min":{"1min":0.45700000000000002,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":6.0460000000000003,"5min":14.071999999999999,"15min":14.071999999999999},"last":1.4379999999999999}]},{"osd":5,"last update":"Mon Oct 9 08:57:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.548,"15min":1.296},"min":{"1min":0.40600000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.9960000000000004,"5min":6.556,"15min":7.3419999999999996},"last":0.76500000000000001},{"interface":"front","average":{"1min":2.4710000000000001,"5min":1.998,"15min":1.4370000000000001},"min":{"1min":0.41499999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.8609999999999998,"5min":9.6080000000000005,"15min":9.6080000000000005},"last":0.98799999999999999}]},{"osd":6,"last update":"Mon Oct 9 08:57:14 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":2.2480000000000002,"15min":1.516},"min":{"1min":0.42999999999999999,"5min":0.40799999999999997,"15min":0.33900000000000002},"max":{"1min":7.3689999999999998,"5min":10.077999999999999,"15min":10.077999999999999},"last":1.4139999999999999},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.6200000000000001,"15min":1.365},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":6.0430000000000001,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":0.79500000000000004}]},{"osd":7,"last update":"Mon Oct 9 08:57:14 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":2.2040000000000002,"15min":1.526},"min":{"1min":0.53100000000000003,"5min":0.44,"15min":0.38800000000000001},"max":{"1min":6.0739999999999998,"5min":17.77,"15min":17.77},"last":1.46},{"interface":"front","average":{"1min":1.173,"5min":1.4910000000000001,"15min":1.3420000000000001},"min":{"1min":0.47599999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.6399999999999997,"5min":7.5469999999999997,"15min":7.5469999999999997},"last":0.81699999999999995}]}]},{"osd":3,"up_from":93,"seq":399431958651,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2370240,"kb_used_data":2162296,"kb_used_omap":7,"kb_used_meta":207928,"kb_avail":91374912,"statfs":{"total":95995035648,"available":93567909888,"internally_reserved":0,"allocated":2214191104,"data_stored":2213597307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":212918888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.2070000000000001,"15min":0.97399999999999998},"min":{"1min":0.28100000000000003,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.4470000000000001,"5min":13.916,"15min":13.916},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.117,"5min":1.1020000000000001,"15min":0.96599999999999997},"min":{"1min":0.36599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.4119999999999999,"5min":6.6589999999999998,"15min":7.5919999999999996},"last":0.65100000000000002}]},{"osd":1,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.4510000000000001,"15min":1.103},"min":{"1min":0.43099999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.9870000000000001,"5min":8.0909999999999993,"15min":12.300000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.423,"5min":1.3899999999999999,"15min":1.0840000000000001},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.245},"max":{"1min":6.9729999999999999,"5min":8.8209999999999997,"15min":12.564},"last":0.95499999999999996}]},{"osd":2,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.718,"15min":1.2749999999999999},"min":{"1min":0.35799999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":4.6600000000000001,"5min":9.9260000000000002,"15min":28.655000000000001},"last":2.4809999999999999},{"interface":"front","average":{"1min":1.2509999999999999,"5min":1.569,"15min":1.268},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.27100000000000002},"max":{"1min":6.1070000000000002,"5min":9.8670000000000009,"15min":30.785},"last":3.1339999999999999}]},{"osd":4,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":1.607,"15min":1.2929999999999999},"min":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.9870000000000001,"5min":8.8699999999999992,"15min":12.877000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.8200000000000001,"15min":1.3580000000000001},"min":{"1min":0.57399999999999995,"5min":0.376,"15min":0.35199999999999998},"max":{"1min":2.4409999999999998,"5min":10.026,"15min":10.026},"last":0.626}]},{"osd":5,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.5649999999999999,"15min":1.2829999999999999},"min":{"1min":0.62,"5min":0.374,"15min":0.374},"max":{"1min":2.4860000000000002,"5min":7.9900000000000002,"15min":8.0050000000000008},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.913,"15min":1.423},"min":{"1min":0.42999999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.6960000000000002,"5min":9.1189999999999998,"15min":12.204000000000001},"last":0.46700000000000003}]},{"osd":6,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.823,"15min":1.4279999999999999},"min":{"1min":0.68999999999999995,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":4,"5min":7.1920000000000002,"15min":10.048},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.2829999999999999,"5min":2.0449999999999999,"15min":1.536},"min":{"1min":0.54300000000000004,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.5389999999999997,"5min":8.1980000000000004,"15min":12.275},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 08:56:34 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.8799999999999999,"15min":1.46},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.7450000000000001,"5min":10.284000000000001,"15min":12.244},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.861,"15min":1.4710000000000001},"min":{"1min":0.47499999999999998,"5min":0.435,"15min":0.39000000000000001},"max":{"1min":2.0979999999999999,"5min":13.339,"15min":13.339},"last":0.58199999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019221,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2433720,"kb_used_data":2128304,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91311432,"statfs":{"total":95995035648,"available":93502906368,"internally_reserved":0,"allocated":2179383296,"data_stored":2178914865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:57:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6899999999999999,"5min":2.4670000000000001,"15min":1.5249999999999999},"min":{"1min":0.31,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":47.978999999999999,"5min":47.978999999999999,"15min":47.978999999999999},"last":3.5089999999999999},{"interface":"front","average":{"1min":3.7370000000000001,"5min":2.4390000000000001,"15min":1.4970000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":49.213000000000001,"5min":49.213000000000001,"15min":49.213000000000001},"last":1.8069999999999999}]},{"osd":1,"last update":"Mon Oct 9 08:56:31 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.8819999999999999,"15min":1.2130000000000001},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.6160000000000001,"5min":8.9169999999999998,"15min":8.9169999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.8049999999999999,"15min":1.236},"min":{"1min":0.374,"5min":0.313,"15min":0.313},"max":{"1min":3.1219999999999999,"5min":11.712999999999999,"15min":11.712999999999999},"last":3.4689999999999999}]},{"osd":2,"last update":"Mon Oct 9 08:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.8440000000000001,"15min":1.387},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.32800000000000001},"max":{"1min":3.9780000000000002,"5min":17.582999999999998,"15min":17.582999999999998},"last":3.2469999999999999},{"interface":"front","average":{"1min":1.524,"5min":1.788,"15min":1.252},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.1900000000000004,"5min":9.1229999999999993,"15min":9.1229999999999993},"last":4.3170000000000002}]},{"osd":3,"last update":"Mon Oct 9 08:56:31 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":2.052,"15min":1.3640000000000001},"min":{"1min":0.39000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.0489999999999999,"5min":9.5969999999999995,"15min":9.5969999999999995},"last":1.742},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.7410000000000001,"15min":1.288},"min":{"1min":0.495,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.202,"5min":9.1199999999999992,"15min":9.1199999999999992},"last":0.434}]},{"osd":5,"last update":"Mon Oct 9 08:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":1.706,"15min":1.179},"min":{"1min":0.441,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.3,"5min":11.762,"15min":11.762},"last":3.1509999999999998},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.9950000000000001,"15min":1.3320000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.996,"5min":12.85,"15min":12.85},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 08:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":2.3220000000000001,"15min":1.4239999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.8540000000000001,"5min":10.657999999999999,"15min":10.657999999999999},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.89900000000000002,"5min":2.0600000000000001,"15min":1.3979999999999999},"min":{"1min":0.55000000000000004,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.518,"5min":8.6310000000000002,"15min":8.6310000000000002},"last":3.1970000000000001}]},{"osd":7,"last update":"Mon Oct 9 08:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":2.2549999999999999,"15min":1.4790000000000001},"min":{"1min":0.53000000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.052,"5min":13.933999999999999,"15min":13.933999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.83999999999999997,"5min":2.468,"15min":1.6339999999999999},"min":{"1min":0.43099999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.7909999999999999,"5min":13.321999999999999,"15min":13.911},"last":0.60999999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822989,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2483240,"kb_used_data":2173728,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91261912,"statfs":{"total":95995035648,"available":93452197888,"internally_reserved":0,"allocated":2225897472,"data_stored":2225441169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 08:57:09 2023","interfaces":[{"interface":"back","average":{"1min":2.992,"5min":2.016,"15min":1.302},"min":{"1min":0.34499999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":24.329000000000001,"5min":24.329000000000001,"15min":24.329000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.3500000000000001,"5min":2.0009999999999999,"15min":1.284},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":23.704999999999998,"5min":23.704999999999998,"15min":23.704999999999998},"last":0.51400000000000001}]},{"osd":1,"last update":"Mon Oct 9 08:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":1.7290000000000001,"15min":1.1970000000000001},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.8410000000000002,"5min":9.1630000000000003,"15min":9.1630000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.5489999999999999,"5min":2.5089999999999999,"15min":1.4470000000000001},"min":{"1min":0.41099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.218,"5min":35.042000000000002,"15min":35.042000000000002},"last":7.5510000000000002}]},{"osd":2,"last update":"Mon Oct 9 08:57:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.9970000000000001,"15min":1.333},"min":{"1min":0.433,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":26.911999999999999,"5min":26.911999999999999,"15min":26.911999999999999},"last":1.024},{"interface":"front","average":{"1min":2.4169999999999998,"5min":1.7529999999999999,"15min":1.262},"min":{"1min":0.40000000000000002,"5min":0.32900000000000001,"15min":0.32600000000000001},"max":{"1min":25.538,"5min":25.538,"15min":25.538},"last":4.0030000000000001}]},{"osd":3,"last update":"Mon Oct 9 08:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.651,"15min":1.2210000000000001},"min":{"1min":0.39200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.4500000000000002,"5min":7.915,"15min":7.915},"last":2.3610000000000002},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.7929999999999999,"15min":1.2350000000000001},"min":{"1min":0.55000000000000004,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.734,"5min":7.9509999999999996,"15min":7.9509999999999996},"last":0.54700000000000004}]},{"osd":4,"last update":"Mon Oct 9 08:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.016,"15min":1.2729999999999999},"min":{"1min":0.42899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":1.986,"5min":9.2379999999999995,"15min":9.2379999999999995},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.875,"15min":1.282},"min":{"1min":0.41599999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.4990000000000001,"5min":8.141,"15min":8.141},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 08:56:24 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":2.2890000000000001,"15min":1.415},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.1709999999999998,"5min":11.919,"15min":11.919},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.89300000000000002,"5min":2.0099999999999998,"15min":1.3759999999999999},"min":{"1min":0.46300000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.3520000000000001,"5min":11.234,"15min":11.234},"last":0.94999999999999996}]},{"osd":7,"last update":"Mon Oct 9 08:56:24 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":2.2509999999999999,"15min":1.4950000000000001},"min":{"1min":0.50600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.413,"5min":15.849,"15min":15.849},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.995,"5min":2.4359999999999999,"15min":1.534},"min":{"1min":0.48599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":2.2400000000000002,"5min":12.346,"15min":12.346},"last":0.879}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3393781760,"data_stored":3393781760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3783983104,"data_stored":3783983104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2652307456,"data_stored":2652307456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1862664192,"data_stored":1862664192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2190868480,"data_stored":2190868480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2234515456,"data_stored":2234515456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1856176128,"data_stored":1856176128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T08:57:28.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-10-09T08:57:28.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:28.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:28.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:28.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:28.751 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:28.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:28.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:28.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:28.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:28.892 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:28 smithi016 bash[75179]: cluster 2023-10-09T08:57:27.690306+0000 mgr.y (mgr.25421) 597 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T08:57:28.892 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:28 smithi016 bash[77892]: cluster 2023-10-09T08:57:27.690306+0000 mgr.y (mgr.25421) 597 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T08:57:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:28.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:28.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:28.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:28 smithi094 bash[59677]: cluster 2023-10-09T08:57:27.690306+0000 mgr.y (mgr.25421) 597 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T08:57:29.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:29.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:29.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:29.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:29.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:29.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:29.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:29.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:29.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:29.339 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 97483 97467 70.8213 43.5625 0.0148502 0.014113 2023-10-09T08:57:29.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:29.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:29.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:29.453 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:29.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:29.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:29.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:29.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:29.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:29.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:29.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:29.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:29.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:29.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:29.764 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T08:57:29.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:29.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:29.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:29.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:29.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:29.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:29 smithi016 bash[75179]: audit 2023-10-09T08:57:29.279438+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.16:0/2178504770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:57:29.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:29 smithi016 bash[75179]: audit 2023-10-09T08:57:29.281515+0000 mon.a (mon.0) 791 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:57:29.989 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:29 smithi016 bash[77892]: audit 2023-10-09T08:57:29.279438+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.16:0/2178504770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:57:29.989 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:29 smithi016 bash[77892]: audit 2023-10-09T08:57:29.281515+0000 mon.a (mon.0) 791 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:57:30.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:30.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:30.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:29 smithi094 bash[59677]: audit 2023-10-09T08:57:29.279438+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.16:0/2178504770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:57:30.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:29 smithi094 bash[59677]: audit 2023-10-09T08:57:29.281515+0000 mon.a (mon.0) 791 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:57:30.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:30.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:30.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:30.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:30.272 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:30.339 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 98288 98272 70.5856 50.3125 0.0170107 0.0141609 2023-10-09T08:57:30.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:30.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:30.507 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:30.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:30.635 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:30.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:30.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:30.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:30.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:30.902 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:30.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:30 smithi094 bash[59677]: cluster 2023-10-09T08:57:29.690744+0000 mgr.y (mgr.25421) 598 : cluster 0 pgmap v571: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s 2023-10-09T08:57:30.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:30 smithi094 bash[59677]: cluster 2023-10-09T08:57:29.745732+0000 mon.a (mon.0) 792 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:57:30.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:30 smithi094 bash[59677]: audit 2023-10-09T08:57:29.755074+0000 mon.a (mon.0) 793 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:57:30.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:30 smithi094 bash[59677]: cluster 2023-10-09T08:57:29.755171+0000 mon.a (mon.0) 794 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-10-09T08:57:30.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:30 smithi094 bash[59677]: cluster 2023-10-09T08:57:29.761026+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:30.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:30 smithi094 bash[59677]: cluster 2023-10-09T08:57:29.761054+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:30.911 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:30 smithi094 bash[59677]: cluster 2023-10-09T08:57:29.761257+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:30.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:30.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:30.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:31.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[75179]: cluster 2023-10-09T08:57:29.690744+0000 mgr.y (mgr.25421) 598 : cluster 0 pgmap v571: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[75179]: cluster 2023-10-09T08:57:29.745732+0000 mon.a (mon.0) 792 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[75179]: audit 2023-10-09T08:57:29.755074+0000 mon.a (mon.0) 793 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[75179]: cluster 2023-10-09T08:57:29.755171+0000 mon.a (mon.0) 794 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[75179]: cluster 2023-10-09T08:57:29.761026+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[75179]: cluster 2023-10-09T08:57:29.761054+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[75179]: cluster 2023-10-09T08:57:29.761257+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:31.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[77892]: cluster 2023-10-09T08:57:29.690744+0000 mgr.y (mgr.25421) 598 : cluster 0 pgmap v571: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s 2023-10-09T08:57:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[77892]: cluster 2023-10-09T08:57:29.745732+0000 mon.a (mon.0) 792 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:57:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[77892]: audit 2023-10-09T08:57:29.755074+0000 mon.a (mon.0) 793 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:57:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[77892]: cluster 2023-10-09T08:57:29.755171+0000 mon.a (mon.0) 794 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-10-09T08:57:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[77892]: cluster 2023-10-09T08:57:29.761026+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[77892]: cluster 2023-10-09T08:57:29.761054+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:30 smithi016 bash[77892]: cluster 2023-10-09T08:57:29.761257+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T08:57:31.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:31.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:31.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:31.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:31.189 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:31.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:31.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:31.318 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:31.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:31.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:31.339 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 99078 99062 70.3445 49.375 0.0120644 0.0142092 2023-10-09T08:57:31.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:31.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:31.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:31.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:31.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:31.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:31.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:31.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:31.693 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:31.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:31.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:31.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:31.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:31.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:31.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:31.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:31.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:32.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:32.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:32.109 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:32.135 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:32.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:32.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:32.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:32.339 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 99860 99844 70.1032 48.875 0.0129088 0.0142583 2023-10-09T08:57:32.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:32.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:32.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:32.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:32.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:32.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:32.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:32.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:32.809 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:32 smithi016 bash[75179]: cluster 2023-10-09T08:57:31.691438+0000 mgr.y (mgr.25421) 599 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T08:57:32.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:32 smithi016 bash[77892]: cluster 2023-10-09T08:57:31.691438+0000 mgr.y (mgr.25421) 599 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T08:57:32.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:32.898 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:57:32.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:32.961 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:57:32.963 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:32.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:33.006 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:57:33.047 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:57:33.069 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:33.078 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:32 smithi094 bash[59677]: cluster 2023-10-09T08:57:31.691438+0000 mgr.y (mgr.25421) 599 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T08:57:33.085 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:57:33.100 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:33.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:33.127 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:57:33.168 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:57:33.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:33.207 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:57:33.210 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.210 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.210 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.211 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.211 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.211 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.211 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.211 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:57:33.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:33.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:33.339 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0131 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 100515 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 69.7919 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 26.369 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 156.75 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 25.0625 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1116 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 421.904 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2508 2023-10-09T08:57:33.340 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 401 2023-10-09T08:57:33.341 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0143233 2023-10-09T08:57:33.341 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0156565 2023-10-09T08:57:33.341 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 1.0974 2023-10-09T08:57:33.341 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00139597 2023-10-09T08:57:33.345 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T08:57:33.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:33.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:33.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:33.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:33.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:33.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:33.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:33.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:33.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:33.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:33.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:33.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:33.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:33.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:33.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:33.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:33.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:33.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:34.041 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:34.049 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:33 smithi094 bash[59677]: audit 2023-10-09T08:57:32.847780+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.16:0/2945119143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:34.049 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:33 smithi094 bash[59677]: audit 2023-10-09T08:57:33.504924+0000 mon.a (mon.0) 795 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:34.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:34.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:34.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:34.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T08:57:34.084 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:33 smithi016 bash[75179]: audit 2023-10-09T08:57:32.847780+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.16:0/2945119143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:34.085 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:33 smithi016 bash[75179]: audit 2023-10-09T08:57:33.504924+0000 mon.a (mon.0) 795 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:34.087 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:33 smithi016 bash[77892]: audit 2023-10-09T08:57:32.847780+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.16:0/2945119143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:34.087 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:33 smithi016 bash[77892]: audit 2023-10-09T08:57:33.504924+0000 mon.a (mon.0) 795 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:34.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:34.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:34.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:34.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:34.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:34.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:34.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:34.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:34.460 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:34.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:34.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:34.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:34.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:34.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:34.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:34.729 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:34.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:34.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:34.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:34.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:34.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:34.988 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:35.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:35.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:35.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:35.221 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:35 smithi094 bash[59677]: cluster 2023-10-09T08:57:33.692098+0000 mgr.y (mgr.25421) 600 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s 2023-10-09T08:57:35.222 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:35 smithi094 bash[59677]: audit 2023-10-09T08:57:34.227050+0000 mon.a (mon.0) 796 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:35.222 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:35 smithi094 bash[59677]: audit 2023-10-09T08:57:34.228632+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:35.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:35.275 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:35.336 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:35.337 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:35.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:35.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:35.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:35 smithi016 bash[75179]: cluster 2023-10-09T08:57:33.692098+0000 mgr.y (mgr.25421) 600 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s 2023-10-09T08:57:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:35 smithi016 bash[75179]: audit 2023-10-09T08:57:34.227050+0000 mon.a (mon.0) 796 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:35 smithi016 bash[75179]: audit 2023-10-09T08:57:34.228632+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:35 smithi016 bash[77892]: cluster 2023-10-09T08:57:33.692098+0000 mgr.y (mgr.25421) 600 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s 2023-10-09T08:57:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:35 smithi016 bash[77892]: audit 2023-10-09T08:57:34.227050+0000 mon.a (mon.0) 796 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:35 smithi016 bash[77892]: audit 2023-10-09T08:57:34.228632+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:35.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:35.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:35.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:35.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:35.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:35.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:35.790 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:57:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:57:35] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T08:57:35.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:35.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:35.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:35.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:36.047 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:36.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:36.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:36.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:36.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:36.302 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:36.326 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:36.431 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:36.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:36.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:36.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:36.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:36.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:36.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:36.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:36.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:36.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:57:36.954 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:36.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:36.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:37.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:37.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:37.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:37.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:37.224 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:37 smithi094 bash[59677]: cluster 2023-10-09T08:57:35.692754+0000 mgr.y (mgr.25421) 601 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 766 op/s 2023-10-09T08:57:37.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:37.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:37.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:37.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:37.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:37 smithi016 bash[77892]: cluster 2023-10-09T08:57:35.692754+0000 mgr.y (mgr.25421) 601 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 766 op/s 2023-10-09T08:57:37.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:37 smithi016 bash[75179]: cluster 2023-10-09T08:57:35.692754+0000 mgr.y (mgr.25421) 601 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 766 op/s 2023-10-09T08:57:37.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:37.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:37.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:37.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:37.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:37.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:37.662 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:37.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:37.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:37.790 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:37.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:37.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:37.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:37.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:37.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:38.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:38.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:38.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:38.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:38.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:38.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:38.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:38.317 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:38.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:38.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:38.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:38.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:38.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:38.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:38.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:38.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:38.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:38.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:38.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:38.738 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:38.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:38.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:38.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:38.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:38.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:38.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:39.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:39.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:39.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:39.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:39.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:39.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:39.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:39.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:39.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:39.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:39.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:39 smithi094 bash[59677]: cluster 2023-10-09T08:57:37.693292+0000 mgr.y (mgr.25421) 602 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 32 MiB/s wr, 743 op/s 2023-10-09T08:57:39.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:39 smithi016 bash[75179]: cluster 2023-10-09T08:57:37.693292+0000 mgr.y (mgr.25421) 602 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 32 MiB/s wr, 743 op/s 2023-10-09T08:57:39.497 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:39 smithi016 bash[77892]: cluster 2023-10-09T08:57:37.693292+0000 mgr.y (mgr.25421) 602 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 32 MiB/s wr, 743 op/s 2023-10-09T08:57:39.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:39.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:39.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:39.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:39.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:39.857 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:39.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:39.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:39.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:40.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:40.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:40.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:40.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:40.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:40.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:40.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:40.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:40.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:40.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:40.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:40.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:40.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:40.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:40.835 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:40.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:40.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:40.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[75179]: cluster 2023-10-09T08:57:39.693699+0000 mgr.y (mgr.25421) 603 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 32 MiB/s wr, 743 op/s 2023-10-09T08:57:40.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[75179]: audit 2023-10-09T08:57:39.769679+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[75179]: audit 2023-10-09T08:57:39.787478+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[75179]: audit 2023-10-09T08:57:40.133296+0000 mon.a (mon.0) 800 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[75179]: audit 2023-10-09T08:57:40.147188+0000 mon.a (mon.0) 801 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.853 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[77892]: cluster 2023-10-09T08:57:39.693699+0000 mgr.y (mgr.25421) 603 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 32 MiB/s wr, 743 op/s 2023-10-09T08:57:40.853 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[77892]: audit 2023-10-09T08:57:39.769679+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.853 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[77892]: audit 2023-10-09T08:57:39.787478+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.853 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[77892]: audit 2023-10-09T08:57:40.133296+0000 mon.a (mon.0) 800 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.853 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:40 smithi016 bash[77892]: audit 2023-10-09T08:57:40.147188+0000 mon.a (mon.0) 801 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.900 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:40.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:40.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:40.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:40 smithi094 bash[59677]: cluster 2023-10-09T08:57:39.693699+0000 mgr.y (mgr.25421) 603 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 32 MiB/s wr, 743 op/s 2023-10-09T08:57:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:40 smithi094 bash[59677]: audit 2023-10-09T08:57:39.769679+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:40 smithi094 bash[59677]: audit 2023-10-09T08:57:39.787478+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:40 smithi094 bash[59677]: audit 2023-10-09T08:57:40.133296+0000 mon.a (mon.0) 800 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:40 smithi094 bash[59677]: audit 2023-10-09T08:57:40.147188+0000 mon.a (mon.0) 801 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:41.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:41.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:41.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:41.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:41.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:41.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:41.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:41.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:41.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:41.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:41.470 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:41.498 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:41.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:41.601 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:41.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:41.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:41.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:41.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:41.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:41.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:41.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:41.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:41.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:41.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:41 smithi016 bash[75179]: audit 2023-10-09T08:57:40.780875+0000 mon.a (mon.0) 802 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:41 smithi016 bash[75179]: audit 2023-10-09T08:57:40.782628+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:41 smithi016 bash[75179]: audit 2023-10-09T08:57:40.794843+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:42.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:41 smithi016 bash[77892]: audit 2023-10-09T08:57:40.780875+0000 mon.a (mon.0) 802 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:42.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:41 smithi016 bash[77892]: audit 2023-10-09T08:57:40.782628+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:42.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:41 smithi016 bash[77892]: audit 2023-10-09T08:57:40.794843+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:42.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:42.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:42.031 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:41 smithi094 bash[59677]: audit 2023-10-09T08:57:40.780875+0000 mon.a (mon.0) 802 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:42.031 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:41 smithi094 bash[59677]: audit 2023-10-09T08:57:40.782628+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:42.031 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:41 smithi094 bash[59677]: audit 2023-10-09T08:57:40.794843+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:57:42.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:42.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:42.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:42.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:42.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:42.405 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:42.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:42.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:42.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:42.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:42.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:42.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:42.793 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:42 smithi016 bash[75179]: cluster 2023-10-09T08:57:41.694233+0000 mgr.y (mgr.25421) 604 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 27 MiB/s wr, 772 op/s 2023-10-09T08:57:42.793 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:42 smithi016 bash[77892]: cluster 2023-10-09T08:57:41.694233+0000 mgr.y (mgr.25421) 604 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 27 MiB/s wr, 772 op/s 2023-10-09T08:57:42.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:42.822 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:42.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:42.925 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:42.934 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:42 smithi094 bash[59677]: cluster 2023-10-09T08:57:41.694233+0000 mgr.y (mgr.25421) 604 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 27 MiB/s wr, 772 op/s 2023-10-09T08:57:42.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:42.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:42.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:43.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:43.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:43.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:43.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:43.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:43.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:43.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:43.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:43.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:43.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:43.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:43.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:43.479 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:43.490 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:43.503 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:43.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:43.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:43.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:43.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:43.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:43.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:43.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:43.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:43.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:43.818 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:57:43.818 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T08:57:43.818 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35034.0:108053 7.d e143/134 7:b03e6f51:::benchmark_data_smithi016_128518_object7535:head v 143'6743, mlcod=143'6741)", 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:57:43.803501+0000", 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.011898906000000001, 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011931084, 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:57:43.819 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803501+0000", 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803501+0000", 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803500+0000", 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803511+0000", 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0589e-05 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:57:43.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803525+0000", 2023-10-09T08:57:43.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4313e-05 2023-10-09T08:57:43.822 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.822 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.822 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:57:43.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803533+0000", 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0770000000000008e-06 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803620+0000", 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6222000000000002e-05 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.823 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.803640+0000", 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0516000000000001e-05 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.35034.0:108047 7.b e143/134)", 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:57:43.804945+0000", 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.010454747, 2023-10-09T08:57:43.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011476340999999999, 2023-10-09T08:57:43.825 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:57:43.825 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-10-09T08:57:43.825 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:57:43.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.825 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.804945+0000", 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.804945+0000", 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:57:43.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.804945+0000", 2023-10-09T08:57:43.827 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:57:43.827 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.827 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.827 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:57:43.827 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.804953+0000", 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9069999999999996e-06 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.804964+0000", 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0190000000000001e-05 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.804971+0000", 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9009999999999997e-06 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.35034.0:108047 7.b e143/134)", 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:57:43.805471+0000", 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0099291229999999998, 2023-10-09T08:57:43.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010979408, 2023-10-09T08:57:43.830 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:57:43.830 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-10-09T08:57:43.831 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:57:43.831 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.831 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:57:43.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.805471+0000", 2023-10-09T08:57:43.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.831 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.832 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.832 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:57:43.832 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.805471+0000", 2023-10-09T08:57:43.832 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.832 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.833 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:57:43.833 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.805470+0000", 2023-10-09T08:57:43.833 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:57:43.833 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:57:43.834 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.805475+0000", 2023-10-09T08:57:43.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7110000000000003e-06 2023-10-09T08:57:43.834 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:57:43.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.805488+0000", 2023-10-09T08:57:43.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2789000000000001e-05 2023-10-09T08:57:43.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:57:43.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.805494+0000", 2023-10-09T08:57:43.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.601e-06 2023-10-09T08:57:43.836 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.837 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:57:43.837 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.838 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35034.0:108046 7.b 7:db8ca1b5:::benchmark_data_smithi016_128518_object7528:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T08:57:43.838 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:57:43.790681+0000", 2023-10-09T08:57:43.838 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.024719142999999999, 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.025802961999999999, 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35034", 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 108046 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:57:43.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.790681+0000", 2023-10-09T08:57:43.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.841 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:57:43.841 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.790681+0000", 2023-10-09T08:57:43.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.841 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:57:43.841 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.790679+0000", 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.790686+0000", 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2389999999999999e-06 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:57:43.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.790697+0000", 2023-10-09T08:57:43.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1765e-05 2023-10-09T08:57:43.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:57:43.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.790705+0000", 2023-10-09T08:57:43.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7570000000000004e-06 2023-10-09T08:57:43.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:57:43.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.796877+0000", 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.006171447 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.797026+0000", 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000149357 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.797174+0000", 2023-10-09T08:57:43.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000147863 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.800910+0000", 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0037362609999999998 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801054+0000", 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000143812 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.846 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35034.0:108047 7.b 7:dc3c6f79:::benchmark_data_smithi016_128518_object7529:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:57:43.796586+0000", 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.018813882000000001, 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.019950206000000002, 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35034", 2023-10-09T08:57:43.847 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 108047 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.796586+0000", 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.796586+0000", 2023-10-09T08:57:43.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.796584+0000", 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.796591+0000", 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0609999999999998e-06 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.796608+0000", 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6259000000000001e-05 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.796616+0000", 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8849999999999999e-06 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:57:43.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801169+0000", 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0045530709999999997 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801593+0000", 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000424523 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801971+0000", 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000377396 2023-10-09T08:57:43.851 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.35034.0:108046 7.b e143/134)", 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:57:43.801228+0000", 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.014172301, 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015348138000000001, 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:57:43.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801228+0000", 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801228+0000", 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801227+0000", 2023-10-09T08:57:43.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801231+0000", 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3490000000000002e-06 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801242+0000", 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0377000000000001e-05 2023-10-09T08:57:43.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:57:43.801248+0000", 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.942e-06 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 6 2023-10-09T08:57:43.855 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:57:43.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:43.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:43.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:44.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:44.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:44.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:44.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:44.188 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:44.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:44.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:44.290 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:44.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:44.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:44.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:44.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:44.467 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:44.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:44.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:44.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:44.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:44.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:44.744 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:44.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:44.799 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:44 smithi016 bash[75179]: cluster 2023-10-09T08:57:43.694676+0000 mgr.y (mgr.25421) 605 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 17 MiB/s wr, 760 op/s 2023-10-09T08:57:44.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:44 smithi016 bash[75179]: audit 2023-10-09T08:57:44.136023+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.16:0/3949162553' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:57:44.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:44 smithi016 bash[75179]: audit 2023-10-09T08:57:44.136991+0000 mon.a (mon.0) 805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:57:44.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:44 smithi016 bash[77892]: cluster 2023-10-09T08:57:43.694676+0000 mgr.y (mgr.25421) 605 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 17 MiB/s wr, 760 op/s 2023-10-09T08:57:44.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:44 smithi016 bash[77892]: audit 2023-10-09T08:57:44.136023+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.16:0/3949162553' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:57:44.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:44 smithi016 bash[77892]: audit 2023-10-09T08:57:44.136991+0000 mon.a (mon.0) 805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:57:44.820 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 7 pgp_num to 16 2023-10-09T08:57:44.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:44.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:44.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:44.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:44.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:44 smithi094 bash[59677]: cluster 2023-10-09T08:57:43.694676+0000 mgr.y (mgr.25421) 605 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 17 MiB/s wr, 760 op/s 2023-10-09T08:57:44.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:44 smithi094 bash[59677]: audit 2023-10-09T08:57:44.136023+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.16:0/3949162553' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:57:44.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:44 smithi094 bash[59677]: audit 2023-10-09T08:57:44.136991+0000 mon.a (mon.0) 805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T08:57:45.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:45.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:45.156 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:45.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:45.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:45.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:45.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:45.415 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:45.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:45.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:45.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:45.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:45.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:45.790 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:57:45] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T08:57:45.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:45.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:45.912 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:45.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:45.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:45.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:45.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:45 smithi094 bash[59677]: audit 2023-10-09T08:57:44.810039+0000 mon.a (mon.0) 806 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T08:57:45.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:45 smithi094 bash[59677]: cluster 2023-10-09T08:57:44.810154+0000 mon.a (mon.0) 807 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-10-09T08:57:45.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:45 smithi094 bash[59677]: cluster 2023-10-09T08:57:44.811732+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:45.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:45 smithi094 bash[59677]: cluster 2023-10-09T08:57:44.813515+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:45.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:45 smithi094 bash[59677]: cluster 2023-10-09T08:57:44.826295+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:46.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:46.055 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:46.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:46.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:46.170 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[75179]: audit 2023-10-09T08:57:44.810039+0000 mon.a (mon.0) 806 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T08:57:46.170 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[75179]: cluster 2023-10-09T08:57:44.810154+0000 mon.a (mon.0) 807 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-10-09T08:57:46.170 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[75179]: cluster 2023-10-09T08:57:44.811732+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:46.171 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[75179]: cluster 2023-10-09T08:57:44.813515+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:46.171 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[75179]: cluster 2023-10-09T08:57:44.826295+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:46.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[77892]: audit 2023-10-09T08:57:44.810039+0000 mon.a (mon.0) 806 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T08:57:46.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[77892]: cluster 2023-10-09T08:57:44.810154+0000 mon.a (mon.0) 807 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-10-09T08:57:46.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[77892]: cluster 2023-10-09T08:57:44.811732+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:46.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[77892]: cluster 2023-10-09T08:57:44.813515+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:46.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:45 smithi016 bash[77892]: cluster 2023-10-09T08:57:44.826295+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T08:57:46.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:46.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:46.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:46.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:46.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:46.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:46.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:46.464 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:46.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:46.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:46.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:46.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:46.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:46.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:46.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:46.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:46.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:46.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:46.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:46 smithi016 bash[75179]: cluster 2023-10-09T08:57:45.695265+0000 mgr.y (mgr.25421) 606 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 720 op/s 2023-10-09T08:57:46.852 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:46 smithi016 bash[77892]: cluster 2023-10-09T08:57:45.695265+0000 mgr.y (mgr.25421) 606 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 720 op/s 2023-10-09T08:57:46.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:46.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:46.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:47.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:47.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:47.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:46 smithi094 bash[59677]: cluster 2023-10-09T08:57:45.695265+0000 mgr.y (mgr.25421) 606 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 3.8 MiB/s wr, 720 op/s 2023-10-09T08:57:47.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.292 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:47.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:47.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:47.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:47.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:47.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:47.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.673 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:47.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:47.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:47.905 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.932 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:47.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:47.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:47.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T08:57:48.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:48.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:48.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:48.176 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:48.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:48.306 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:48.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:48.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:48.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:48.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:48.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:48.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:48.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:48.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:48.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:48.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:48.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:48.859 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:48.962 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:48.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:48 smithi094 bash[59677]: cluster 2023-10-09T08:57:47.695821+0000 mgr.y (mgr.25421) 607 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 563 op/s 2023-10-09T08:57:48.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:48.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:49.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:49.117 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:49.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:49.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:49.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:49.249 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:48 smithi016 bash[75179]: cluster 2023-10-09T08:57:47.695821+0000 mgr.y (mgr.25421) 607 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 563 op/s 2023-10-09T08:57:49.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:48 smithi016 bash[77892]: cluster 2023-10-09T08:57:47.695821+0000 mgr.y (mgr.25421) 607 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 563 op/s 2023-10-09T08:57:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:49.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:49.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:49.380 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:49.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:49.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:49.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:49.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:49.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:49.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:49.655 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:49.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:49.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:49.761 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:49.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:49.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:49.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:49.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:49.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:49.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:49.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:49 smithi094 bash[59677]: audit 2023-10-09T08:57:49.216875+0000 mon.a (mon.0) 808 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:50.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:50.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:50.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:50.168 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:49 smithi016 bash[75179]: audit 2023-10-09T08:57:49.216875+0000 mon.a (mon.0) 808 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:50.169 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:49 smithi016 bash[77892]: audit 2023-10-09T08:57:49.216875+0000 mon.a (mon.0) 808 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:57:50.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:50.293 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:50.318 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:50.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:50.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:50.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:50.550 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:50.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:50.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:50.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:50.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:50.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:50.832 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:50.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:50.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:50.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:51.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:51.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:51.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:51.215 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:51.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:51.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:51.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:51.380 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:51.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:51.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:51.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:51 smithi094 bash[59677]: cluster 2023-10-09T08:57:49.696404+0000 mgr.y (mgr.25421) 608 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 721 op/s 2023-10-09T08:57:51.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:51.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:51.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:51.625 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:51 smithi016 bash[75179]: cluster 2023-10-09T08:57:49.696404+0000 mgr.y (mgr.25421) 608 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 721 op/s 2023-10-09T08:57:51.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:51 smithi016 bash[77892]: cluster 2023-10-09T08:57:49.696404+0000 mgr.y (mgr.25421) 608 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 721 op/s 2023-10-09T08:57:51.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:51.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:51.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:51.754 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:51.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:51.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:51.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:51.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:51.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:51.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:52.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:52.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:52.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:52.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:52.182 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:52.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:52.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:52.288 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:52.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:52.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:52.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:52.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:52.450 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T08:57:52.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:52 smithi094 bash[59677]: audit 2023-10-09T08:57:51.910690+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.16:0/1937541361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:52.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:52 smithi094 bash[59677]: audit 2023-10-09T08:57:51.913791+0000 mon.a (mon.0) 809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:52.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:52.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:52.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:52.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:52.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:52.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:52.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:52.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:52.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:52 smithi016 bash[75179]: audit 2023-10-09T08:57:51.910690+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.16:0/1937541361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:52.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:52 smithi016 bash[75179]: audit 2023-10-09T08:57:51.913791+0000 mon.a (mon.0) 809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:52.727 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:52 smithi016 bash[77892]: audit 2023-10-09T08:57:51.910690+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.16:0/1937541361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:52.727 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:52 smithi016 bash[77892]: audit 2023-10-09T08:57:51.913791+0000 mon.a (mon.0) 809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:57:52.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:52.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:52.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:52.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:52.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:52.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:53.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:53.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:53.136 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:57:53.136 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:57:53.136 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6] 2023-10-09T08:57:53.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T08:57:53.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:53.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:53.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:53.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:53.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:53.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:53.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:53.487 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:53.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:53.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:53.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[75179]: cluster 2023-10-09T08:57:51.698544+0000 mgr.y (mgr.25421) 609 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T08:57:53.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[75179]: cluster 2023-10-09T08:57:52.434344+0000 mon.a (mon.0) 810 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:57:53.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[75179]: cluster 2023-10-09T08:57:52.438079+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T08:57:53.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[75179]: cluster 2023-10-09T08:57:52.440972+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T08:57:53.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[75179]: audit 2023-10-09T08:57:52.442350+0000 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:57:53.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[75179]: cluster 2023-10-09T08:57:52.442609+0000 mon.a (mon.0) 812 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-10-09T08:57:53.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[77892]: cluster 2023-10-09T08:57:51.698544+0000 mgr.y (mgr.25421) 609 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T08:57:53.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[77892]: cluster 2023-10-09T08:57:52.434344+0000 mon.a (mon.0) 810 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:57:53.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[77892]: cluster 2023-10-09T08:57:52.438079+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T08:57:53.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[77892]: cluster 2023-10-09T08:57:52.440972+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T08:57:53.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[77892]: audit 2023-10-09T08:57:52.442350+0000 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:57:53.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:53 smithi016 bash[77892]: cluster 2023-10-09T08:57:52.442609+0000 mon.a (mon.0) 812 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-10-09T08:57:53.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:53.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:53.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:53 smithi094 bash[59677]: cluster 2023-10-09T08:57:51.698544+0000 mgr.y (mgr.25421) 609 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T08:57:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:53 smithi094 bash[59677]: cluster 2023-10-09T08:57:52.434344+0000 mon.a (mon.0) 810 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:57:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:53 smithi094 bash[59677]: cluster 2023-10-09T08:57:52.438079+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T08:57:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:53 smithi094 bash[59677]: cluster 2023-10-09T08:57:52.440972+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T08:57:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:53 smithi094 bash[59677]: audit 2023-10-09T08:57:52.442350+0000 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:57:53.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:53 smithi094 bash[59677]: cluster 2023-10-09T08:57:52.442609+0000 mon.a (mon.0) 812 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-10-09T08:57:53.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:53.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:53.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:53.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:53.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:53.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:54.016 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:54.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:54.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:54.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:54.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:54.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:54.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:54.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:54.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:54.534 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:57:54.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:54.572 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:57:54.577 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:54.618 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:57:54.658 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:57:54.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:54.694 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:57:54.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:54.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:54.743 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:57:54.786 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:57:54.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:54.839 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:57:54.841 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:54.844 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.844 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.844 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.844 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.844 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.844 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.845 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.845 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:57:54.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:54.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:54.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:55.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:55.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:55.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:55.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.243 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:55.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:55.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:55.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:55.458 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:55 smithi016 bash[75179]: cluster 2023-10-09T08:57:53.699048+0000 mgr.y (mgr.25421) 610 : cluster 0 pgmap v586: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T08:57:55.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:55 smithi016 bash[75179]: audit 2023-10-09T08:57:54.463478+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.16:0/327627859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:55.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:55.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:55 smithi094 bash[59677]: cluster 2023-10-09T08:57:53.699048+0000 mgr.y (mgr.25421) 610 : cluster 0 pgmap v586: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T08:57:55.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:55 smithi094 bash[59677]: audit 2023-10-09T08:57:54.463478+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.16:0/327627859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:55.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:55.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:55.620 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.643 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:55.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:55 smithi016 bash[77892]: cluster 2023-10-09T08:57:53.699048+0000 mgr.y (mgr.25421) 610 : cluster 0 pgmap v586: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T08:57:55.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:55 smithi016 bash[77892]: audit 2023-10-09T08:57:54.463478+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.16:0/327627859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:57:55.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:55.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:55.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:55.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.913 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:55.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:55.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:57:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:57:55] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T08:57:56.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:56.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:56.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:56.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:56.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:56.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:56.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:56.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:56.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:56.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:56.367 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:56.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:56.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:56.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:56.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T08:57:56.570 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:56.597 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:56.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:56.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:56.701 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:56.729 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:56.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:56.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:56.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:56.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:56.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:56.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:57.000 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:57.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:57.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:57.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:57.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:57.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:57.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:57.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:57.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:57.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:57.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:57 smithi016 bash[75179]: cluster 2023-10-09T08:57:55.699619+0000 mgr.y (mgr.25421) 611 : cluster 0 pgmap v587: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T08:57:57.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:57 smithi016 bash[77892]: cluster 2023-10-09T08:57:55.699619+0000 mgr.y (mgr.25421) 611 : cluster 0 pgmap v587: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T08:57:57.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:57.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:57.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:57.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:57.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:57 smithi094 bash[59677]: cluster 2023-10-09T08:57:55.699619+0000 mgr.y (mgr.25421) 611 : cluster 0 pgmap v587: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T08:57:57.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:57.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:57.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:57.942 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:57.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:57:57.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:57.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:58.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:58.069 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:57:58.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:58.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:57:58.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:58.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:57:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:58.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:57:58.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:58.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:58.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:58.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:58.460 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:58.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:58.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:58.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:58.592 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:58.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:58.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:58.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:58.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:58.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:57:58.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:58.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:57:58.858 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:58.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:58.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:58.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:57:58.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:59.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:59.136 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:59.163 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:59.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:59.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:59.291 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:59.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:59.397 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:59.424 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:57:59.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:57:59.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:59.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:57:59 smithi016 bash[75179]: cluster 2023-10-09T08:57:57.700291+0000 mgr.y (mgr.25421) 612 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T08:57:59.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:57:59 smithi016 bash[77892]: cluster 2023-10-09T08:57:57.700291+0000 mgr.y (mgr.25421) 612 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T08:57:59.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:57:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:57:59.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:59.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:57:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:57:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:57:59 smithi094 bash[59677]: cluster 2023-10-09T08:57:57.700291+0000 mgr.y (mgr.25421) 612 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T08:57:59.812 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:57:59.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:59.856 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:57:59.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:57:59.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:59.988 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:57:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:57:59.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:00.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:00.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:00.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:00.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:00.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:00.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:00.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:00.366 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:00.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:00.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:00.487 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:00.494 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:00.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:00 smithi016 bash[75179]: cluster 2023-10-09T08:57:59.700926+0000 mgr.y (mgr.25421) 613 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T08:58:00.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:00 smithi016 bash[77892]: cluster 2023-10-09T08:57:59.700926+0000 mgr.y (mgr.25421) 613 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T08:58:00.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:00.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:00.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:00.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:00.643 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:00 smithi094 bash[59677]: cluster 2023-10-09T08:57:59.700926+0000 mgr.y (mgr.25421) 613 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T08:58:00.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:00.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:00.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:00.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:00.793 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:00.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:00.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:00.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:00.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:00.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:00.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:01.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:01.059 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:01.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:01.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:01.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:01.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:01.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:01.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:01.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:01.431 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:01.454 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:01.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:01.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:01.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:01.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:01.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:01.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:01.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:01.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:01.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:01.970 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:01.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:02.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:02.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:02.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:02.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:02.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:02.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:02.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:02.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:02.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:02.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:02.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:02.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:02.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:02.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:02.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:02.513 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:02.601 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:02.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:02.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:02.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:02.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:02.758 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:02.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:02.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:02.874 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:02 smithi016 bash[75179]: cluster 2023-10-09T08:58:01.701842+0000 mgr.y (mgr.25421) 614 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T08:58:02.874 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:02 smithi016 bash[77892]: cluster 2023-10-09T08:58:01.701842+0000 mgr.y (mgr.25421) 614 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T08:58:02.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:02.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:02.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:02 smithi094 bash[59677]: cluster 2023-10-09T08:58:01.701842+0000 mgr.y (mgr.25421) 614 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T08:58:03.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:03.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:03.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:03.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:03.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:03.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:03.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:03.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:03.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:03.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:03.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:03.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:03.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:03.660 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:03.685 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:03.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:03.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:03.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:03.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:03.875 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:58:03.875 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:58:03.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:03.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:03.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:03.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:04.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:04.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:04.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:04.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:04.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:04.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:04.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:04.351 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:04.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:04.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:04.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:04.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:04.592 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:04.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:04.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:04.728 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:04.752 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:04.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:04.854 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:04.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:04 smithi094 bash[59677]: cluster 2023-10-09T08:58:03.702442+0000 mgr.y (mgr.25421) 615 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T08:58:04.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:04 smithi094 bash[59677]: audit 2023-10-09T08:58:04.219797+0000 mon.a (mon.0) 813 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:04.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:04.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:04.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:05.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:05.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:05.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:04 smithi016 bash[75179]: cluster 2023-10-09T08:58:03.702442+0000 mgr.y (mgr.25421) 615 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T08:58:05.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:04 smithi016 bash[75179]: audit 2023-10-09T08:58:04.219797+0000 mon.a (mon.0) 813 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:05.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:04 smithi016 bash[77892]: cluster 2023-10-09T08:58:03.702442+0000 mgr.y (mgr.25421) 615 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T08:58:05.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:04 smithi016 bash[77892]: audit 2023-10-09T08:58:04.219797+0000 mon.a (mon.0) 813 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:05.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:05.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:05.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:05.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:05.406 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:05.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:05.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:05.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:05.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:05.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:05.695 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:05.801 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:05.829 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:05.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:05.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:05.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:05.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:58:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:58:05] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-09T08:58:06.061 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:06.088 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:06.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:06.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:06.220 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:06.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:06.325 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:06.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:06.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:06.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:06.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:06.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:06.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:06.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:06.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:06.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:06.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:06.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:06.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:06.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:06.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:06.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:06.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:06.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:06.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:06.995 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:06 smithi016 bash[75179]: cluster 2023-10-09T08:58:05.703182+0000 mgr.y (mgr.25421) 616 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T08:58:06.996 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:06 smithi016 bash[77892]: cluster 2023-10-09T08:58:05.703182+0000 mgr.y (mgr.25421) 616 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T08:58:07.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:07.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:07.138 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:06 smithi094 bash[59677]: cluster 2023-10-09T08:58:05.703182+0000 mgr.y (mgr.25421) 616 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T08:58:07.157 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:07.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:07.291 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:07.394 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:07.426 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:07.529 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:07.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:07.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:07.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:07.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:07.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:07.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:07.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:07.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:07.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:07.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:07.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:08.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:08.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:08.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:08.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:08.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:08.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:08.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:08.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:08.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:08.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:08.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:08.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:08.493 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:08.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:08.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:08.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T08:58:08.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:08.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:08.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:08.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:08.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:08.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:08.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:08.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:08.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:08.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:08.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:08 smithi094 bash[59677]: cluster 2023-10-09T08:58:07.703647+0000 mgr.y (mgr.25421) 617 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T08:58:08.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:08 smithi016 bash[75179]: cluster 2023-10-09T08:58:07.703647+0000 mgr.y (mgr.25421) 617 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T08:58:09.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:08 smithi016 bash[77892]: cluster 2023-10-09T08:58:07.703647+0000 mgr.y (mgr.25421) 617 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T08:58:09.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:09.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:09.145 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:09.173 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:09.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:09.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:09.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:09.421 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:09.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:09.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:09.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:09.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:09.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:09.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:09.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:09.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:09.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:09.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:09.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:09.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:09.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:09.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:10.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:10.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:10.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:10.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:10.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:10.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:10.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:10.394 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:10.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:10.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:10.526 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:10.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:10.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:10.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:10.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:10.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:10.798 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:10.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:10.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:10.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:10.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:10.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:11.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:11.059 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:11.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:11.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:11.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:11.284 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:11.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:11.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:11.413 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:11.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:11.486 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:11 smithi016 bash[75179]: cluster 2023-10-09T08:58:09.704210+0000 mgr.y (mgr.25421) 618 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T08:58:11.546 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:11.556 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:11 smithi094 bash[59677]: cluster 2023-10-09T08:58:09.704210+0000 mgr.y (mgr.25421) 618 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T08:58:11.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:11.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:11.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:11.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:11.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:11 smithi016 bash[77892]: cluster 2023-10-09T08:58:09.704210+0000 mgr.y (mgr.25421) 618 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T08:58:11.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:11.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:11.946 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:11.971 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:11.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:12.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:12.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:12.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:12.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:12.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:12.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:12.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:12.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:12.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:12.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:12.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:12.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:12.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:12.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:12.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:12.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:12.780 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:12.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:12.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:12.910 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:12.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:12.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:13.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:13.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:13.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:13.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:13.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:13.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:13.552 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:13.577 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:13.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:13.689 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:13 smithi016 bash[75179]: cluster 2023-10-09T08:58:11.704845+0000 mgr.y (mgr.25421) 619 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 731 op/s 2023-10-09T08:58:13.689 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:13 smithi016 bash[77892]: cluster 2023-10-09T08:58:11.704845+0000 mgr.y (mgr.25421) 619 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 731 op/s 2023-10-09T08:58:13.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:13.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:13 smithi094 bash[59677]: cluster 2023-10-09T08:58:11.704845+0000 mgr.y (mgr.25421) 619 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 731 op/s 2023-10-09T08:58:13.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:13.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:13.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:14.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:14.113 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:14.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:14.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:14.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:14.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:14.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:14.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:14.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:14.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:14.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:14.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:14.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:14.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:14.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:14.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:14 smithi016 bash[75179]: debug 2023-10-09T08:58:14.473+0000 7fa046a4d700 -1 mon.a@0(leader).osd e145 definitely_dead 0 2023-10-09T08:58:14.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:14.808 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:14.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:14.912 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:14.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:14.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:15.041 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:15.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:15.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:15.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:15.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:15.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:15.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:15.347 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:15.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:15.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:15.456 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.4. 2023-10-09T08:58:15.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:15.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:15.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[75179]: cluster 2023-10-09T08:58:13.705388+0000 mgr.y (mgr.25421) 620 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T08:58:15.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[75179]: audit 2023-10-09T08:58:14.476501+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.16:0/4022939903' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T08:58:15.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[75179]: audit 2023-10-09T08:58:14.477412+0000 mon.a (mon.0) 814 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T08:58:15.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[75179]: cluster 2023-10-09T08:58:14.477502+0000 mon.a (mon.0) 815 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T08:58:15.478 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[77892]: cluster 2023-10-09T08:58:13.705388+0000 mgr.y (mgr.25421) 620 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T08:58:15.478 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[77892]: audit 2023-10-09T08:58:14.476501+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.16:0/4022939903' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T08:58:15.478 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[77892]: audit 2023-10-09T08:58:14.477412+0000 mon.a (mon.0) 814 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T08:58:15.478 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[77892]: cluster 2023-10-09T08:58:14.477502+0000 mon.a (mon.0) 815 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T08:58:15.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:15.586 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:15 smithi094 bash[59677]: cluster 2023-10-09T08:58:13.705388+0000 mgr.y (mgr.25421) 620 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T08:58:15.586 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:15 smithi094 bash[59677]: audit 2023-10-09T08:58:14.476501+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.16:0/4022939903' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T08:58:15.586 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:15 smithi094 bash[59677]: audit 2023-10-09T08:58:14.477412+0000 mon.a (mon.0) 814 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T08:58:15.586 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:15 smithi094 bash[59677]: cluster 2023-10-09T08:58:14.477502+0000 mon.a (mon.0) 815 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T08:58:15.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:15.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:15.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:15.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:15.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:15.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:15.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:15.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:15.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:15.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:15.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:15.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:58:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:58:15] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-09T08:58:16.001 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:16.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:16.105 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:16.135 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:16.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:16.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:16.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:16.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:16.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:16.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:16.490 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T08:58:16.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:16.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:16.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:16.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:16.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.450195+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: audit 2023-10-09T08:58:15.452271+0000 mon.a (mon.0) 816 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.452326+0000 mon.a (mon.0) 817 : cluster 0 osdmap e146: 8 total, 8 up, 6 in 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.475274+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.475355+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.481230+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.481446+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.485710+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: audit 2023-10-09T08:58:15.884680+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.16:0/3831862251' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:58:16.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[75179]: audit 2023-10-09T08:58:15.885205+0000 mon.a (mon.0) 818 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.450195+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: audit 2023-10-09T08:58:15.452271+0000 mon.a (mon.0) 816 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.452326+0000 mon.a (mon.0) 817 : cluster 0 osdmap e146: 8 total, 8 up, 6 in 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.475274+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.475355+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.481230+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.481446+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.485710+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: audit 2023-10-09T08:58:15.884680+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.16:0/3831862251' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:58:16.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:16 smithi016 bash[77892]: audit 2023-10-09T08:58:15.885205+0000 mon.a (mon.0) 818 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:58:16.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:16.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:16.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.450195+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: audit 2023-10-09T08:58:15.452271+0000 mon.a (mon.0) 816 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T08:58:16.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.452326+0000 mon.a (mon.0) 817 : cluster 0 osdmap e146: 8 total, 8 up, 6 in 2023-10-09T08:58:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.475274+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.475355+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.481230+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.481446+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.485710+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T08:58:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: audit 2023-10-09T08:58:15.884680+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.16:0/3831862251' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:58:16.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:16 smithi094 bash[59677]: audit 2023-10-09T08:58:15.885205+0000 mon.a (mon.0) 818 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T08:58:16.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:16.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:16.898 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:16.933 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:16.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:16.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:17.036 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:17.068 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:17.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:17.171 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:17.203 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:17.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:17.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:17.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:17.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:17.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:17.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:17.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:17.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:17.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:17.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:15.706048+0000 mgr.y (mgr.25421) 621 : cluster 0 pgmap v598: 89 pgs: 7 peering, 82 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.456034+0000 mon.a (mon.0) 819 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.456672+0000 mon.a (mon.0) 820 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.463867+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.467566+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: audit 2023-10-09T08:58:16.468747+0000 mon.a (mon.0) 821 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.468859+0000 mon.a (mon.0) 822 : cluster 0 osdmap e147: 8 total, 8 up, 6 in 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.470390+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.471667+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.602 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.474493+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.602 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.476662+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.602 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.478819+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.602 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:17 smithi094 bash[59677]: cluster 2023-10-09T08:58:16.480887+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:17.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:17.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:17.679 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:58:17.716 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:58:17.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:17.737 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:58:17.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:17.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:17.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:15.706048+0000 mgr.y (mgr.25421) 621 : cluster 0 pgmap v598: 89 pgs: 7 peering, 82 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T08:58:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.456034+0000 mon.a (mon.0) 819 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T08:58:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.456672+0000 mon.a (mon.0) 820 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:58:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.463867+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.467566+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: audit 2023-10-09T08:58:16.468747+0000 mon.a (mon.0) 821 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:58:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.468859+0000 mon.a (mon.0) 822 : cluster 0 osdmap e147: 8 total, 8 up, 6 in 2023-10-09T08:58:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.470390+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.471667+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.474493+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.476662+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.478819+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[77892]: cluster 2023-10-09T08:58:16.480887+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:15.706048+0000 mgr.y (mgr.25421) 621 : cluster 0 pgmap v598: 89 pgs: 7 peering, 82 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T08:58:17.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.456034+0000 mon.a (mon.0) 819 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T08:58:17.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.456672+0000 mon.a (mon.0) 820 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:58:17.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.463867+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.467566+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: audit 2023-10-09T08:58:16.468747+0000 mon.a (mon.0) 821 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T08:58:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.468859+0000 mon.a (mon.0) 822 : cluster 0 osdmap e147: 8 total, 8 up, 6 in 2023-10-09T08:58:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.470390+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.471667+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.474493+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.476662+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.478819+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.754 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:17 smithi016 bash[75179]: cluster 2023-10-09T08:58:16.480887+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T08:58:17.777 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:58:17.806 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:58:17.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:17.866 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:58:17.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:17.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:17.893 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:58:17.943 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:58:17.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:58:17.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:18.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:18.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:18.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:18.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:18.281 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:18.384 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:18.408 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:18.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:18.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:18.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:18.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:18.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:18.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.476646+0000 mon.a (mon.0) 823 : cluster 0 osdmap e148: 8 total, 8 up, 6 in 2023-10-09T08:58:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.477999+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T08:58:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.482889+0000 osd.0 (osd.0) 2 : cluster 0 7.9 starting backfill to osd.6 from (0'0,0'0] MAX to 145'7946 2023-10-09T08:58:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.483732+0000 osd.6 (osd.6) 70 : cluster 0 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 145'7975 2023-10-09T08:58:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.491722+0000 osd.0 (osd.0) 3 : cluster 0 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 145'8119 2023-10-09T08:58:18.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.528692+0000 osd.0 (osd.0) 4 : cluster 0 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 145'8076 2023-10-09T08:58:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.538014+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T08:58:18.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:18 smithi094 bash[59677]: audit 2023-10-09T08:58:17.636990+0000 mon.a (mon.0) 824 : audit 0 from='client.? 172.21.15.16:0/1752024721' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:58:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.476646+0000 mon.a (mon.0) 823 : cluster 0 osdmap e148: 8 total, 8 up, 6 in 2023-10-09T08:58:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.477999+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T08:58:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.482889+0000 osd.0 (osd.0) 2 : cluster 0 7.9 starting backfill to osd.6 from (0'0,0'0] MAX to 145'7946 2023-10-09T08:58:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.483732+0000 osd.6 (osd.6) 70 : cluster 0 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 145'7975 2023-10-09T08:58:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.491722+0000 osd.0 (osd.0) 3 : cluster 0 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 145'8119 2023-10-09T08:58:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.528692+0000 osd.0 (osd.0) 4 : cluster 0 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 145'8076 2023-10-09T08:58:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.538014+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T08:58:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[75179]: audit 2023-10-09T08:58:17.636990+0000 mon.a (mon.0) 824 : audit 0 from='client.? 172.21.15.16:0/1752024721' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:58:18.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.476646+0000 mon.a (mon.0) 823 : cluster 0 osdmap e148: 8 total, 8 up, 6 in 2023-10-09T08:58:18.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.477999+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T08:58:18.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.482889+0000 osd.0 (osd.0) 2 : cluster 0 7.9 starting backfill to osd.6 from (0'0,0'0] MAX to 145'7946 2023-10-09T08:58:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.483732+0000 osd.6 (osd.6) 70 : cluster 0 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 145'7975 2023-10-09T08:58:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.491722+0000 osd.0 (osd.0) 3 : cluster 0 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 145'8119 2023-10-09T08:58:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.528692+0000 osd.0 (osd.0) 4 : cluster 0 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 145'8076 2023-10-09T08:58:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.538014+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T08:58:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:18 smithi016 bash[77892]: audit 2023-10-09T08:58:17.636990+0000 mon.a (mon.0) 824 : audit 0 from='client.? 172.21.15.16:0/1752024721' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:58:18.772 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:18.796 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:18.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:18.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:18.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:18.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:19.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:19.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:19.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:19.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:19.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:19.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:19.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:19.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:19.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:19.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:19.464 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:19.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:19 smithi016 bash[75179]: cluster 2023-10-09T08:58:17.706618+0000 mgr.y (mgr.25421) 622 : cluster 0 pgmap v601: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 575 op/s 2023-10-09T08:58:19.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:19 smithi016 bash[75179]: audit 2023-10-09T08:58:19.221922+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:19.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:19 smithi016 bash[75179]: audit 2023-10-09T08:58:19.223198+0000 mon.a (mon.0) 826 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:19.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:19 smithi016 bash[77892]: cluster 2023-10-09T08:58:17.706618+0000 mgr.y (mgr.25421) 622 : cluster 0 pgmap v601: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 575 op/s 2023-10-09T08:58:19.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:19 smithi016 bash[77892]: audit 2023-10-09T08:58:19.221922+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:19.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:19 smithi016 bash[77892]: audit 2023-10-09T08:58:19.223198+0000 mon.a (mon.0) 826 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:19.498 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:19.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:19.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:19.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:19.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:19.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:19.740 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:19 smithi094 bash[59677]: cluster 2023-10-09T08:58:17.706618+0000 mgr.y (mgr.25421) 622 : cluster 0 pgmap v601: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 575 op/s 2023-10-09T08:58:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:19 smithi094 bash[59677]: audit 2023-10-09T08:58:19.221922+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:19 smithi094 bash[59677]: audit 2023-10-09T08:58:19.223198+0000 mon.a (mon.0) 826 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:19.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:19.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:19.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:19.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:19.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:20.000 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:20.020 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:20.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:20.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:20.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:20.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:20.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:20.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:20.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:20.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:20.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:20.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:20.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:20.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:20.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:20.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T08:58:20.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:20.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:20.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:20.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:20.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:20.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:20.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:20.814 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:20 smithi016 bash[75179]: cluster 2023-10-09T08:58:19.707323+0000 mgr.y (mgr.25421) 623 : cluster 0 pgmap v602: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 15 peering, 1 active+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 423 op/s; 6299/212169 objects degraded (2.969%); 13201/212169 objects misplaced (6.222%) 2023-10-09T08:58:20.814 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:20 smithi016 bash[77892]: cluster 2023-10-09T08:58:19.707323+0000 mgr.y (mgr.25421) 623 : cluster 0 pgmap v602: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 15 peering, 1 active+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 423 op/s; 6299/212169 objects degraded (2.969%); 13201/212169 objects misplaced (6.222%) 2023-10-09T08:58:20.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:20.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:20.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:20.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:20.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:20.989 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:20 smithi094 bash[59677]: cluster 2023-10-09T08:58:19.707323+0000 mgr.y (mgr.25421) 623 : cluster 0 pgmap v602: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 15 peering, 1 active+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 423 op/s; 6299/212169 objects degraded (2.969%); 13201/212169 objects misplaced (6.222%) 2023-10-09T08:58:21.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:21.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:21.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:21.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:21.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:21.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:21.379 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:21.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:21.509 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:21.533 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:21.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:21.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:58:21.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:21.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:21.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:21 smithi094 bash[59677]: cluster 2023-10-09T08:58:20.488559+0000 mon.a (mon.0) 827 : cluster 3 Health check failed: Degraded data redundancy: 6299/212169 objects degraded (2.969%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:21.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:21 smithi016 bash[75179]: cluster 2023-10-09T08:58:20.488559+0000 mon.a (mon.0) 827 : cluster 3 Health check failed: Degraded data redundancy: 6299/212169 objects degraded (2.969%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:21 smithi016 bash[77892]: cluster 2023-10-09T08:58:20.488559+0000 mon.a (mon.0) 827 : cluster 3 Health check failed: Degraded data redundancy: 6299/212169 objects degraded (2.969%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:21.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:21.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:21.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:21.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:21.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:21.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:22.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:22.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:22.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:22.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:22.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:22.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:22.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:22.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:22.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:22.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:22.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:22.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:22.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:22.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:22.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:22.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:22.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:22.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:22.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:22.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:22 smithi016 bash[77892]: cluster 2023-10-09T08:58:21.708019+0000 mgr.y (mgr.25421) 624 : cluster 0 pgmap v603: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 6347/209679 objects degraded (3.027%); 21312/209679 objects misplaced (10.164%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T08:58:22.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:22 smithi016 bash[75179]: cluster 2023-10-09T08:58:21.708019+0000 mgr.y (mgr.25421) 624 : cluster 0 pgmap v603: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 6347/209679 objects degraded (3.027%); 21312/209679 objects misplaced (10.164%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T08:58:22.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:22 smithi094 bash[59677]: cluster 2023-10-09T08:58:21.708019+0000 mgr.y (mgr.25421) 624 : cluster 0 pgmap v603: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 6347/209679 objects degraded (3.027%); 21312/209679 objects misplaced (10.164%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T08:58:22.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:22.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:22.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:22.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:22.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:22.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:23.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:23.040 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:23.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:23.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:23.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:23.145 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6] out_osds: [7, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:58:23.145 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:58:23.146 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T08:58:23.146 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T08:58:23.146 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T08:58:23.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T08:58:23.168 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:23.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:23.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:23.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:23.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:23.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:23.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:23.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:23.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:23.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:23.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:23.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:23.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:23.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:23 smithi094 bash[59677]: cluster 2023-10-09T08:58:22.497607+0000 mon.a (mon.0) 828 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 8 pgs peering) 2023-10-09T08:58:23.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:23.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:23.845 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:23.864 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:23 smithi016 bash[75179]: cluster 2023-10-09T08:58:22.497607+0000 mon.a (mon.0) 828 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 8 pgs peering) 2023-10-09T08:58:23.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:23 smithi016 bash[77892]: cluster 2023-10-09T08:58:22.497607+0000 mon.a (mon.0) 828 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 8 pgs peering) 2023-10-09T08:58:23.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:23.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:23.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:23.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:24.001 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:24.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:24.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:24.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:24.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:24.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:24.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:24.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:24.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:24.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:24.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:24.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:24.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:24.821 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:24.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:24.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:24.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:24.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:24.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:25.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:25.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:25.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:25.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:25.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:25.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:25.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:25.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:25.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:25.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:25.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:25.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:25.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:25.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:25.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:25.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:25.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:25.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:25.740 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:25 smithi016 bash[75179]: cluster 2023-10-09T08:58:23.708744+0000 mgr.y (mgr.25421) 625 : cluster 0 pgmap v604: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 153/205482 objects degraded (0.074%); 25046/205482 objects misplaced (12.189%); 3.5 MiB/s, 57 objects/s recovering 2023-10-09T08:58:25.742 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:58:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:58:25] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.33.4" 2023-10-09T08:58:25.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:25 smithi016 bash[77892]: cluster 2023-10-09T08:58:23.708744+0000 mgr.y (mgr.25421) 625 : cluster 0 pgmap v604: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 153/205482 objects degraded (0.074%); 25046/205482 objects misplaced (12.189%); 3.5 MiB/s, 57 objects/s recovering 2023-10-09T08:58:25.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:25 smithi094 bash[59677]: cluster 2023-10-09T08:58:23.708744+0000 mgr.y (mgr.25421) 625 : cluster 0 pgmap v604: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 153/205482 objects degraded (0.074%); 25046/205482 objects misplaced (12.189%); 3.5 MiB/s, 57 objects/s recovering 2023-10-09T08:58:25.763 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:25.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:25.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:25.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:25.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:25.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:25.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:26.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:26.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:26.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:26.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:26.130 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:26.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:26.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:26.171 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:58:26.171 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T08:58:26.171 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T08:58:26.171 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T08:58:26.171 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.171 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:71324 6.4 6:20004afc:::benchmark_data_smithi016_105291_object71323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e104)", 2023-10-09T08:58:26.171 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:32.448505+0000", 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 413.71673376699999, 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.350809377, 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 71324 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.172 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.448505+0000", 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.448505+0000", 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.448504+0000", 2023-10-09T08:58:26.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.448530+0000", 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6191e-05 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.448543+0000", 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2714000000000001e-05 2023-10-09T08:58:26.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.448551+0000", 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7169999999999997e-06 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.449630+0000", 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010786859999999999 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.449654+0000", 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4153999999999999e-05 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.430181+0000", 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.98052657600000004 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.430198+0000", 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7591000000000001e-05 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.773536+0000", 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.343337872 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.773787+0000", 2023-10-09T08:58:26.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025076900000000002 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.773945+0000", 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000158269 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.781130+0000", 2023-10-09T08:58:26.178 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.007184607 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.787182+0000", 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0060523139999999996 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.796028+0000", 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0088457729999999995 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.796124+0000", 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5686999999999994e-05 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.799315+0000", 2023-10-09T08:58:26.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0031909379999999999 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:71328 6.4 6:2c5a2def:::benchmark_data_smithi016_105291_object71327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e104)", 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:32.450365+0000", 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 413.71487407400002, 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3514623659999998, 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.182 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 71328 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450365+0000", 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450365+0000", 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450364+0000", 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450387+0000", 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3119999999999999e-05 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450402+0000", 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4674999999999999e-05 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450409+0000", 2023-10-09T08:58:26.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7579999999999992e-06 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450464+0000", 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4301000000000001e-05 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.450474+0000", 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0567000000000001e-05 2023-10-09T08:58:26.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.432849+0000", 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.98237455299999998 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.432866+0000", 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7513e-05 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.778069+0000", 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.345202558 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.778214+0000", 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014546200000000001 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:58:26.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.778363+0000", 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014876399999999999 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.787248+0000", 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0088851959999999997 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.787438+0000", 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018995700000000001 2023-10-09T08:58:26.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.799451+0000", 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012012851999999999 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.799549+0000", 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8299000000000002e-05 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.801827+0000", 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022780529999999999 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:71356 6.4 6:2e878a9d:::benchmark_data_smithi016_105291_object71355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e104)", 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:32.478872+0000", 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 413.68636748300003, 2023-10-09T08:58:26.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3369140549999998, 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 71356 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478872+0000", 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478872+0000", 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478871+0000", 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:58:26.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478894+0000", 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3771e-05 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478904+0000", 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9499999999999996e-06 2023-10-09T08:58:26.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478912+0000", 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8699999999999992e-06 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478970+0000", 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7294999999999999e-05 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.478979+0000", 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8460000000000003e-06 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.432948+0000", 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.95396840000000005 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.432954+0000", 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5610000000000004e-06 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.781164+0000", 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3482098819999999 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.781304+0000", 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000140094 2023-10-09T08:58:26.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.781473+0000", 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016812500000000001 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.787411+0000", 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0059389239999999999 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.787457+0000", 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5495999999999999e-05 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.812776+0000", 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.025319355000000002 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.812892+0000", 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000115454 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.815786+0000", 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028939009999999999 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.200 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:71384 6.4 6:2af2546d:::benchmark_data_smithi016_105291_object71383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e104)", 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:51:32.513202+0000", 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 413.65203722299998, 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3148360889999999, 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 71384 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513202+0000", 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513202+0000", 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513200+0000", 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513224+0000", 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3881000000000001e-05 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513242+0000", 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7609e-05 2023-10-09T08:58:26.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513250+0000", 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9340000000000006e-06 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513316+0000", 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6847000000000006e-05 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:32.513330+0000", 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3583e-05 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.432981+0000", 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.91965122899999996 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:33.432992+0000", 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0719e-05 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.783521+0000", 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3505290990000001 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.783897+0000", 2023-10-09T08:58:26.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00037612700000000003 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.784437+0000", 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00053989200000000004 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.789704+0000", 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.005267107 2023-10-09T08:58:26.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.789727+0000", 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2436e-05 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.822932+0000", 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.033205819999999997 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.822981+0000", 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8659000000000003e-05 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:51:34.828038+0000", 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0050569040000000001 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:160490 6.d 6:bf8d29fa:::benchmark_data_smithi016_105291_object59363:head [delete] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:52.403362+0000", 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 273.76187727799999, 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.049496821, 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.210 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 160490 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403362+0000", 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403362+0000", 2023-10-09T08:58:26.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403359+0000", 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403371+0000", 2023-10-09T08:58:26.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1649e-05 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403388+0000", 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7371e-05 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403404+0000", 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5379e-05 2023-10-09T08:58:26.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403661+0000", 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025782099999999998 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.403684+0000", 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2027000000000001e-05 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.414527+0000", 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0108431360000001 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.419160+0000", 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0046338029999999997 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-10-09T08:58:26.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.419552+0000", 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000391171 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.441222+0000", 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021670657999999999 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452606+0000", 2023-10-09T08:58:26.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011383248 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452765+0000", 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015909 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452819+0000", 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4880000000000003e-05 2023-10-09T08:58:26.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452859+0000", 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9124e-05 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:160527 6.d 6:b414fb63:::benchmark_data_smithi016_105291_object59400:head [delete] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:52.484178+0000", 2023-10-09T08:58:26.218 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 273.68106135199997, 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.96881788899999999, 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 160527 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484178+0000", 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484178+0000", 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484176+0000", 2023-10-09T08:58:26.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484186+0000", 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9199999999999999e-06 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484207+0000", 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1525000000000001e-05 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484219+0000", 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2277e-05 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484280+0000", 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0965000000000003e-05 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.222 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.484291+0000", 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0385e-05 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.441253+0000", 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.95696188000000004 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.441350+0000", 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7243000000000004e-05 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.441464+0000", 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000114015 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452712+0000", 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011248381999999999 2023-10-09T08:58:26.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452873+0000", 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016031800000000001 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452919+0000", 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6173e-05 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452961+0000", 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2676999999999998e-05 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452996+0000", 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4171999999999997e-05 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:160529 6.d 6:be54c741:::benchmark_data_smithi016_105291_object59402:head [delete] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:52.493592+0000", 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 273.67164666600002, 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.97109383299999996, 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 160529 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.227 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493592+0000", 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493592+0000", 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493590+0000", 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493601+0000", 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0117e-05 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493625+0000", 2023-10-09T08:58:26.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4151e-05 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493638+0000", 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.308e-05 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493702+0000", 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3844999999999996e-05 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:52.493710+0000", 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4910000000000005e-06 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.444798+0000", 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.95108795499999998 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.444947+0000", 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014878299999999999 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.445082+0000", 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013549699999999999 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.452878+0000", 2023-10-09T08:58:26.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0077952680000000002 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.459979+0000", 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0071014030000000001 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.464576+0000", 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0045968820000000001 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.464631+0000", 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5155999999999998e-05 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:53.464686+0000", 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5133000000000002e-05 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163858 6.1 6:83e6e4b0:::benchmark_data_smithi016_105291_object62731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.869043+0000", 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.29619568700002, 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.318889435, 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.235 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163858 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869043+0000", 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869043+0000", 2023-10-09T08:58:26.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869041+0000", 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869049+0000", 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4580000000000004e-06 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869062+0000", 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2798e-05 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869069+0000", 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.763e-06 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869143+0000", 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3749000000000002e-05 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869357+0000", 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021367700000000001 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 6,7", 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.869530+0000", 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017322999999999999 2023-10-09T08:58:26.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.877833+0000", 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0083031119999999996 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.881937+0000", 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0041039240000000001 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184634+0000", 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.302697353 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184702+0000", 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7854999999999993e-05 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.187933+0000", 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0032305260000000001 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163863 6.1 6:86fe2f93:::benchmark_data_smithi016_105291_object62736:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.872870+0000", 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.29236943799998, 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31756042899999998, 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.242 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163863 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.872870+0000", 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.872870+0000", 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.872868+0000", 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.872879+0000", 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1072e-05 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.872920+0000", 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0951000000000001e-05 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.872933+0000", 2023-10-09T08:58:26.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3839000000000001e-05 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.873388+0000", 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00045473099999999999 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.873563+0000", 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017523899999999999 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 6,7", 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.873739+0000", 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017613399999999999 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.878045+0000", 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043052890000000003 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.881994+0000", 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039492269999999996 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.188012+0000", 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30601756299999999 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.188073+0000", 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1457999999999993e-05 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.190430+0000", 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0023570330000000001 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163867 6.1 6:8f645c9f:::benchmark_data_smithi016_105291_object62740:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.885867+0000", 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.27937164999997, 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30680696800000001, 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163867 2023-10-09T08:58:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.885867+0000", 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.885867+0000", 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.885866+0000", 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.885872+0000", 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9490000000000001e-06 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.885885+0000", 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2150000000000001e-05 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.885893+0000", 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3240000000000003e-06 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.885938+0000", 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5550000000000003e-05 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.886097+0000", 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000158391 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 6,7", 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.886257+0000", 2023-10-09T08:58:26.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016024299999999999 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.889494+0000", 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0032373879999999999 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.890710+0000", 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001215697 2023-10-09T08:58:26.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.190483+0000", 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29977322000000001 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.190530+0000", 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6638999999999998e-05 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.192674+0000", 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021444250000000002 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163878 6.d 6:b0f22552:::benchmark_data_smithi016_105291_object62751:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.903712+0000", 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.261527441, 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28129084100000001, 2023-10-09T08:58:26.257 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163878 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903712+0000", 2023-10-09T08:58:26.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903712+0000", 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903710+0000", 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903719+0000", 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8689999999999993e-06 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903735+0000", 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6546999999999999e-05 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903746+0000", 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0519e-05 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903808+0000", 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2217000000000004e-05 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.903966+0000", 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015773600000000001 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904093+0000", 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012725900000000001 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184649+0000", 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28055609399999998 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184687+0000", 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7719999999999998e-05 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184903+0000", 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021636 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184947+0000", 2023-10-09T08:58:26.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3532999999999998e-05 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.185002+0000", 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5915999999999998e-05 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:163877 6.b e116/95 6:d9759120:::benchmark_data_smithi016_105291_object62750:head v 116'10236, mlcod=116'10234)", 2023-10-09T08:58:26.264 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.904035+0000", 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.261204319, 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28054212000000001, 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904035+0000", 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904035+0000", 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904034+0000", 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904042+0000", 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2090000000000005e-06 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904053+0000", 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1233000000000001e-05 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904061+0000", 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7970000000000001e-06 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904153+0000", 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1413e-05 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.904178+0000", 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5369000000000001e-05 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184511+0000", 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28033335500000001 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184577+0000", 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5497e-05 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:163880 6.4 6:256a8b8f:::benchmark_data_smithi016_105291_object62753:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.906858+0000", 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.25838133100001, 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29724886, 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 163880 2023-10-09T08:58:26.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.906858+0000", 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.906858+0000", 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.906856+0000", 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.906865+0000", 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9120000000000004e-06 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.906879+0000", 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4346e-05 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.906887+0000", 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3909999999999998e-06 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.906942+0000", 2023-10-09T08:58:26.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4985999999999998e-05 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.907061+0000", 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011859899999999999 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.907159+0000", 2023-10-09T08:58:26.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7712000000000002e-05 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.192533+0000", 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28537446300000002 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.192594+0000", 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1137999999999996e-05 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.203621+0000", 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011026393000000001 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.203678+0000", 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6937000000000001e-05 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.204107+0000", 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042901899999999999 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.44690.0:163858 6.1 e116/108 ondisk, result = 0)", 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.908008+0000", 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.25723066, 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.279953271, 2023-10-09T08:58:26.277 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.908008+0000", 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.908008+0000", 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.908007+0000", 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.908014+0000", 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9210000000000003e-06 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184522+0000", 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27650787900000001 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184534+0000", 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2097e-05 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184607+0000", 2023-10-09T08:58:26.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3107000000000003e-05 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184623+0000", 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5954e-05 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.187962+0000", 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0033385770000000001 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.281 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:163882 6.b e116/95 6:de953810:::benchmark_data_smithi016_105291_object62755:head v 116'10237, mlcod=116'10234)", 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:53:58.909946+0000", 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 267.25529336400001, 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29125302800000002, 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.282 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909946+0000", 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909946+0000", 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909945+0000", 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:58.909954+0000", 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2499999999999995e-06 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184493+0000", 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27453916900000003 2023-10-09T08:58:26.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184513+0000", 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9791000000000001e-05 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184621+0000", 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000108009 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.184651+0000", 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9969999999999999e-05 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201156+0000", 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016504959999999999 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:53:59.201199+0000", 2023-10-09T08:58:26.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2592999999999997e-05 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:178439 6.1 6:88c66dc2:::benchmark_data_smithi016_105291_object77312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:54:27.869418+0000", 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 238.29582069899999, 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31044587299999998, 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.287 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 178439 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869418+0000", 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869418+0000", 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869417+0000", 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869425+0000", 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6859999999999995e-06 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869440+0000", 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4844000000000001e-05 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869449+0000", 2023-10-09T08:58:26.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1759999999999999e-06 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.879210+0000", 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0097604739999999999 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.879356+0000", 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014636200000000001 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.879523+0000", 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000166441 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.888011+0000", 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0084881669999999996 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.892712+0000", 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0047009299999999999 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.897514+0000", 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048022680000000002 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.178708+0000", 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.281194043 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.178771+0000", 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3532000000000001e-05 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.179864+0000", 2023-10-09T08:58:26.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010927580000000001 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:178441 6.1 6:868191b9:::benchmark_data_smithi016_105291_object77314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:54:27.869457+0000", 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 238.29578219800001, 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31138808400000001, 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.295 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 178441 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869457+0000", 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869457+0000", 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869456+0000", 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869459+0000", 2023-10-09T08:58:26.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7669999999999999e-06 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869465+0000", 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4720000000000003e-06 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.869470+0000", 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9350000000000002e-06 2023-10-09T08:58:26.298 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.883112+0000", 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.013642837 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.883211+0000", 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8906999999999993e-05 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.883350+0000", 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000139121 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.893306+0000", 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0099556920000000004 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.896043+0000", 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0027369690000000001 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.897560+0000", 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0015164550000000001 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.179957+0000", 2023-10-09T08:58:26.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.282397071 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.180013+0000", 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6419000000000003e-05 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.180845+0000", 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00083186200000000005 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.302 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:178444 6.1 6:820dbf68:::benchmark_data_smithi016_105291_object77317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:54:27.878457+0000", 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 238.28678201, 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.307127555, 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.303 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 178444 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.878457+0000", 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.304 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.878457+0000", 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.878456+0000", 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.878460+0000", 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5990000000000004e-06 2023-10-09T08:58:26.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.878467+0000", 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8229999999999997e-06 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.878472+0000", 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5510000000000001e-06 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.306 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887030+0000", 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0085582049999999993 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for degraded object", 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887104+0000", 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4080999999999998e-05 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.172222+0000", 2023-10-09T08:58:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28511837099999998 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.172530+0000", 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00030791400000000001 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.172984+0000", 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00045320799999999997 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.180913+0000", 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0079297980000000001 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.180924+0000", 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0827e-05 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.180973+0000", 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9280000000000003e-05 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.184593+0000", 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0036192630000000002 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.184629+0000", 2023-10-09T08:58:26.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6526000000000002e-05 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.185585+0000", 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00095540000000000002 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.44690.0:178446 6.d 6:b71e36d8:::benchmark_data_smithi016_105291_object77319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T08:58:26.311 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:54:27.887826+0000", 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 238.277412666, 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28122025499999997, 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.44690", 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 178446 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.312 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887826+0000", 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887826+0000", 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887824+0000", 2023-10-09T08:58:26.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887834+0000", 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2189999999999993e-06 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887849+0000", 2023-10-09T08:58:26.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5243e-05 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.887859+0000", 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9380000000000004e-06 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.888079+0000", 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021974000000000001 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.888217+0000", 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000138257 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.888350+0000", 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000133588 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T08:58:26.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.893216+0000", 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048652419999999997 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.896074+0000", 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028585199999999998 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.168864+0000", 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27278977700000001 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.168942+0000", 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7832999999999998e-05 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.169047+0000", 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010481599999999999 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.318 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.44690.0:178461 6.3 e122/121 6:c41b982e:::benchmark_data_smithi016_105291_object77334:head v 122'11096, mlcod=122'11094)", 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:54:27.900107+0000", 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 238.26513203900001, 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28123902899999997, 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.900107+0000", 2023-10-09T08:58:26.319 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.900107+0000", 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.900106+0000", 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.320 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:27.900114+0000", 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4359999999999996e-06 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.168268+0000", 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26815395600000003 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T08:58:26.321 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.168292+0000", 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4409999999999998e-05 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.168541+0000", 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00024835699999999999 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.168567+0000", 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6523e-05 2023-10-09T08:58:26.322 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.181302+0000", 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012735171999999999 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:54:28.181346+0000", 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3862000000000001e-05 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.323 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T08:58:26.324 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T08:58:26.324 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:58:26.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:26.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:26.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:26.425 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:26.428 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:26.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:26.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:26.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:26.601 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:26.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:26.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:26.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:26.737 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:26 smithi016 bash[75179]: cluster 2023-10-09T08:58:25.709238+0000 mgr.y (mgr.25421) 626 : cluster 0 pgmap v605: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 153/204966 objects degraded (0.075%); 25046/204966 objects misplaced (12.220%); 3.0 MiB/s, 50 objects/s recovering 2023-10-09T08:58:26.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:26 smithi016 bash[75179]: cluster 2023-10-09T08:58:26.438837+0000 mon.a (mon.0) 829 : cluster 3 Health check update: Degraded data redundancy: 153/204966 objects degraded (0.075%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:26.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:26 smithi016 bash[77892]: cluster 2023-10-09T08:58:25.709238+0000 mgr.y (mgr.25421) 626 : cluster 0 pgmap v605: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 153/204966 objects degraded (0.075%); 25046/204966 objects misplaced (12.220%); 3.0 MiB/s, 50 objects/s recovering 2023-10-09T08:58:26.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:26 smithi016 bash[77892]: cluster 2023-10-09T08:58:26.438837+0000 mon.a (mon.0) 829 : cluster 3 Health check update: Degraded data redundancy: 153/204966 objects degraded (0.075%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:26.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:26.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:26.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:26.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:26.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:26.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:26.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:26 smithi094 bash[59677]: cluster 2023-10-09T08:58:25.709238+0000 mgr.y (mgr.25421) 626 : cluster 0 pgmap v605: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 153/204966 objects degraded (0.075%); 25046/204966 objects misplaced (12.220%); 3.0 MiB/s, 50 objects/s recovering 2023-10-09T08:58:26.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:26 smithi094 bash[59677]: cluster 2023-10-09T08:58:26.438837+0000 mon.a (mon.0) 829 : cluster 3 Health check update: Degraded data redundancy: 153/204966 objects degraded (0.075%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:26.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:27.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:27.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:27.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:27.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:27.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:27.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:27.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:27.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:27.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:27.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:27.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:27.570 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:27.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:27.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:27.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:27.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:27.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:27.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:27.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:27.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:27.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:28.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:28.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:28.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:28.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:28.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:28.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:28.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:28.393 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:28.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:28.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:28.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:28.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:28.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:28.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:28.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:28.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:28.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:28.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:28.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:28.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:28.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:29.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:29.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:28 smithi094 bash[59677]: cluster 2023-10-09T08:58:27.709912+0000 mgr.y (mgr.25421) 627 : cluster 0 pgmap v606: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 202/200655 objects degraded (0.101%); 24228/200655 objects misplaced (12.074%); 6.5 MiB/s, 105 objects/s recovering 2023-10-09T08:58:29.038 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:29.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:29.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:29.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T08:58:29.240 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:28 smithi016 bash[75179]: cluster 2023-10-09T08:58:27.709912+0000 mgr.y (mgr.25421) 627 : cluster 0 pgmap v606: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 202/200655 objects degraded (0.101%); 24228/200655 objects misplaced (12.074%); 6.5 MiB/s, 105 objects/s recovering 2023-10-09T08:58:29.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:28 smithi016 bash[77892]: cluster 2023-10-09T08:58:27.709912+0000 mgr.y (mgr.25421) 627 : cluster 0 pgmap v606: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 202/200655 objects degraded (0.101%); 24228/200655 objects misplaced (12.074%); 6.5 MiB/s, 105 objects/s recovering 2023-10-09T08:58:29.244 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:29.272 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:29.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:29.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:29.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:29.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:29.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:29.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:29.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:29.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:29.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:29.808 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:29.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:29.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:29.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:29.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:29.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:30.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:30.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:30.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:30.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:30.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:30.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:30.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:30.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:30.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:30.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:30.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:30.497 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:30.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:30.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:30.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:30.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:30.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:30 smithi094 bash[59677]: cluster 2023-10-09T08:58:29.713226+0000 mgr.y (mgr.25421) 628 : cluster 0 pgmap v607: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 309/197070 objects degraded (0.157%); 23538/197070 objects misplaced (11.944%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T08:58:30.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:30 smithi094 bash[59677]: audit 2023-10-09T08:58:29.714156+0000 mon.a (mon.0) 830 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:30.748 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:30.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:30.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:30.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:30.880 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:30.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:30.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:30.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:30.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:30 smithi016 bash[75179]: cluster 2023-10-09T08:58:29.713226+0000 mgr.y (mgr.25421) 628 : cluster 0 pgmap v607: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 309/197070 objects degraded (0.157%); 23538/197070 objects misplaced (11.944%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T08:58:31.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:30 smithi016 bash[75179]: audit 2023-10-09T08:58:29.714156+0000 mon.a (mon.0) 830 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:30 smithi016 bash[77892]: cluster 2023-10-09T08:58:29.713226+0000 mgr.y (mgr.25421) 628 : cluster 0 pgmap v607: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 309/197070 objects degraded (0.157%); 23538/197070 objects misplaced (11.944%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T08:58:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:30 smithi016 bash[77892]: audit 2023-10-09T08:58:29.714156+0000 mon.a (mon.0) 830 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:31.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:31.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:31.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:31.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:31.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:31.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:31.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:31.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:31.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:31.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:31.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:31.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:31.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:31.498 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:31.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:31.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:31.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:31.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:31.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:31.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:31.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:31.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:31.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:31.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:31.891 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:31 smithi016 bash[75179]: cluster 2023-10-09T08:58:31.721187+0000 mon.a (mon.0) 831 : cluster 3 Health check update: Degraded data redundancy: 366/192858 objects degraded (0.190%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:31.891 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:31 smithi016 bash[77892]: cluster 2023-10-09T08:58:31.721187+0000 mon.a (mon.0) 831 : cluster 3 Health check update: Degraded data redundancy: 366/192858 objects degraded (0.190%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:31.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:31.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:32.026 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:32.036 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:31 smithi094 bash[59677]: cluster 2023-10-09T08:58:31.721187+0000 mon.a (mon.0) 831 : cluster 3 Health check update: Degraded data redundancy: 366/192858 objects degraded (0.190%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:32.053 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:32.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:32.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:32.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:32.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:32.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:32.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:32.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:32.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:32.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:32.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:32.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:32.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:32.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:32.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:32 smithi016 bash[75179]: cluster 2023-10-09T08:58:31.713877+0000 mgr.y (mgr.25421) 629 : cluster 0 pgmap v608: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 366/192858 objects degraded (0.190%); 22710/192858 objects misplaced (11.776%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T08:58:32.926 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:32 smithi016 bash[77892]: cluster 2023-10-09T08:58:31.713877+0000 mgr.y (mgr.25421) 629 : cluster 0 pgmap v608: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 366/192858 objects degraded (0.190%); 22710/192858 objects misplaced (11.776%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T08:58:32.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:32.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:32 smithi094 bash[59677]: cluster 2023-10-09T08:58:31.713877+0000 mgr.y (mgr.25421) 629 : cluster 0 pgmap v608: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 366/192858 objects degraded (0.190%); 22710/192858 objects misplaced (11.776%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T08:58:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:32.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:32.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:33.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:33.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:33.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:33.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:33.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:33.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:33.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:33.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:33.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:33.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:33.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:33.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:33.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:33.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:33.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:33.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:33.775 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:33.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:33.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:33.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:33.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:33.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:33.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:34.037 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:34.062 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:34.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:34.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:34.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:34.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:34.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:34.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:34.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:34.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:34.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:34.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:34.592 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:34.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:34.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:34.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:34.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:34.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:34.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:34.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:35.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:35.128 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:35.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:35.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:35.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:35.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:35.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:35.371 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:35.379 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:35 smithi094 bash[59677]: cluster 2023-10-09T08:58:33.714506+0000 mgr.y (mgr.25421) 630 : cluster 0 pgmap v609: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 432/188925 objects degraded (0.229%); 22091/188925 objects misplaced (11.693%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T08:58:35.379 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:35 smithi094 bash[59677]: audit 2023-10-09T08:58:34.230935+0000 mon.a (mon.0) 832 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:35.379 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:35 smithi094 bash[59677]: audit 2023-10-09T08:58:34.232252+0000 mon.a (mon.0) 833 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:35.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:35.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:35.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:35.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:35 smithi016 bash[75179]: cluster 2023-10-09T08:58:33.714506+0000 mgr.y (mgr.25421) 630 : cluster 0 pgmap v609: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 432/188925 objects degraded (0.229%); 22091/188925 objects misplaced (11.693%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T08:58:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:35 smithi016 bash[75179]: audit 2023-10-09T08:58:34.230935+0000 mon.a (mon.0) 832 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:35 smithi016 bash[75179]: audit 2023-10-09T08:58:34.232252+0000 mon.a (mon.0) 833 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:35 smithi016 bash[77892]: cluster 2023-10-09T08:58:33.714506+0000 mgr.y (mgr.25421) 630 : cluster 0 pgmap v609: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 432/188925 objects degraded (0.229%); 22091/188925 objects misplaced (11.693%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T08:58:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:35 smithi016 bash[77892]: audit 2023-10-09T08:58:34.230935+0000 mon.a (mon.0) 832 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:35.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:35 smithi016 bash[77892]: audit 2023-10-09T08:58:34.232252+0000 mon.a (mon.0) 833 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:35.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:35.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:35.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:35.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:35.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:35.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:35.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:35.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:35.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:35.927 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:58:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:58:35] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.33.4" 2023-10-09T08:58:35.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:35.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:35.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:36.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:36.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:36.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:36.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:36.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:36.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:36.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:36.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:36.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:36.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:36.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:36.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:36.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:36.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:36.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:36.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:36.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:36.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:36.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:36.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:36.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:36.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:36.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:36.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:37.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:37.105 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:37.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:37.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:37 smithi094 bash[59677]: cluster 2023-10-09T08:58:35.715075+0000 mgr.y (mgr.25421) 631 : cluster 0 pgmap v610: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 432/188379 objects degraded (0.229%); 22091/188379 objects misplaced (11.727%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T08:58:37.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:37.366 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:37.394 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:37.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:37 smithi016 bash[75179]: cluster 2023-10-09T08:58:35.715075+0000 mgr.y (mgr.25421) 631 : cluster 0 pgmap v610: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 432/188379 objects degraded (0.229%); 22091/188379 objects misplaced (11.727%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T08:58:37.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:37 smithi016 bash[77892]: cluster 2023-10-09T08:58:35.715075+0000 mgr.y (mgr.25421) 631 : cluster 0 pgmap v610: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 432/188379 objects degraded (0.229%); 22091/188379 objects misplaced (11.727%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T08:58:37.523 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:37.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:37.653 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:37.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.780 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:37.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:37.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:38.012 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:38.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:38.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:38.139 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:38.164 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:38.211 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:38.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:38.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:38.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:38.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:38.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:38.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:38 smithi094 bash[59677]: cluster 2023-10-09T08:58:38.231411+0000 mon.a (mon.0) 834 : cluster 3 Health check update: Degraded data redundancy: 488/184050 objects degraded (0.265%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:38.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:38 smithi016 bash[75179]: cluster 2023-10-09T08:58:38.231411+0000 mon.a (mon.0) 834 : cluster 3 Health check update: Degraded data redundancy: 488/184050 objects degraded (0.265%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:38.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:38 smithi016 bash[77892]: cluster 2023-10-09T08:58:38.231411+0000 mon.a (mon.0) 834 : cluster 3 Health check update: Degraded data redundancy: 488/184050 objects degraded (0.265%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:38.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:38.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:38.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:38.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:38.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:38.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:38.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:38.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:38.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:38.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:38.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:38.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:39.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:39.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:39.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:39.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:39.211 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:39.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:39.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:39.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.366 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:39.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:39.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:39 smithi016 bash[75179]: cluster 2023-10-09T08:58:37.715737+0000 mgr.y (mgr.25421) 632 : cluster 0 pgmap v611: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 488/184050 objects degraded (0.265%); 21354/184050 objects misplaced (11.602%); 10 MiB/s, 166 objects/s recovering 2023-10-09T08:58:39.392 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:39 smithi016 bash[77892]: cluster 2023-10-09T08:58:37.715737+0000 mgr.y (mgr.25421) 632 : cluster 0 pgmap v611: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 488/184050 objects degraded (0.265%); 21354/184050 objects misplaced (11.602%); 10 MiB/s, 166 objects/s recovering 2023-10-09T08:58:39.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.483 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:39 smithi094 bash[59677]: cluster 2023-10-09T08:58:37.715737+0000 mgr.y (mgr.25421) 632 : cluster 0 pgmap v611: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 488/184050 objects degraded (0.265%); 21354/184050 objects misplaced (11.602%); 10 MiB/s, 166 objects/s recovering 2023-10-09T08:58:39.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:39.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:39.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.634 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:39.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:39.740 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:39.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:39.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:39.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:39.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:39.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:40.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:40.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:40.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:40.162 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:40.272 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:40.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:40.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:40.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:40.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:40.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:40.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:40.673 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:40.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:40.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:40.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:40.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:40.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:40.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:40.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:41.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:41.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:41.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:41.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:41.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:41.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:41.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:41.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:41.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:41.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:41.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:41.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:41.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:41.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:41.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:41.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:41.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:41 smithi016 bash[75179]: cluster 2023-10-09T08:58:39.716328+0000 mgr.y (mgr.25421) 633 : cluster 0 pgmap v612: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 590/180246 objects degraded (0.327%); 20629/180246 objects misplaced (11.445%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T08:58:41.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:41 smithi016 bash[75179]: audit 2023-10-09T08:58:40.873868+0000 mon.a (mon.0) 835 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:41.643 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:41 smithi016 bash[77892]: cluster 2023-10-09T08:58:39.716328+0000 mgr.y (mgr.25421) 633 : cluster 0 pgmap v612: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 590/180246 objects degraded (0.327%); 20629/180246 objects misplaced (11.445%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T08:58:41.643 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:41 smithi016 bash[77892]: audit 2023-10-09T08:58:40.873868+0000 mon.a (mon.0) 835 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:41.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:41.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:41.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:41 smithi094 bash[59677]: cluster 2023-10-09T08:58:39.716328+0000 mgr.y (mgr.25421) 633 : cluster 0 pgmap v612: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 590/180246 objects degraded (0.327%); 20629/180246 objects misplaced (11.445%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T08:58:41.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:41 smithi094 bash[59677]: audit 2023-10-09T08:58:40.873868+0000 mon.a (mon.0) 835 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:41.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:41.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:41.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:41.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:41.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:41.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:41.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:42.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:42.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:42.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:42.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:42.184 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:42.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:42.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:42.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:42.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:42.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:42.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:42.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:42.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:42.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:42.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:42.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:42.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:42.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:42.739 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:42.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:42.842 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:42.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:42.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:42.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:43.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:43.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:43.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:43.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:43.241 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:43.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:43.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:43.401 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:43.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:43.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:43.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:43.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:43.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:43 smithi094 bash[59677]: cluster 2023-10-09T08:58:41.716969+0000 mgr.y (mgr.25421) 634 : cluster 0 pgmap v613: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 664/175962 objects degraded (0.377%); 19913/175962 objects misplaced (11.317%); 10 MiB/s, 160 objects/s recovering 2023-10-09T08:58:43.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:43.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:43.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:43 smithi016 bash[75179]: cluster 2023-10-09T08:58:41.716969+0000 mgr.y (mgr.25421) 634 : cluster 0 pgmap v613: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 664/175962 objects degraded (0.377%); 19913/175962 objects misplaced (11.317%); 10 MiB/s, 160 objects/s recovering 2023-10-09T08:58:43.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:43 smithi016 bash[77892]: cluster 2023-10-09T08:58:41.716969+0000 mgr.y (mgr.25421) 634 : cluster 0 pgmap v613: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 664/175962 objects degraded (0.377%); 19913/175962 objects misplaced (11.317%); 10 MiB/s, 160 objects/s recovering 2023-10-09T08:58:43.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:43.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:43.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:43.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:43.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:43.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:44.056 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:44.081 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:44.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:44.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:44.326 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:44.348 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:44.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:44.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:44 smithi016 bash[75179]: cluster 2023-10-09T08:58:44.448112+0000 mon.a (mon.0) 836 : cluster 3 Health check update: Degraded data redundancy: 664/175962 objects degraded (0.377%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:44.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:44 smithi016 bash[77892]: cluster 2023-10-09T08:58:44.448112+0000 mon.a (mon.0) 836 : cluster 3 Health check update: Degraded data redundancy: 664/175962 objects degraded (0.377%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:44.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:44.579 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:44.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:44 smithi094 bash[59677]: cluster 2023-10-09T08:58:44.448112+0000 mon.a (mon.0) 836 : cluster 3 Health check update: Degraded data redundancy: 664/175962 objects degraded (0.377%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:44.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:44.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:44.729 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:44.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:44.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:44.855 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:44.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:44.982 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:44.984 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:58:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:44.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:45.057 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:58:45.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:45.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:45.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:45.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:45.150 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:58:45.204 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:58:45.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:45.240 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:58:45.258 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:45.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:45.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:45.285 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:58:45.325 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:58:45.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:45.388 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:58:45.389 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.389 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.390 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.390 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.390 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.390 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.390 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.390 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:58:45.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:45.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:45.475 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:58:45.475 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T08:58:45.475 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:58:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:45 smithi094 bash[59677]: cluster 2023-10-09T08:58:43.717603+0000 mgr.y (mgr.25421) 635 : cluster 0 pgmap v614: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 768/171666 objects degraded (0.447%); 19206/171666 objects misplaced (11.188%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T08:58:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:45 smithi094 bash[59677]: audit 2023-10-09T08:58:44.942618+0000 mon.a (mon.0) 837 : audit 0 from='client.? 172.21.15.16:0/3111520004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:58:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:45 smithi094 bash[59677]: audit 2023-10-09T08:58:45.050363+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.16:0/1550714787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:58:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:45 smithi094 bash[59677]: audit 2023-10-09T08:58:45.051891+0000 mon.a (mon.0) 838 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:58:45.500 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T08:58:45.507 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:45.532 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:45.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:45.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:45.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:45.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:45.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:45.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[75179]: cluster 2023-10-09T08:58:43.717603+0000 mgr.y (mgr.25421) 635 : cluster 0 pgmap v614: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 768/171666 objects degraded (0.447%); 19206/171666 objects misplaced (11.188%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T08:58:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[75179]: audit 2023-10-09T08:58:44.942618+0000 mon.a (mon.0) 837 : audit 0 from='client.? 172.21.15.16:0/3111520004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:58:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[75179]: audit 2023-10-09T08:58:45.050363+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.16:0/1550714787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:58:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[75179]: audit 2023-10-09T08:58:45.051891+0000 mon.a (mon.0) 838 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:58:45.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[77892]: cluster 2023-10-09T08:58:43.717603+0000 mgr.y (mgr.25421) 635 : cluster 0 pgmap v614: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 768/171666 objects degraded (0.447%); 19206/171666 objects misplaced (11.188%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T08:58:45.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[77892]: audit 2023-10-09T08:58:44.942618+0000 mon.a (mon.0) 837 : audit 0 from='client.? 172.21.15.16:0/3111520004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:58:45.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[77892]: audit 2023-10-09T08:58:45.050363+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.16:0/1550714787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:58:45.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[77892]: audit 2023-10-09T08:58:45.051891+0000 mon.a (mon.0) 838 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:58:45.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:45.776 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:58:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:58:45] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.33.4" 2023-10-09T08:58:45.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:45.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:45.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:45.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:45.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:45.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:46.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:46.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:46.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:46.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:46.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:46.190 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:46.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:46.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:46.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:46.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:46.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:46.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:46.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:46.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:46.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:46.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:46.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:46.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: cluster 2023-10-09T08:58:45.467521+0000 mon.a (mon.0) 839 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:58:46.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: cluster 2023-10-09T08:58:45.478353+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: audit 2023-10-09T08:58:45.482945+0000 mon.a (mon.0) 840 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: cluster 2023-10-09T08:58:45.483167+0000 mon.a (mon.0) 841 : cluster 0 osdmap e149: 8 total, 8 up, 6 in 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: cluster 2023-10-09T08:58:45.523885+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: cluster 2023-10-09T08:58:45.528266+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: cluster 2023-10-09T08:58:45.530749+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[75179]: cluster 2023-10-09T08:58:45.718045+0000 mgr.y (mgr.25421) 636 : cluster 0 pgmap v616: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 574 op/s; 768/171147 objects degraded (0.449%); 19206/171147 objects misplaced (11.222%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: cluster 2023-10-09T08:58:45.467521+0000 mon.a (mon.0) 839 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: cluster 2023-10-09T08:58:45.478353+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.636 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: audit 2023-10-09T08:58:45.482945+0000 mon.a (mon.0) 840 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:58:46.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: cluster 2023-10-09T08:58:45.483167+0000 mon.a (mon.0) 841 : cluster 0 osdmap e149: 8 total, 8 up, 6 in 2023-10-09T08:58:46.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: cluster 2023-10-09T08:58:45.523885+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: cluster 2023-10-09T08:58:45.528266+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: cluster 2023-10-09T08:58:45.530749+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:46 smithi016 bash[77892]: cluster 2023-10-09T08:58:45.718045+0000 mgr.y (mgr.25421) 636 : cluster 0 pgmap v616: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 574 op/s; 768/171147 objects degraded (0.449%); 19206/171147 objects misplaced (11.222%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:46.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:46.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:46.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: cluster 2023-10-09T08:58:45.467521+0000 mon.a (mon.0) 839 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T08:58:46.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: cluster 2023-10-09T08:58:45.478353+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: audit 2023-10-09T08:58:45.482945+0000 mon.a (mon.0) 840 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T08:58:46.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: cluster 2023-10-09T08:58:45.483167+0000 mon.a (mon.0) 841 : cluster 0 osdmap e149: 8 total, 8 up, 6 in 2023-10-09T08:58:46.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: cluster 2023-10-09T08:58:45.523885+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: cluster 2023-10-09T08:58:45.528266+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: cluster 2023-10-09T08:58:45.530749+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T08:58:46.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:46 smithi094 bash[59677]: cluster 2023-10-09T08:58:45.718045+0000 mgr.y (mgr.25421) 636 : cluster 0 pgmap v616: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 574 op/s; 768/171147 objects degraded (0.449%); 19206/171147 objects misplaced (11.222%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:46.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:46.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:46.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:46.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:46.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:46.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:47.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:47.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:47.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:47.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:47.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:47.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:47.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:47.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:47.658 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:47.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:47.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:47.920 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.947 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:47.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:48.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:48.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:48.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:48.156 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:48 smithi016 bash[77892]: audit 2023-10-09T08:58:47.136401+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:48.156 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:48 smithi016 bash[77892]: audit 2023-10-09T08:58:47.146811+0000 mon.a (mon.0) 843 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:48.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:48.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:48.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:48 smithi094 bash[59677]: audit 2023-10-09T08:58:47.136401+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:48.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:48 smithi094 bash[59677]: audit 2023-10-09T08:58:47.146811+0000 mon.a (mon.0) 843 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:48.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:48.366 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:48.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:48.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:48.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:48 smithi016 bash[75179]: audit 2023-10-09T08:58:47.136401+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:48.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:48 smithi016 bash[75179]: audit 2023-10-09T08:58:47.146811+0000 mon.a (mon.0) 843 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:48.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:48.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:48.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:48.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:48.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:48.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:48.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:48.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:48.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:48.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:48.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:48.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:48.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:49.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:49.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:49.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:49.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:49.148 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:49 smithi094 bash[59677]: cluster 2023-10-09T08:58:47.718722+0000 mgr.y (mgr.25421) 637 : cluster 0 pgmap v617: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 825/167262 objects degraded (0.493%); 18495/167262 objects misplaced (11.058%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T08:58:49.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:49.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:49.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:49.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:49.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:49.340 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:49.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:49.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:49.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:49 smithi016 bash[75179]: cluster 2023-10-09T08:58:47.718722+0000 mgr.y (mgr.25421) 637 : cluster 0 pgmap v617: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 825/167262 objects degraded (0.493%); 18495/167262 objects misplaced (11.058%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T08:58:49.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:49 smithi016 bash[77892]: cluster 2023-10-09T08:58:47.718722+0000 mgr.y (mgr.25421) 637 : cluster 0 pgmap v617: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 825/167262 objects degraded (0.493%); 18495/167262 objects misplaced (11.058%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T08:58:49.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:49.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:49.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:49.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:49.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:49.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:49.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:49.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:49.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:49.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:49.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:49.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:49.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:50.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:50.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:50.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:50.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:50.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:50.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:50.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:50 smithi016 bash[75179]: audit 2023-10-09T08:58:49.226024+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:50.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:50 smithi016 bash[75179]: audit 2023-10-09T08:58:49.227273+0000 mon.a (mon.0) 845 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:50.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:50 smithi016 bash[75179]: cluster 2023-10-09T08:58:49.451664+0000 mon.a (mon.0) 846 : cluster 3 Health check update: Degraded data redundancy: 825/167262 objects degraded (0.493%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:50.298 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:50 smithi016 bash[77892]: audit 2023-10-09T08:58:49.226024+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:50.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:50 smithi016 bash[77892]: audit 2023-10-09T08:58:49.227273+0000 mon.a (mon.0) 845 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:50.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:50 smithi016 bash[77892]: cluster 2023-10-09T08:58:49.451664+0000 mon.a (mon.0) 846 : cluster 3 Health check update: Degraded data redundancy: 825/167262 objects degraded (0.493%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:50.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:50.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:50.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:50.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:50.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:50.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:50 smithi094 bash[59677]: audit 2023-10-09T08:58:49.226024+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:50.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:50 smithi094 bash[59677]: audit 2023-10-09T08:58:49.227273+0000 mon.a (mon.0) 845 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:58:50.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:50 smithi094 bash[59677]: cluster 2023-10-09T08:58:49.451664+0000 mon.a (mon.0) 846 : cluster 3 Health check update: Degraded data redundancy: 825/167262 objects degraded (0.493%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:50.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:50.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:50.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:50.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:50.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:50.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:50.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:50.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:50.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:50.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:50.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:50.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:51.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:51.125 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:51.149 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:51.252 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:51.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:51.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:51.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:51.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:51.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:51.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:51.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:51.678 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:51.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:51.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:51 smithi094 bash[59677]: cluster 2023-10-09T08:58:49.719309+0000 mgr.y (mgr.25421) 638 : cluster 0 pgmap v618: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 895/163779 objects degraded (0.546%); 17848/163779 objects misplaced (10.898%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T08:58:51.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:51 smithi016 bash[75179]: cluster 2023-10-09T08:58:49.719309+0000 mgr.y (mgr.25421) 638 : cluster 0 pgmap v618: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 895/163779 objects degraded (0.546%); 17848/163779 objects misplaced (10.898%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T08:58:51.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:51 smithi016 bash[77892]: cluster 2023-10-09T08:58:49.719309+0000 mgr.y (mgr.25421) 638 : cluster 0 pgmap v618: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 895/163779 objects degraded (0.546%); 17848/163779 objects misplaced (10.898%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T08:58:51.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:51.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:51.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:51.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:51.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:52.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:52.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:52.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:52.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:52.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:52.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:52.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:52.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:52.357 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:52.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:52.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:52.460 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:52.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:52.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:52.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:52.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:52.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:52.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:52.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:52.750 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:52.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:52.854 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:52.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:52.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:52.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:53.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:53.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:53.126 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:53.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:53.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:53.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:53.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:53.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:53.389 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:53.419 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:53.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:53.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:53.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:53.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:53.593 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:53 smithi016 bash[75179]: cluster 2023-10-09T08:58:51.719939+0000 mgr.y (mgr.25421) 639 : cluster 0 pgmap v619: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 963/159621 objects degraded (0.603%); 17221/159621 objects misplaced (10.789%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:53.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:53 smithi016 bash[77892]: cluster 2023-10-09T08:58:51.719939+0000 mgr.y (mgr.25421) 639 : cluster 0 pgmap v619: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 963/159621 objects degraded (0.603%); 17221/159621 objects misplaced (10.789%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:53.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:53.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:53.704 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:53 smithi094 bash[59677]: cluster 2023-10-09T08:58:51.719939+0000 mgr.y (mgr.25421) 639 : cluster 0 pgmap v619: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 963/159621 objects degraded (0.603%); 17221/159621 objects misplaced (10.789%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:53.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:53.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:53.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:53.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:53.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:53.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:53.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:54.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:54.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:54.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:54.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T08:58:54.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:54.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:54.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:54.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T08:58:54.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:54.314 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:58:54.314 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:58:54.319 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:54.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:54.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:54.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:54.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:54.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:54.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:54.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:54 smithi016 bash[75179]: cluster 2023-10-09T08:58:54.456025+0000 mon.a (mon.0) 847 : cluster 3 Health check update: Degraded data redundancy: 963/159621 objects degraded (0.603%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:54.630 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:54 smithi016 bash[77892]: cluster 2023-10-09T08:58:54.456025+0000 mon.a (mon.0) 847 : cluster 3 Health check update: Degraded data redundancy: 963/159621 objects degraded (0.603%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:54.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:54.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:54.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:54.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:54 smithi094 bash[59677]: cluster 2023-10-09T08:58:54.456025+0000 mon.a (mon.0) 847 : cluster 3 Health check update: Degraded data redundancy: 963/159621 objects degraded (0.603%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:54.758 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:54.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:54.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:54.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:54.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:54.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:54.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:54.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:55.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:55.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:55.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:55.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:55.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:55.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:55.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:55.425 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:55.450 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:55.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:55.557 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:55.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:55 smithi094 bash[59677]: cluster 2023-10-09T08:58:53.720526+0000 mgr.y (mgr.25421) 640 : cluster 0 pgmap v620: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 1069/155679 objects degraded (0.687%); 16543/155679 objects misplaced (10.626%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:55.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:55 smithi094 bash[59677]: audit 2023-10-09T08:58:54.695768+0000 mon.a (mon.0) 848 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:55 smithi094 bash[59677]: audit 2023-10-09T08:58:54.709999+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.568 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:55 smithi094 bash[59677]: audit 2023-10-09T08:58:55.392222+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:55.568 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:55 smithi094 bash[59677]: audit 2023-10-09T08:58:55.393169+0000 mon.a (mon.0) 851 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:55.568 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:55 smithi094 bash[59677]: audit 2023-10-09T08:58:55.405254+0000 mon.a (mon.0) 852 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:55.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:55.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[77892]: cluster 2023-10-09T08:58:53.720526+0000 mgr.y (mgr.25421) 640 : cluster 0 pgmap v620: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 1069/155679 objects degraded (0.687%); 16543/155679 objects misplaced (10.626%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[77892]: audit 2023-10-09T08:58:54.695768+0000 mon.a (mon.0) 848 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[77892]: audit 2023-10-09T08:58:54.709999+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[77892]: audit 2023-10-09T08:58:55.392222+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[77892]: audit 2023-10-09T08:58:55.393169+0000 mon.a (mon.0) 851 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[77892]: audit 2023-10-09T08:58:55.405254+0000 mon.a (mon.0) 852 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[75179]: cluster 2023-10-09T08:58:53.720526+0000 mgr.y (mgr.25421) 640 : cluster 0 pgmap v620: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 1069/155679 objects degraded (0.687%); 16543/155679 objects misplaced (10.626%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T08:58:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[75179]: audit 2023-10-09T08:58:54.695768+0000 mon.a (mon.0) 848 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[75179]: audit 2023-10-09T08:58:54.709999+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[75179]: audit 2023-10-09T08:58:55.392222+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[75179]: audit 2023-10-09T08:58:55.393169+0000 mon.a (mon.0) 851 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:55.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[75179]: audit 2023-10-09T08:58:55.405254+0000 mon.a (mon.0) 852 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:58:55.741 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:58:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:58:55] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.33.4" 2023-10-09T08:58:55.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:55.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:55.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:55.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:55.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:55.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:56.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:56.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:56.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:56.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:56.230 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:56.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:56.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:56.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:56.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:56.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:56.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:56.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T08:58:56.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:56.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:56.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:56.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:56.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:56.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:56.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:56.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:56.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:56.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:56.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:56.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:56.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:56.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:57.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:57.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:57.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:57.212 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:57.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:57.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:57.339 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:57.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:57.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:57.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:58:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:58:57.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:57.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:57.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:57.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:57.729 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:57 smithi094 bash[59677]: cluster 2023-10-09T08:58:55.721091+0000 mgr.y (mgr.25421) 641 : cluster 0 pgmap v621: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 519 op/s; 1069/155175 objects degraded (0.689%); 16543/155175 objects misplaced (10.661%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T08:58:57.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:57 smithi016 bash[75179]: cluster 2023-10-09T08:58:55.721091+0000 mgr.y (mgr.25421) 641 : cluster 0 pgmap v621: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 519 op/s; 1069/155175 objects degraded (0.689%); 16543/155175 objects misplaced (10.661%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T08:58:57.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:57 smithi016 bash[77892]: cluster 2023-10-09T08:58:55.721091+0000 mgr.y (mgr.25421) 641 : cluster 0 pgmap v621: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 519 op/s; 1069/155175 objects degraded (0.689%); 16543/155175 objects misplaced (10.661%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T08:58:57.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:57.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:57.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:57.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:57.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:58.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:58.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:58.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:58.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:58.251 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:58.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:58.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:58.313 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6] out_osds: [7, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:58:58.314 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:58:58.314 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 5, 6] 2023-10-09T08:58:58.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T08:58:58.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:58.388 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:58.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:58.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:58.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:58.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:58.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:58.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:58.654 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:58.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:58.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[75179]: cluster 2023-10-09T08:58:57.497958+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T08:58:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[75179]: cluster 2023-10-09T08:58:57.502268+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T08:58:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[75179]: cluster 2023-10-09T08:58:57.502342+0000 mon.a (mon.0) 853 : cluster 0 osdmap e150: 8 total, 8 up, 6 in 2023-10-09T08:58:58.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[75179]: cluster 2023-10-09T08:58:57.721636+0000 mgr.y (mgr.25421) 642 : cluster 0 pgmap v623: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 533 op/s; 1076/151239 objects degraded (0.711%); 15928/151239 objects misplaced (10.532%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T08:58:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[77892]: cluster 2023-10-09T08:58:57.497958+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T08:58:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[77892]: cluster 2023-10-09T08:58:57.502268+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T08:58:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[77892]: cluster 2023-10-09T08:58:57.502342+0000 mon.a (mon.0) 853 : cluster 0 osdmap e150: 8 total, 8 up, 6 in 2023-10-09T08:58:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:58 smithi016 bash[77892]: cluster 2023-10-09T08:58:57.721636+0000 mgr.y (mgr.25421) 642 : cluster 0 pgmap v623: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 533 op/s; 1076/151239 objects degraded (0.711%); 15928/151239 objects misplaced (10.532%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T08:58:58.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:58.764 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:58 smithi094 bash[59677]: cluster 2023-10-09T08:58:57.497958+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T08:58:58.764 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:58 smithi094 bash[59677]: cluster 2023-10-09T08:58:57.502268+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T08:58:58.764 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:58 smithi094 bash[59677]: cluster 2023-10-09T08:58:57.502342+0000 mon.a (mon.0) 853 : cluster 0 osdmap e150: 8 total, 8 up, 6 in 2023-10-09T08:58:58.764 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:58 smithi094 bash[59677]: cluster 2023-10-09T08:58:57.721636+0000 mgr.y (mgr.25421) 642 : cluster 0 pgmap v623: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 533 op/s; 1076/151239 objects degraded (0.711%); 15928/151239 objects misplaced (10.532%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T08:58:58.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:58:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:58.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:58:58.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:58.907 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:58.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:59.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:59.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:59.128 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:59.128 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:58:59.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:59.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:58:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:58:59.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:59.332 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:58:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:58:59.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:59.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:58:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:58:59.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:59.581 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:59 smithi094 bash[59677]: cluster 2023-10-09T08:58:58.503781+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T08:58:59.581 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:59 smithi094 bash[59677]: cluster 2023-10-09T08:58:58.507765+0000 mon.a (mon.0) 854 : cluster 0 osdmap e151: 8 total, 8 up, 6 in 2023-10-09T08:58:59.581 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:59 smithi094 bash[59677]: cluster 2023-10-09T08:58:58.508940+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T08:58:59.582 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:58:59 smithi094 bash[59677]: cluster 2023-10-09T08:58:59.457189+0000 mon.a (mon.0) 855 : cluster 3 Health check update: Degraded data redundancy: 1076/151239 objects degraded (0.711%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:59.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:58:59.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:59.708 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[75179]: cluster 2023-10-09T08:58:58.503781+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T08:58:59.708 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[75179]: cluster 2023-10-09T08:58:58.507765+0000 mon.a (mon.0) 854 : cluster 0 osdmap e151: 8 total, 8 up, 6 in 2023-10-09T08:58:59.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[75179]: cluster 2023-10-09T08:58:58.508940+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T08:58:59.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[75179]: cluster 2023-10-09T08:58:59.457189+0000 mon.a (mon.0) 855 : cluster 3 Health check update: Degraded data redundancy: 1076/151239 objects degraded (0.711%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:59.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[77892]: cluster 2023-10-09T08:58:58.503781+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T08:58:59.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[77892]: cluster 2023-10-09T08:58:58.507765+0000 mon.a (mon.0) 854 : cluster 0 osdmap e151: 8 total, 8 up, 6 in 2023-10-09T08:58:59.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[77892]: cluster 2023-10-09T08:58:58.508940+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T08:58:59.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:58:59 smithi016 bash[77892]: cluster 2023-10-09T08:58:59.457189+0000 mon.a (mon.0) 855 : cluster 3 Health check update: Degraded data redundancy: 1076/151239 objects degraded (0.711%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:58:59.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:58:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:58:59.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:59.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:58:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:58:59.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:59.992 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:58:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:58:59.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:00.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:00.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:00.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:00.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:00.233 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:00.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:00.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:00.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:00.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:00.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:00.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:00.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:00.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:00.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:00.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:00.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:00.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:00.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:00.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:00 smithi016 bash[75179]: cluster 2023-10-09T08:58:59.722166+0000 mgr.y (mgr.25421) 643 : cluster 0 pgmap v625: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 1154/147882 objects degraded (0.780%); 15272/147882 objects misplaced (10.327%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T08:59:00.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:00 smithi016 bash[77892]: cluster 2023-10-09T08:58:59.722166+0000 mgr.y (mgr.25421) 643 : cluster 0 pgmap v625: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 1154/147882 objects degraded (0.780%); 15272/147882 objects misplaced (10.327%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T08:59:00.766 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:00 smithi094 bash[59677]: cluster 2023-10-09T08:58:59.722166+0000 mgr.y (mgr.25421) 643 : cluster 0 pgmap v625: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 1154/147882 objects degraded (0.780%); 15272/147882 objects misplaced (10.327%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T08:59:00.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:00.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:00.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:00.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:00.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:00.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:01.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:01.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:01.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:01.168 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:59:01.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:01.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:01.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:01.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:01.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:01.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:01.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:01.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:01.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:01.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:01.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:01.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:01.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:01.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:01.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:01.735 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:01.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:01.841 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:01.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:01.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:01.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:02.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:02.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:02.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:02.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:02.183 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:59:02.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:02.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:02.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:02.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:02.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:02.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:02.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:02.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:02.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:02.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:02.815 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:02 smithi094 bash[59677]: cluster 2023-10-09T08:59:01.722881+0000 mgr.y (mgr.25421) 644 : cluster 0 pgmap v626: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 1156/144186 objects degraded (0.802%); 15076/144186 objects misplaced (10.456%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T08:59:02.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:02.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:02.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:02.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:02 smithi016 bash[75179]: cluster 2023-10-09T08:59:01.722881+0000 mgr.y (mgr.25421) 644 : cluster 0 pgmap v626: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 1156/144186 objects degraded (0.802%); 15076/144186 objects misplaced (10.456%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T08:59:02.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:02 smithi016 bash[77892]: cluster 2023-10-09T08:59:01.722881+0000 mgr.y (mgr.25421) 644 : cluster 0 pgmap v626: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 1156/144186 objects degraded (0.802%); 15076/144186 objects misplaced (10.456%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T08:59:02.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:02.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:03.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:03.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:03.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:03.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:03.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:03.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:03.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:03.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:03.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:03.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:03.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:03.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:03.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:03.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:03.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:03.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:03.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:03.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:03.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:03.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:03.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:03.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:03.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:04.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:04.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:04.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:04.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:04.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:04.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:04.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:04.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:04.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:04.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:04.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:04.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:04.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:04.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:04.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:04.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:04.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:04.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:04.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:04.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:04.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:04.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:04.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:04.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:04.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:04.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:05.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:04 smithi016 bash[75179]: cluster 2023-10-09T08:59:03.723508+0000 mgr.y (mgr.25421) 645 : cluster 0 pgmap v627: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 1217/140235 objects degraded (0.868%); 14407/140235 objects misplaced (10.273%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T08:59:05.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:04 smithi016 bash[75179]: audit 2023-10-09T08:59:04.221247+0000 mon.a (mon.0) 856 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:05.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:04 smithi016 bash[75179]: cluster 2023-10-09T08:59:04.461136+0000 mon.a (mon.0) 857 : cluster 3 Health check update: Degraded data redundancy: 1217/140235 objects degraded (0.868%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:05.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:04 smithi016 bash[77892]: cluster 2023-10-09T08:59:03.723508+0000 mgr.y (mgr.25421) 645 : cluster 0 pgmap v627: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 1217/140235 objects degraded (0.868%); 14407/140235 objects misplaced (10.273%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T08:59:05.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:04 smithi016 bash[77892]: audit 2023-10-09T08:59:04.221247+0000 mon.a (mon.0) 856 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:05.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:04 smithi016 bash[77892]: cluster 2023-10-09T08:59:04.461136+0000 mon.a (mon.0) 857 : cluster 3 Health check update: Degraded data redundancy: 1217/140235 objects degraded (0.868%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:05.001 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:05.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:05.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:05.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:05.136 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:04 smithi094 bash[59677]: cluster 2023-10-09T08:59:03.723508+0000 mgr.y (mgr.25421) 645 : cluster 0 pgmap v627: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 1217/140235 objects degraded (0.868%); 14407/140235 objects misplaced (10.273%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T08:59:05.136 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:04 smithi094 bash[59677]: audit 2023-10-09T08:59:04.221247+0000 mon.a (mon.0) 856 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:05.136 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:04 smithi094 bash[59677]: cluster 2023-10-09T08:59:04.461136+0000 mon.a (mon.0) 857 : cluster 3 Health check update: Degraded data redundancy: 1217/140235 objects degraded (0.868%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:05.152 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:05.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:05.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:05.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:05.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:05.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:05.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:05.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:05.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:05.663 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:05.698 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:05.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:05.740 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:59:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:59:05] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.33.4" 2023-10-09T08:59:05.801 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:05.825 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:05.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:05.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:05.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:05.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:05.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:06.067 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:06.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:06.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:06.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:06.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:06.224 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:06.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:06.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:06.353 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:06.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:06.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:06.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:06.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:06.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:06.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:06.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:06.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:06.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:06.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:06.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:06.846 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:06.854 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:06 smithi094 bash[59677]: cluster 2023-10-09T08:59:05.724047+0000 mgr.y (mgr.25421) 646 : cluster 0 pgmap v628: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 1217/139671 objects degraded (0.871%); 14407/139671 objects misplaced (10.315%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T08:59:06.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:06.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:06.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:06.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:07.108 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:07.143 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:06 smithi016 bash[75179]: cluster 2023-10-09T08:59:05.724047+0000 mgr.y (mgr.25421) 646 : cluster 0 pgmap v628: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 1217/139671 objects degraded (0.871%); 14407/139671 objects misplaced (10.315%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T08:59:07.143 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:06 smithi016 bash[77892]: cluster 2023-10-09T08:59:05.724047+0000 mgr.y (mgr.25421) 646 : cluster 0 pgmap v628: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 1217/139671 objects degraded (0.871%); 14407/139671 objects misplaced (10.315%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T08:59:07.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:07.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:07.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.404 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:07.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:07.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:07.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:07.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:07.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:07.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:07.772 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.796 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:07.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:07.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:07.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:08.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:08.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:08.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:08.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:08.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:08.341 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:08.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:08.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:08.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:08.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:08.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:08.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:08.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:08.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:08.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:08.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:08.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:08.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:08.981 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:08 smithi094 bash[59677]: cluster 2023-10-09T08:59:07.724681+0000 mgr.y (mgr.25421) 647 : cluster 0 pgmap v629: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 526 op/s; 1259/135426 objects degraded (0.930%); 14247/135426 objects misplaced (10.520%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:59:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:08.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:09.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:09.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:09.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:09.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:09.121 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:08 smithi016 bash[75179]: cluster 2023-10-09T08:59:07.724681+0000 mgr.y (mgr.25421) 647 : cluster 0 pgmap v629: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 526 op/s; 1259/135426 objects degraded (0.930%); 14247/135426 objects misplaced (10.520%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:59:09.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:08 smithi016 bash[77892]: cluster 2023-10-09T08:59:07.724681+0000 mgr.y (mgr.25421) 647 : cluster 0 pgmap v629: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 526 op/s; 1259/135426 objects degraded (0.930%); 14247/135426 objects misplaced (10.520%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T08:59:09.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:09.234 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:09.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:09.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:09.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:09.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:09.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:09.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:09.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:09.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:09.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:09.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:09.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:09.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:09.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:09.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:09.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:09.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:09.891 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:09.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:09.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:09.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:10.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:09 smithi016 bash[75179]: cluster 2023-10-09T08:59:09.464277+0000 mon.a (mon.0) 858 : cluster 3 Health check update: Degraded data redundancy: 1259/135426 objects degraded (0.930%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:10.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:09 smithi016 bash[77892]: cluster 2023-10-09T08:59:09.464277+0000 mon.a (mon.0) 858 : cluster 3 Health check update: Degraded data redundancy: 1259/135426 objects degraded (0.930%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:10.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:10.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:10.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:10.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:10.161 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:09 smithi094 bash[59677]: cluster 2023-10-09T08:59:09.464277+0000 mon.a (mon.0) 858 : cluster 3 Health check update: Degraded data redundancy: 1259/135426 objects degraded (0.930%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:10.174 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:10.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:10.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:10.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:10.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:10.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:10.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:10.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:10.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:10.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:10.678 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:10.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:10.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:10.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:10.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:10.963 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:10.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:11.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:11.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:11.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:11.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:11.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:11.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:11.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:11.357 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:11.464 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:11.487 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:11.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:11 smithi016 bash[77892]: cluster 2023-10-09T08:59:09.725326+0000 mgr.y (mgr.25421) 648 : cluster 0 pgmap v630: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 1391/131682 objects degraded (1.056%); 13520/131682 objects misplaced (10.267%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T08:59:11.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:11 smithi016 bash[75179]: cluster 2023-10-09T08:59:09.725326+0000 mgr.y (mgr.25421) 648 : cluster 0 pgmap v630: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 1391/131682 objects degraded (1.056%); 13520/131682 objects misplaced (10.267%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T08:59:11.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:11.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:11.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:11.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:11.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:11.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:11 smithi094 bash[59677]: cluster 2023-10-09T08:59:09.725326+0000 mgr.y (mgr.25421) 648 : cluster 0 pgmap v630: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 1391/131682 objects degraded (1.056%); 13520/131682 objects misplaced (10.267%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T08:59:11.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:11.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:11.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:11.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:11.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:11.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:11.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:12.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:12.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:12.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:12.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:12.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:12.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:12.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:12.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:12.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:12.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:12.531 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:12.559 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:12.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:12.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:12.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:12.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:12.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:12.793 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:12.821 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:12.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:12.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:12.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:12.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:12.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:12.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:13.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:13.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:13.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:13.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:13.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:13.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:13.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:13.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:13.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:13.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:13.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:13.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:13.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:13.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:13.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:13.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:13.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:13.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:13.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:13 smithi016 bash[75179]: cluster 2023-10-09T08:59:11.725863+0000 mgr.y (mgr.25421) 649 : cluster 0 pgmap v631: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 1434/127395 objects degraded (1.126%); 13351/127395 objects misplaced (10.480%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T08:59:13.624 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:13 smithi016 bash[77892]: cluster 2023-10-09T08:59:11.725863+0000 mgr.y (mgr.25421) 649 : cluster 0 pgmap v631: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 1434/127395 objects degraded (1.126%); 13351/127395 objects misplaced (10.480%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T08:59:13.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:13.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:13.726 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:13.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:13 smithi094 bash[59677]: cluster 2023-10-09T08:59:11.725863+0000 mgr.y (mgr.25421) 649 : cluster 0 pgmap v631: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 1434/127395 objects degraded (1.126%); 13351/127395 objects misplaced (10.480%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T08:59:13.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:13.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:13.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:13.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:13.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:13.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:14.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:14.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:14.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:14.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:14.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:14.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:14.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:14.425 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:14.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:14.554 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:14.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:14.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:14.689 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:14 smithi094 bash[59677]: cluster 2023-10-09T08:59:14.465627+0000 mon.a (mon.0) 859 : cluster 3 Health check update: Degraded data redundancy: 1434/127395 objects degraded (1.126%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:14.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:14.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:14 smithi016 bash[75179]: cluster 2023-10-09T08:59:14.465627+0000 mon.a (mon.0) 859 : cluster 3 Health check update: Degraded data redundancy: 1434/127395 objects degraded (1.126%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:14.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:14 smithi016 bash[77892]: cluster 2023-10-09T08:59:14.465627+0000 mon.a (mon.0) 859 : cluster 3 Health check update: Degraded data redundancy: 1434/127395 objects degraded (1.126%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T08:59:14.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:14.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:14.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:14.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:14.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:14.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:15.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:15.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:15.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:15.215 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:15.242 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:15.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:15.345 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:15.370 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:15.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:15.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:15.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:15.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:15.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:15.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:15.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:15.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:15.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:15.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:15 smithi016 bash[75179]: cluster 2023-10-09T08:59:13.726477+0000 mgr.y (mgr.25421) 650 : cluster 0 pgmap v632: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 582 op/s; 1512/123222 objects degraded (1.227%); 12661/123222 objects misplaced (10.275%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:59:15.742 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:59:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:59:15] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.33.4" 2023-10-09T08:59:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:15 smithi016 bash[77892]: cluster 2023-10-09T08:59:13.726477+0000 mgr.y (mgr.25421) 650 : cluster 0 pgmap v632: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 582 op/s; 1512/123222 objects degraded (1.227%); 12661/123222 objects misplaced (10.275%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:59:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:15 smithi094 bash[59677]: cluster 2023-10-09T08:59:13.726477+0000 mgr.y (mgr.25421) 650 : cluster 0 pgmap v632: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 582 op/s; 1512/123222 objects degraded (1.227%); 12661/123222 objects misplaced (10.275%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T08:59:15.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:15.771 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:15.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:15.874 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:15.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:15.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:15.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:16.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:16.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:16.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:16.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:16.160 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:16.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:16.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:16.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:16.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:16.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:16.565 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:16.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:16.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:16.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:16.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:16.828 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:16.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:16.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:16.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:17.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:17.094 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:17.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:17.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:17.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:17.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:17.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:17.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:17.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:17.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:17.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:17.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:17.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:17.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:17.599 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:17 smithi016 bash[75179]: cluster 2023-10-09T08:59:15.727040+0000 mgr.y (mgr.25421) 651 : cluster 0 pgmap v633: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 1512/122691 objects degraded (1.232%); 12661/122691 objects misplaced (10.319%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T08:59:17.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:17.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:17.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:17.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:17 smithi094 bash[59677]: cluster 2023-10-09T08:59:15.727040+0000 mgr.y (mgr.25421) 651 : cluster 0 pgmap v633: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 1512/122691 objects degraded (1.232%); 12661/122691 objects misplaced (10.319%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T08:59:17.758 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:17.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:17.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:17.872 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:17 smithi016 bash[77892]: cluster 2023-10-09T08:59:15.727040+0000 mgr.y (mgr.25421) 651 : cluster 0 pgmap v633: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 1512/122691 objects degraded (1.232%); 12661/122691 objects misplaced (10.319%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T08:59:17.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:17.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:17.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:18.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:18.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:18.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:18.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:18.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:18.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:18.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:18.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:18.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:18.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:18.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:18.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:18.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:18.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:18.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:18.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:18.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:18.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:18.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:18.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:18.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:18 smithi016 bash[75179]: cluster 2023-10-09T08:59:17.727724+0000 mgr.y (mgr.25421) 652 : cluster 0 pgmap v634: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 585 op/s; 1552/118599 objects degraded (1.309%); 12485/118599 objects misplaced (10.527%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T08:59:18.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:18 smithi016 bash[77892]: cluster 2023-10-09T08:59:17.727724+0000 mgr.y (mgr.25421) 652 : cluster 0 pgmap v634: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 585 op/s; 1552/118599 objects degraded (1.309%); 12485/118599 objects misplaced (10.527%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T08:59:18.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:18.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:18.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:18.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:18.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:18 smithi094 bash[59677]: cluster 2023-10-09T08:59:17.727724+0000 mgr.y (mgr.25421) 652 : cluster 0 pgmap v634: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 585 op/s; 1552/118599 objects degraded (1.309%); 12485/118599 objects misplaced (10.527%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T08:59:19.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:19.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:19.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:19.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:19.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:19.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:19.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:19.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:19.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:19.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:19.541 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:19.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:19.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:19.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:19 smithi016 bash[75179]: audit 2023-10-09T08:59:19.224097+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:19.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:19 smithi016 bash[75179]: cluster 2023-10-09T08:59:19.466857+0000 mon.a (mon.0) 861 : cluster 3 Health check update: Degraded data redundancy: 1552/118599 objects degraded (1.309%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:59:19.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:19.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:19.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:19.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:19.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:19.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:19.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:19 smithi016 bash[77892]: audit 2023-10-09T08:59:19.224097+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:19.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:19 smithi016 bash[77892]: cluster 2023-10-09T08:59:19.466857+0000 mon.a (mon.0) 861 : cluster 3 Health check update: Degraded data redundancy: 1552/118599 objects degraded (1.309%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:59:19.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:19.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:19.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:19 smithi094 bash[59677]: audit 2023-10-09T08:59:19.224097+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:19.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:19 smithi094 bash[59677]: cluster 2023-10-09T08:59:19.466857+0000 mon.a (mon.0) 861 : cluster 3 Health check update: Degraded data redundancy: 1552/118599 objects degraded (1.309%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:59:20.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:20.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:20.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:20.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:20.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:20.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:20.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:20.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:20.373 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:20.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:20.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:20.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:20.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:20.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:20.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:20.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:20.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:20.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:20 smithi094 bash[59677]: cluster 2023-10-09T08:59:19.728341+0000 mgr.y (mgr.25421) 653 : cluster 0 pgmap v635: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 573 op/s; 1658/114765 objects degraded (1.445%); 11750/114765 objects misplaced (10.238%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T08:59:20.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:20.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:20.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:20.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:20.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:20.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:20 smithi016 bash[77892]: cluster 2023-10-09T08:59:19.728341+0000 mgr.y (mgr.25421) 653 : cluster 0 pgmap v635: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 573 op/s; 1658/114765 objects degraded (1.445%); 11750/114765 objects misplaced (10.238%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T08:59:20.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:20 smithi016 bash[75179]: cluster 2023-10-09T08:59:19.728341+0000 mgr.y (mgr.25421) 653 : cluster 0 pgmap v635: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 573 op/s; 1658/114765 objects degraded (1.445%); 11750/114765 objects misplaced (10.238%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T08:59:20.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:20.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:20.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:21.040 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:21.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:21.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:21.186 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:59:21.193 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:21.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:21.250 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:59:21.291 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:59:21.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:21.335 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:59:21.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:21.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:21.400 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:59:21.441 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:59:21.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:21.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:21.486 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:59:21.531 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:59:21.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T08:59:21.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:21 smithi016 bash[75179]: audit 2023-10-09T08:59:21.137474+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.16:0/522387067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:59:21.557 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:21 smithi016 bash[77892]: audit 2023-10-09T08:59:21.137474+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.16:0/522387067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:59:21.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:21.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:21.709 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:21.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:21.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:21.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:21.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:21.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:21.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:21.969 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:21 smithi094 bash[59677]: audit 2023-10-09T08:59:21.137474+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.16:0/522387067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:59:21.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:21.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:21.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:22.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:22.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:22.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:22.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:22.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:22.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:22.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:22.355 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:22.381 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:22.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:22.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:22.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:22.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:22.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:22.591 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T08:59:22.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:22.634 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:22 smithi094 bash[59677]: cluster 2023-10-09T08:59:21.729031+0000 mgr.y (mgr.25421) 654 : cluster 0 pgmap v636: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 1739/110505 objects degraded (1.574%); 11553/110505 objects misplaced (10.455%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T08:59:22.634 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:22 smithi094 bash[59677]: audit 2023-10-09T08:59:22.447480+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.16:0/116902657' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:59:22.635 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:22 smithi094 bash[59677]: audit 2023-10-09T08:59:22.448467+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:59:22.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:22.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:22.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:22.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:22 smithi016 bash[75179]: cluster 2023-10-09T08:59:21.729031+0000 mgr.y (mgr.25421) 654 : cluster 0 pgmap v636: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 1739/110505 objects degraded (1.574%); 11553/110505 objects misplaced (10.455%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T08:59:22.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:22 smithi016 bash[75179]: audit 2023-10-09T08:59:22.447480+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.16:0/116902657' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:59:22.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:22 smithi016 bash[75179]: audit 2023-10-09T08:59:22.448467+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:59:22.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:22 smithi016 bash[77892]: cluster 2023-10-09T08:59:21.729031+0000 mgr.y (mgr.25421) 654 : cluster 0 pgmap v636: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 1739/110505 objects degraded (1.574%); 11553/110505 objects misplaced (10.455%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T08:59:22.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:22 smithi016 bash[77892]: audit 2023-10-09T08:59:22.447480+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.16:0/116902657' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:59:22.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:22 smithi016 bash[77892]: audit 2023-10-09T08:59:22.448467+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T08:59:22.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:22.793 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T08:59:22.793 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T08:59:22.793 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T08:59:22.794 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T08:59:22.794 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T08:59:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:22.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:22.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:22.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:22.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:22.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:23.056 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:23.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:23.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:23.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:23.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:23.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:23.318 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:23.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:23.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:23.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:23.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:23.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:23.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:23.578 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:23.579 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: cluster 2023-10-09T08:59:22.556689+0000 mon.a (mon.0) 863 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:59:23.579 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: cluster 2023-10-09T08:59:22.565184+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.579 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: audit 2023-10-09T08:59:22.570765+0000 mon.a (mon.0) 864 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:59:23.580 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: cluster 2023-10-09T08:59:22.570902+0000 mon.a (mon.0) 865 : cluster 0 osdmap e152: 8 total, 8 up, 6 in 2023-10-09T08:59:23.580 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: cluster 2023-10-09T08:59:22.583961+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.580 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: cluster 2023-10-09T08:59:22.584008+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.580 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: cluster 2023-10-09T08:59:22.646546+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.586 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.6. 2023-10-09T08:59:23.608 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:23.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:23.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:23.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:23.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: cluster 2023-10-09T08:59:22.556689+0000 mon.a (mon.0) 863 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:59:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: cluster 2023-10-09T08:59:22.565184+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: audit 2023-10-09T08:59:22.570765+0000 mon.a (mon.0) 864 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:59:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: cluster 2023-10-09T08:59:22.570902+0000 mon.a (mon.0) 865 : cluster 0 osdmap e152: 8 total, 8 up, 6 in 2023-10-09T08:59:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: cluster 2023-10-09T08:59:22.583961+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: cluster 2023-10-09T08:59:22.584008+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: cluster 2023-10-09T08:59:22.646546+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: audit 2023-10-09T08:59:23.399602+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.16:0/2707274906' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: audit 2023-10-09T08:59:23.400656+0000 mon.a (mon.0) 866 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[77892]: cluster 2023-10-09T08:59:23.400746+0000 mon.a (mon.0) 867 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: debug 2023-10-09T08:59:23.396+0000 7fa046a4d700 -1 mon.a@0(leader).osd e152 definitely_dead 0 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: cluster 2023-10-09T08:59:22.556689+0000 mon.a (mon.0) 863 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: cluster 2023-10-09T08:59:22.565184+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: audit 2023-10-09T08:59:22.570765+0000 mon.a (mon.0) 864 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T08:59:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: cluster 2023-10-09T08:59:22.570902+0000 mon.a (mon.0) 865 : cluster 0 osdmap e152: 8 total, 8 up, 6 in 2023-10-09T08:59:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: cluster 2023-10-09T08:59:22.583961+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: cluster 2023-10-09T08:59:22.584008+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: cluster 2023-10-09T08:59:22.646546+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T08:59:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: audit 2023-10-09T08:59:23.399602+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.16:0/2707274906' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T08:59:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: audit 2023-10-09T08:59:23.400656+0000 mon.a (mon.0) 866 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T08:59:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:23 smithi016 bash[75179]: cluster 2023-10-09T08:59:23.400746+0000 mon.a (mon.0) 867 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T08:59:23.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:23.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:23.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:23.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:23.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:23.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: audit 2023-10-09T08:59:23.399602+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.16:0/2707274906' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T08:59:23.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: audit 2023-10-09T08:59:23.400656+0000 mon.a (mon.0) 866 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T08:59:23.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:23 smithi094 bash[59677]: cluster 2023-10-09T08:59:23.400746+0000 mon.a (mon.0) 867 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T08:59:24.000 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:24.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:24.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:24.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:24.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:24.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:24.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:24.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:24.501 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:24.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:24.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:24.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:24.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:24 smithi094 bash[59677]: cluster 2023-10-09T08:59:23.567627+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:24 smithi094 bash[59677]: audit 2023-10-09T08:59:23.570576+0000 mon.a (mon.0) 868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T08:59:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:24 smithi094 bash[59677]: cluster 2023-10-09T08:59:23.570690+0000 mon.a (mon.0) 869 : cluster 0 osdmap e153: 8 total, 8 up, 5 in 2023-10-09T08:59:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:24 smithi094 bash[59677]: cluster 2023-10-09T08:59:23.581093+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:24 smithi094 bash[59677]: cluster 2023-10-09T08:59:23.729928+0000 mgr.y (mgr.25421) 655 : cluster 0 pgmap v639: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 1841/106572 objects degraded (1.727%); 10929/106572 objects misplaced (10.255%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T08:59:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:24 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.467902+0000 mon.a (mon.0) 870 : cluster 3 Health check update: Degraded data redundancy: 1739/110505 objects degraded (1.574%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:59:24.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:24 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.473319+0000 mon.a (mon.0) 871 : cluster 0 osdmap e154: 8 total, 8 up, 5 in 2023-10-09T08:59:24.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:24.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:24.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:24.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[75179]: cluster 2023-10-09T08:59:23.567627+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:24.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[75179]: audit 2023-10-09T08:59:23.570576+0000 mon.a (mon.0) 868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T08:59:24.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[75179]: cluster 2023-10-09T08:59:23.570690+0000 mon.a (mon.0) 869 : cluster 0 osdmap e153: 8 total, 8 up, 5 in 2023-10-09T08:59:24.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[75179]: cluster 2023-10-09T08:59:23.581093+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:24.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[75179]: cluster 2023-10-09T08:59:23.729928+0000 mgr.y (mgr.25421) 655 : cluster 0 pgmap v639: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 1841/106572 objects degraded (1.727%); 10929/106572 objects misplaced (10.255%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T08:59:24.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.467902+0000 mon.a (mon.0) 870 : cluster 3 Health check update: Degraded data redundancy: 1739/110505 objects degraded (1.574%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:59:24.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.473319+0000 mon.a (mon.0) 871 : cluster 0 osdmap e154: 8 total, 8 up, 5 in 2023-10-09T08:59:24.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[77892]: cluster 2023-10-09T08:59:23.567627+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:24.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[77892]: audit 2023-10-09T08:59:23.570576+0000 mon.a (mon.0) 868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T08:59:24.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[77892]: cluster 2023-10-09T08:59:23.570690+0000 mon.a (mon.0) 869 : cluster 0 osdmap e153: 8 total, 8 up, 5 in 2023-10-09T08:59:24.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[77892]: cluster 2023-10-09T08:59:23.581093+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:24.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[77892]: cluster 2023-10-09T08:59:23.729928+0000 mgr.y (mgr.25421) 655 : cluster 0 pgmap v639: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 1841/106572 objects degraded (1.727%); 10929/106572 objects misplaced (10.255%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T08:59:24.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.467902+0000 mon.a (mon.0) 870 : cluster 3 Health check update: Degraded data redundancy: 1739/110505 objects degraded (1.574%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T08:59:24.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:24 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.473319+0000 mon.a (mon.0) 871 : cluster 0 osdmap e154: 8 total, 8 up, 5 in 2023-10-09T08:59:24.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:24.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:24.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:25.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:25.046 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:25.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:25.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:25.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:25.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:25.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:25.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:25.300 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:25.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:25.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:25.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:25.437 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:25.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:25.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:25.573 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:25.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:25.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:23.961520+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:25.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.471223+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.477010+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.479140+0000 osd.0 (osd.0) 5 : cluster 0 7.9 starting backfill to osd.2 from (0'0,0'0] MAX to 152'10170 2023-10-09T08:59:25.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.479568+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.480297+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.480302+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.481666+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.483092+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.483794+0000 osd.0 (osd.0) 6 : cluster 0 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 152'10278 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:24.485140+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.484949+0000 mon.a (mon.0) 872 : cluster 0 osdmap e155: 8 total, 8 up, 5 in 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:23.961520+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.471223+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.651 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.477010+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.479140+0000 osd.0 (osd.0) 5 : cluster 0 7.9 starting backfill to osd.2 from (0'0,0'0] MAX to 152'10170 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.479568+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.480297+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.480302+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.481666+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.483092+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.483794+0000 osd.0 (osd.0) 6 : cluster 0 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 152'10278 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:24.485140+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.652 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.484949+0000 mon.a (mon.0) 872 : cluster 0 osdmap e155: 8 total, 8 up, 5 in 2023-10-09T08:59:25.677 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:25.707 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:25.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:25.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:25.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:25.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:25.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:25.989 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:23.961520+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.471223+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.477010+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.479140+0000 osd.0 (osd.0) 5 : cluster 0 7.9 starting backfill to osd.2 from (0'0,0'0] MAX to 152'10170 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.479568+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.480297+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.480302+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.481666+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.483092+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.483794+0000 osd.0 (osd.0) 6 : cluster 0 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 152'10278 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:24.485140+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T08:59:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:25 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.484949+0000 mon.a (mon.0) 872 : cluster 0 osdmap e155: 8 total, 8 up, 5 in 2023-10-09T08:59:25.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:59:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:59:25] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.33.4" 2023-10-09T08:59:26.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:26.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:26.106 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:26.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:26.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:26.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:26.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:26.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:26.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:26.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:26.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:26.549 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:26.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.481659+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.483347+0000 osd.1 (osd.1) 2 : cluster 0 7.1 starting backfill to osd.5 from (0'0,0'0] MAX to 152'10242 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.483892+0000 osd.1 (osd.1) 3 : cluster 0 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 152'10438 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.484141+0000 osd.1 (osd.1) 4 : cluster 0 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 152'10358 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.488675+0000 osd.2 (osd.2) 1 : cluster 0 7.c starting backfill to osd.1 from (0'0,0'0] MAX to 152'10245 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.488940+0000 osd.0 (osd.0) 7 : cluster 0 7.7 starting backfill to osd.2 from (0'0,0'0] MAX to 152'10605 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.489200+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.489475+0000 osd.3 (osd.3) 3 : cluster 0 7.3 starting backfill to osd.1 from (0'0,0'0] MAX to 152'10060 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.543089+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.691543+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[75179]: cluster 2023-10-09T08:59:25.730416+0000 mgr.y (mgr.25421) 656 : cluster 0 pgmap v642: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 357 op/s; 1841/106218 objects degraded (1.733%); 10929/106218 objects misplaced (10.289%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T08:59:26.653 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:26.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:26.784 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:26.791 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.481659+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.791 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.483347+0000 osd.1 (osd.1) 2 : cluster 0 7.1 starting backfill to osd.5 from (0'0,0'0] MAX to 152'10242 2023-10-09T08:59:26.791 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.483892+0000 osd.1 (osd.1) 3 : cluster 0 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 152'10438 2023-10-09T08:59:26.791 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.484141+0000 osd.1 (osd.1) 4 : cluster 0 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 152'10358 2023-10-09T08:59:26.791 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.488675+0000 osd.2 (osd.2) 1 : cluster 0 7.c starting backfill to osd.1 from (0'0,0'0] MAX to 152'10245 2023-10-09T08:59:26.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.488940+0000 osd.0 (osd.0) 7 : cluster 0 7.7 starting backfill to osd.2 from (0'0,0'0] MAX to 152'10605 2023-10-09T08:59:26.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.489200+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.489475+0000 osd.3 (osd.3) 3 : cluster 0 7.3 starting backfill to osd.1 from (0'0,0'0] MAX to 152'10060 2023-10-09T08:59:26.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.543089+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.691543+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:26 smithi094 bash[59677]: cluster 2023-10-09T08:59:25.730416+0000 mgr.y (mgr.25421) 656 : cluster 0 pgmap v642: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 357 op/s; 1841/106218 objects degraded (1.733%); 10929/106218 objects misplaced (10.289%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T08:59:26.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:26.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:26.919 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.481659+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.919 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.483347+0000 osd.1 (osd.1) 2 : cluster 0 7.1 starting backfill to osd.5 from (0'0,0'0] MAX to 152'10242 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.483892+0000 osd.1 (osd.1) 3 : cluster 0 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 152'10438 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.484141+0000 osd.1 (osd.1) 4 : cluster 0 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 152'10358 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.488675+0000 osd.2 (osd.2) 1 : cluster 0 7.c starting backfill to osd.1 from (0'0,0'0] MAX to 152'10245 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.488940+0000 osd.0 (osd.0) 7 : cluster 0 7.7 starting backfill to osd.2 from (0'0,0'0] MAX to 152'10605 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.489200+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.489475+0000 osd.3 (osd.3) 3 : cluster 0 7.3 starting backfill to osd.1 from (0'0,0'0] MAX to 152'10060 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.543089+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.691543+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T08:59:26.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:26 smithi016 bash[77892]: cluster 2023-10-09T08:59:25.730416+0000 mgr.y (mgr.25421) 656 : cluster 0 pgmap v642: 89 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 357 op/s; 1841/106218 objects degraded (1.733%); 10929/106218 objects misplaced (10.289%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T08:59:26.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:26.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:26.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:27.046 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:27.072 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:27.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:27.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:27.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:27.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:27.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:27.441 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:27.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:27.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:27.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T08:59:27.598 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:27.702 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:27.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:27.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:27.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:27.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:27.964 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:27.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:27.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:27.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:28.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:28.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:28.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:28.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:28.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:28.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:28.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:28.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:28.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:28.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:28.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T08:59:28.486 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:28.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:28.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:28.613 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:28.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:28.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:28.759 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:28.787 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:28.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:28.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:28.892 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:28.921 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:28.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:28.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:28.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:28 smithi016 bash[75179]: cluster 2023-10-09T08:59:27.731065+0000 mgr.y (mgr.25421) 657 : cluster 0 pgmap v643: 89 pgs: 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 3300/104193 objects degraded (3.167%); 14954/104193 objects misplaced (14.352%); 1.7 MiB/s, 28 objects/s recovering 2023-10-09T08:59:29.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:28 smithi016 bash[77892]: cluster 2023-10-09T08:59:27.731065+0000 mgr.y (mgr.25421) 657 : cluster 0 pgmap v643: 89 pgs: 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 3300/104193 objects degraded (3.167%); 14954/104193 objects misplaced (14.352%); 1.7 MiB/s, 28 objects/s recovering 2023-10-09T08:59:29.025 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:29.038 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:28 smithi094 bash[59677]: cluster 2023-10-09T08:59:27.731065+0000 mgr.y (mgr.25421) 657 : cluster 0 pgmap v643: 89 pgs: 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 3300/104193 objects degraded (3.167%); 14954/104193 objects misplaced (14.352%); 1.7 MiB/s, 28 objects/s recovering 2023-10-09T08:59:29.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:29.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:29.154 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:29.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:29.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:29.284 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:29.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:29.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:29.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:29.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:29.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:29.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:29.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:29.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:29.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:29.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:29.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:29.833 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:29 smithi016 bash[75179]: cluster 2023-10-09T08:59:29.470348+0000 mon.a (mon.0) 873 : cluster 3 Health check update: Degraded data redundancy: 3300/104193 objects degraded (3.167%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:29.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:29 smithi016 bash[77892]: cluster 2023-10-09T08:59:29.470348+0000 mon.a (mon.0) 873 : cluster 3 Health check update: Degraded data redundancy: 3300/104193 objects degraded (3.167%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:29.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:29.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:29.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:29.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:29.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:30.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:30.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:30.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:30.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:30.228 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:29 smithi094 bash[59677]: cluster 2023-10-09T08:59:29.470348+0000 mon.a (mon.0) 873 : cluster 3 Health check update: Degraded data redundancy: 3300/104193 objects degraded (3.167%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:30.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:30.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:30.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:30.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T08:59:30.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:30.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:30.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:30.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:30.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:30.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:30.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:30.673 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:30.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:30.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:30.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:30.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:30.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:30.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:30.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:31.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:31.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:31.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:31.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:31.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:31.246 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:31.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:31.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:31.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:31.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:31.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:31.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:31.669 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:31.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:31 smithi094 bash[59677]: cluster 2023-10-09T08:59:29.731791+0000 mgr.y (mgr.25421) 658 : cluster 0 pgmap v644: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 3308/101052 objects degraded (3.274%); 18166/101052 objects misplaced (17.977%); 7.1 MiB/s, 1 keys/s, 114 objects/s recovering 2023-10-09T08:59:31.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:31 smithi016 bash[75179]: cluster 2023-10-09T08:59:29.731791+0000 mgr.y (mgr.25421) 658 : cluster 0 pgmap v644: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 3308/101052 objects degraded (3.274%); 18166/101052 objects misplaced (17.977%); 7.1 MiB/s, 1 keys/s, 114 objects/s recovering 2023-10-09T08:59:31.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:31 smithi016 bash[77892]: cluster 2023-10-09T08:59:29.731791+0000 mgr.y (mgr.25421) 658 : cluster 0 pgmap v644: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 3308/101052 objects degraded (3.274%); 18166/101052 objects misplaced (17.977%); 7.1 MiB/s, 1 keys/s, 114 objects/s recovering 2023-10-09T08:59:31.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:31.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:31.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:31.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:31.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:31.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:32.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:32.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:32.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:32.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.227 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:32.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:32.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:32.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:32.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:32.436 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:59:32.436 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:59:32.466 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.493 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:32.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:32.596 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:32.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:32.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:32.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:32.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:32.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:32.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.896 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:32.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:32.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:33.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:33.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:33.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:33.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:33.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:33.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:33.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:33.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:33.436 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5] out_osds: [7, 4, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T08:59:33.436 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T08:59:33.436 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T08:59:33.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T08:59:33.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:33.552 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:33.656 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:33.678 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:33.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:33 smithi094 bash[59677]: cluster 2023-10-09T08:59:31.732453+0000 mgr.y (mgr.25421) 659 : cluster 0 pgmap v645: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 2.0 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 319 op/s; 3125/98892 objects degraded (3.160%); 18166/98892 objects misplaced (18.370%); 6.9 MiB/s, 1 keys/s, 110 objects/s recovering 2023-10-09T08:59:33.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:33 smithi016 bash[75179]: cluster 2023-10-09T08:59:31.732453+0000 mgr.y (mgr.25421) 659 : cluster 0 pgmap v645: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 2.0 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 319 op/s; 3125/98892 objects degraded (3.160%); 18166/98892 objects misplaced (18.370%); 6.9 MiB/s, 1 keys/s, 110 objects/s recovering 2023-10-09T08:59:33.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:33 smithi016 bash[77892]: cluster 2023-10-09T08:59:31.732453+0000 mgr.y (mgr.25421) 659 : cluster 0 pgmap v645: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 2.0 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 319 op/s; 3125/98892 objects degraded (3.160%); 18166/98892 objects misplaced (18.370%); 6.9 MiB/s, 1 keys/s, 110 objects/s recovering 2023-10-09T08:59:33.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:33.810 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:33.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:33.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:33.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:33.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:34.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:34.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:34.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:34.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:34.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:34.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:34.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:34.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:34.473 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:59:34.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:34.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:34.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:34.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:34.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:34.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:34.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:34.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:34.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:34.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:34.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:35.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:35.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:35.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:35.168 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:35.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:35.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[75179]: cluster 2023-10-09T08:59:33.736263+0000 mgr.y (mgr.25421) 660 : cluster 0 pgmap v646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.9 GiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 3125/92496 objects degraded (3.379%); 16691/92496 objects misplaced (18.045%); 6.5 MiB/s, 1 keys/s, 105 objects/s recovering 2023-10-09T08:59:35.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[75179]: audit 2023-10-09T08:59:34.235994+0000 mon.a (mon.0) 874 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:59:35.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:35.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:35.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:35.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:35.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:35 smithi094 bash[59677]: cluster 2023-10-09T08:59:33.736263+0000 mgr.y (mgr.25421) 660 : cluster 0 pgmap v646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.9 GiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 3125/92496 objects degraded (3.379%); 16691/92496 objects misplaced (18.045%); 6.5 MiB/s, 1 keys/s, 105 objects/s recovering 2023-10-09T08:59:35.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:35 smithi094 bash[59677]: audit 2023-10-09T08:59:34.235994+0000 mon.a (mon.0) 874 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:59:35.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:35 smithi094 bash[59677]: audit 2023-10-09T08:59:34.237772+0000 mon.a (mon.0) 875 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:35.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:35 smithi094 bash[59677]: cluster 2023-10-09T08:59:34.471462+0000 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 3125/98892 objects degraded (3.160%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:35.434 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:35.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[75179]: audit 2023-10-09T08:59:34.237772+0000 mon.a (mon.0) 875 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:35.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[75179]: cluster 2023-10-09T08:59:34.471462+0000 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 3125/98892 objects degraded (3.160%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[77892]: cluster 2023-10-09T08:59:33.736263+0000 mgr.y (mgr.25421) 660 : cluster 0 pgmap v646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.9 GiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 3125/92496 objects degraded (3.379%); 16691/92496 objects misplaced (18.045%); 6.5 MiB/s, 1 keys/s, 105 objects/s recovering 2023-10-09T08:59:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[77892]: audit 2023-10-09T08:59:34.235994+0000 mon.a (mon.0) 874 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T08:59:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[77892]: audit 2023-10-09T08:59:34.237772+0000 mon.a (mon.0) 875 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[77892]: cluster 2023-10-09T08:59:34.471462+0000 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 3125/98892 objects degraded (3.160%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:35.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:35.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:35.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:35.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:35.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:35.801 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:35.842 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:35.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:35.953 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:59:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:59:35] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.33.4" 2023-10-09T08:59:35.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:35.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:36.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:36.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:36.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:36.241 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:36.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:36.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:36.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:36.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:36.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:36.614 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:36.639 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:36.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:36.742 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:36.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:36.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:36.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:36.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:37.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:37.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:37.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:37.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:37.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:37.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:37.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:37.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:37.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:37.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:37.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:37.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:37 smithi094 bash[59677]: cluster 2023-10-09T08:59:35.737036+0000 mgr.y (mgr.25421) 661 : cluster 0 pgmap v647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.9 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 460 op/s; 3125/92037 objects degraded (3.395%); 16691/92037 objects misplaced (18.135%); 5.9 MiB/s, 0 keys/s, 94 objects/s recovering 2023-10-09T08:59:37.524 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T08:59:37.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:37.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:37.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:37.693 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:37 smithi016 bash[77892]: cluster 2023-10-09T08:59:35.737036+0000 mgr.y (mgr.25421) 661 : cluster 0 pgmap v647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.9 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 460 op/s; 3125/92037 objects degraded (3.395%); 16691/92037 objects misplaced (18.135%); 5.9 MiB/s, 0 keys/s, 94 objects/s recovering 2023-10-09T08:59:37.711 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:37.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:37.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:37.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:37.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:37.957 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:37 smithi016 bash[75179]: cluster 2023-10-09T08:59:35.737036+0000 mgr.y (mgr.25421) 661 : cluster 0 pgmap v647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.9 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 460 op/s; 3125/92037 objects degraded (3.395%); 16691/92037 objects misplaced (18.135%); 5.9 MiB/s, 0 keys/s, 94 objects/s recovering 2023-10-09T08:59:37.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:37.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:37.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:38.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:38.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:38.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.249 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:38.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:38.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:38.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:38.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:38.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:38.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:38.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:38.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:38.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.650 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:38.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:38.754 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:38.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:38.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:38.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:39.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:39.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:39.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:39.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:39.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:39.290 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:39.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:39.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:39.430 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:39 smithi016 bash[75179]: cluster 2023-10-09T08:59:37.737707+0000 mgr.y (mgr.25421) 662 : cluster 0 pgmap v648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.8 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 483 op/s; 2958/88797 objects degraded (3.331%); 16145/88797 objects misplaced (18.182%); 5.9 MiB/s, 0 keys/s, 94 objects/s recovering 2023-10-09T08:59:39.430 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:39 smithi016 bash[77892]: cluster 2023-10-09T08:59:37.737707+0000 mgr.y (mgr.25421) 662 : cluster 0 pgmap v648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.8 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 483 op/s; 2958/88797 objects degraded (3.331%); 16145/88797 objects misplaced (18.182%); 5.9 MiB/s, 0 keys/s, 94 objects/s recovering 2023-10-09T08:59:39.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:39.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:39 smithi094 bash[59677]: cluster 2023-10-09T08:59:37.737707+0000 mgr.y (mgr.25421) 662 : cluster 0 pgmap v648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.8 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 483 op/s; 2958/88797 objects degraded (3.331%); 16145/88797 objects misplaced (18.182%); 5.9 MiB/s, 0 keys/s, 94 objects/s recovering 2023-10-09T08:59:39.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:39.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:39.683 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:39.707 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:39.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:39.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:39.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:39.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:39.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:39.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:39.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:40.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:40.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:40.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:40.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:40.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:40.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:40.387 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:40.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:40.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:40.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:40 smithi094 bash[59677]: cluster 2023-10-09T08:59:39.472610+0000 mon.a (mon.0) 877 : cluster 3 Health check update: Degraded data redundancy: 2958/88797 objects degraded (3.331%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:40 smithi016 bash[75179]: cluster 2023-10-09T08:59:39.472610+0000 mon.a (mon.0) 877 : cluster 3 Health check update: Degraded data redundancy: 2958/88797 objects degraded (3.331%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:40 smithi016 bash[77892]: cluster 2023-10-09T08:59:39.472610+0000 mon.a (mon.0) 877 : cluster 3 Health check update: Degraded data redundancy: 2958/88797 objects degraded (3.331%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:40.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:40.554 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:40.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:40.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:40.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:40.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:40.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:40.828 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:40.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:40.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:40.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:40.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:40.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:41.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:41.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:41.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:41.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:41.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:41.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:41.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:41.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:41.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:41.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:41.517 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:41.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:41.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:41.620 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:41.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:41 smithi016 bash[75179]: cluster 2023-10-09T08:59:39.738429+0000 mgr.y (mgr.25421) 663 : cluster 0 pgmap v649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 560 op/s; 3009/84009 objects degraded (3.582%); 15153/84009 objects misplaced (18.037%); 6.1 MiB/s, 0 keys/s, 97 objects/s recovering 2023-10-09T08:59:41.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:41 smithi016 bash[77892]: cluster 2023-10-09T08:59:39.738429+0000 mgr.y (mgr.25421) 663 : cluster 0 pgmap v649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 560 op/s; 3009/84009 objects degraded (3.582%); 15153/84009 objects misplaced (18.037%); 6.1 MiB/s, 0 keys/s, 97 objects/s recovering 2023-10-09T08:59:41.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:41.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:41.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:41.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:41 smithi094 bash[59677]: cluster 2023-10-09T08:59:39.738429+0000 mgr.y (mgr.25421) 663 : cluster 0 pgmap v649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 560 op/s; 3009/84009 objects degraded (3.582%); 15153/84009 objects misplaced (18.037%); 6.1 MiB/s, 0 keys/s, 97 objects/s recovering 2023-10-09T08:59:41.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:41.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:41.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:41.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:41.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:41.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:41.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:41.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:42.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:42.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:42.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:42.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:42.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:42.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:42.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:42.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:42.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:42.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:42.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:42.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:42.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:42.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:42.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:43.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:43.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:43.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:43.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:43.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:43.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:43.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:43.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:43.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:43.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:43.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:43.517 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:43.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:43.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:43.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:43.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:43 smithi016 bash[75179]: cluster 2023-10-09T08:59:41.739059+0000 mgr.y (mgr.25421) 664 : cluster 0 pgmap v650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 534 op/s; 2833/81816 objects degraded (3.463%); 15153/81816 objects misplaced (18.521%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T08:59:43.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:43 smithi016 bash[77892]: cluster 2023-10-09T08:59:41.739059+0000 mgr.y (mgr.25421) 664 : cluster 0 pgmap v650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 534 op/s; 2833/81816 objects degraded (3.463%); 15153/81816 objects misplaced (18.521%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T08:59:43.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:43.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:43 smithi094 bash[59677]: cluster 2023-10-09T08:59:41.739059+0000 mgr.y (mgr.25421) 664 : cluster 0 pgmap v650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 534 op/s; 2833/81816 objects degraded (3.463%); 15153/81816 objects misplaced (18.521%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T08:59:43.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:43.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:43.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:43.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:43.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:43.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:43.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:44.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:44.075 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:44.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:44.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:44.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:44.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:44.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:44.309 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:44.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:44.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:44.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:44.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:44.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:44.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:44.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:44.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:44.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:44.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:44.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:44.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:44 smithi094 bash[59677]: cluster 2023-10-09T08:59:44.473772+0000 mon.a (mon.0) 878 : cluster 3 Health check update: Degraded data redundancy: 2833/81816 objects degraded (3.463%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:44.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:44.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:44.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:45.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:44 smithi016 bash[75179]: cluster 2023-10-09T08:59:44.473772+0000 mon.a (mon.0) 878 : cluster 3 Health check update: Degraded data redundancy: 2833/81816 objects degraded (3.463%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:44 smithi016 bash[77892]: cluster 2023-10-09T08:59:44.473772+0000 mon.a (mon.0) 878 : cluster 3 Health check update: Degraded data redundancy: 2833/81816 objects degraded (3.463%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:45.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:45.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:45.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:45.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:45.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:45.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:45.489 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:45.517 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:45.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:45.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:45.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:45.630 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:45 smithi094 bash[59677]: cluster 2023-10-09T08:59:43.739647+0000 mgr.y (mgr.25421) 665 : cluster 0 pgmap v651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 648 op/s; 2924/75525 objects degraded (3.872%); 13630/75525 objects misplaced (18.047%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T08:59:45.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:45.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:45.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:45.751 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:45.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:45.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:45.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:45.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:45 smithi016 bash[77892]: cluster 2023-10-09T08:59:43.739647+0000 mgr.y (mgr.25421) 665 : cluster 0 pgmap v651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 648 op/s; 2924/75525 objects degraded (3.872%); 13630/75525 objects misplaced (18.047%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T08:59:45.886 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:59:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:59:45] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.33.4" 2023-10-09T08:59:45.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:45 smithi016 bash[75179]: cluster 2023-10-09T08:59:43.739647+0000 mgr.y (mgr.25421) 665 : cluster 0 pgmap v651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 648 op/s; 2924/75525 objects degraded (3.872%); 13630/75525 objects misplaced (18.047%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T08:59:45.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:45.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:45.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:46.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:46.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:46.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:46.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:46.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:46.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:46.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:46.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:46.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:46.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:46.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:46.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:46.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:46.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:46.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:46.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:46.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:46.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:46.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:46 smithi016 bash[75179]: cluster 2023-10-09T08:59:45.740204+0000 mgr.y (mgr.25421) 666 : cluster 0 pgmap v652: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 487 op/s; 2924/74934 objects degraded (3.902%); 13630/74934 objects misplaced (18.189%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T08:59:46.747 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:46 smithi016 bash[77892]: cluster 2023-10-09T08:59:45.740204+0000 mgr.y (mgr.25421) 666 : cluster 0 pgmap v652: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 487 op/s; 2924/74934 objects degraded (3.902%); 13630/74934 objects misplaced (18.189%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T08:59:46.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:46.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:46.849 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:46.858 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:46 smithi094 bash[59677]: cluster 2023-10-09T08:59:45.740204+0000 mgr.y (mgr.25421) 666 : cluster 0 pgmap v652: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 487 op/s; 2924/74934 objects degraded (3.902%); 13630/74934 objects misplaced (18.189%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T08:59:46.875 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:46.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:46.977 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:46.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:47.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:47.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:47.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:47.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:47.260 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:47.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:47.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:47.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:47.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:47.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:47.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:47.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:47.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:47.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:47.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:47.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:47.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:48.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:48.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:48.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:48.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:48.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:48.300 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:48.324 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:48.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:48.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:48.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:48.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:48.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:48.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:48.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:48.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:48.840 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:48 smithi016 bash[75179]: cluster 2023-10-09T08:59:47.740939+0000 mgr.y (mgr.25421) 667 : cluster 0 pgmap v653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 2727/71832 objects degraded (3.796%); 13134/71832 objects misplaced (18.284%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T08:59:48.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:48 smithi016 bash[77892]: cluster 2023-10-09T08:59:47.740939+0000 mgr.y (mgr.25421) 667 : cluster 0 pgmap v653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 2727/71832 objects degraded (3.796%); 13134/71832 objects misplaced (18.284%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T08:59:48.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:48.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:48.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:48.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:49.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:49.101 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:48 smithi094 bash[59677]: cluster 2023-10-09T08:59:47.740939+0000 mgr.y (mgr.25421) 667 : cluster 0 pgmap v653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 2727/71832 objects degraded (3.796%); 13134/71832 objects misplaced (18.284%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T08:59:49.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:49.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:49.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:49.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:49.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:49.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:49.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:49.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:49.380 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:49.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:49.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:49.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:49.511 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:49.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:49.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:49.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:49.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:49.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:49.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:49.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:49.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:49.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:49.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:49.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:49.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:49.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:49 smithi016 bash[75179]: audit 2023-10-09T08:59:49.224830+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:50.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:49 smithi016 bash[75179]: cluster 2023-10-09T08:59:49.474873+0000 mon.a (mon.0) 880 : cluster 3 Health check update: Degraded data redundancy: 2727/71832 objects degraded (3.796%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:49 smithi016 bash[77892]: audit 2023-10-09T08:59:49.224830+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:49 smithi016 bash[77892]: cluster 2023-10-09T08:59:49.474873+0000 mon.a (mon.0) 880 : cluster 3 Health check update: Degraded data redundancy: 2727/71832 objects degraded (3.796%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:50.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:50.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:50.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:50.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:50.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:49 smithi094 bash[59677]: audit 2023-10-09T08:59:49.224830+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T08:59:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:49 smithi094 bash[59677]: cluster 2023-10-09T08:59:49.474873+0000 mon.a (mon.0) 880 : cluster 3 Health check update: Degraded data redundancy: 2727/71832 objects degraded (3.796%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:50.277 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:50.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:50.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:50.431 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:50.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:50.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:50.662 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:50.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:50.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:50.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:50.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:50.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:50.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:50.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:50.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:50.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:51.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:51.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:51.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:51.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:51.191 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T08:59:51.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:51.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:51.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:51.299 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T08:59:51.349 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T08:59:51.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:51.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:51.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:51.390 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T08:59:51.435 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T08:59:51.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:51.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T08:59:51.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:51.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:51.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:51.528 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T08:59:51.581 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T08:59:51.581 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T08:59:51.581 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T08:59:51.592 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.592 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.592 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.592 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.592 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.593 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.593 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.593 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T08:59:51.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:51.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:51 smithi016 bash[75179]: cluster 2023-10-09T08:59:49.741813+0000 mgr.y (mgr.25421) 668 : cluster 0 pgmap v654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 604 op/s; 2829/67032 objects degraded (4.220%); 12125/67032 objects misplaced (18.088%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T08:59:51.627 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:51 smithi016 bash[75179]: audit 2023-10-09T08:59:51.126835+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.16:0/2313745792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:59:51.627 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:51 smithi016 bash[77892]: cluster 2023-10-09T08:59:49.741813+0000 mgr.y (mgr.25421) 668 : cluster 0 pgmap v654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 604 op/s; 2829/67032 objects degraded (4.220%); 12125/67032 objects misplaced (18.088%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T08:59:51.627 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:51 smithi016 bash[77892]: audit 2023-10-09T08:59:51.126835+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.16:0/2313745792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:59:51.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:51.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:51.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:51 smithi094 bash[59677]: cluster 2023-10-09T08:59:49.741813+0000 mgr.y (mgr.25421) 668 : cluster 0 pgmap v654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 604 op/s; 2829/67032 objects degraded (4.220%); 12125/67032 objects misplaced (18.088%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T08:59:51.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:51 smithi094 bash[59677]: audit 2023-10-09T08:59:51.126835+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.16:0/2313745792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T08:59:51.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:51.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:51.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:51.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:51.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:51.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:51.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:52.015 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:52.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:52.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:52.153 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:51 smithi016 bash[75179]: debug 2023-10-09T08:59:51.843+0000 7fa046a4d700 -1 mon.a@0(leader).osd e155 definitely_dead 0 2023-10-09T08:59:52.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:52.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:52.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:52.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:52.433 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:52.459 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:52.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:52.527 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.4. 2023-10-09T08:59:52.535 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T08:59:52.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:52.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:52.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:52.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:52 smithi094 bash[59677]: audit 2023-10-09T08:59:51.846548+0000 mon.a (mon.0) 881 : audit 0 from='client.? 172.21.15.16:0/493357236' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T08:59:52.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:52 smithi094 bash[59677]: audit 2023-10-09T08:59:51.995516+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.16:0/3331531994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:59:52.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:52 smithi094 bash[59677]: audit 2023-10-09T08:59:51.996929+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:59:52.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:52.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:52.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:52 smithi016 bash[75179]: audit 2023-10-09T08:59:51.846548+0000 mon.a (mon.0) 881 : audit 0 from='client.? 172.21.15.16:0/493357236' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T08:59:52.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:52 smithi016 bash[75179]: audit 2023-10-09T08:59:51.995516+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.16:0/3331531994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:59:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:52 smithi016 bash[75179]: audit 2023-10-09T08:59:51.996929+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:59:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:52 smithi016 bash[77892]: audit 2023-10-09T08:59:51.846548+0000 mon.a (mon.0) 881 : audit 0 from='client.? 172.21.15.16:0/493357236' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T08:59:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:52 smithi016 bash[77892]: audit 2023-10-09T08:59:51.995516+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.16:0/3331531994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:59:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:52 smithi016 bash[77892]: audit 2023-10-09T08:59:51.996929+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T08:59:52.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:52.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:52.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:52.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:52.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:53.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:53.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:53.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:53.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:53.211 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.243 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:53.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:53.346 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.381 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:53.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:53.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:51.742795+0000 mgr.y (mgr.25421) 669 : cluster 0 pgmap v655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 530 op/s; 2655/64893 objects degraded (4.091%); 12125/64893 objects misplaced (18.685%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.483072+0000 mon.a (mon.0) 883 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.511134+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.511222+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.515178+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.518381+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: audit 2023-10-09T08:59:52.520594+0000 mon.a (mon.0) 884 : audit 1 from='client.? 172.21.15.16:0/493357236' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: audit 2023-10-09T08:59:52.520990+0000 mon.a (mon.0) 885 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.521085+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.521268+0000 mon.a (mon.0) 886 : cluster 0 osdmap e156: 8 total, 8 up, 6 in 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.523970+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[77892]: cluster 2023-10-09T08:59:52.526695+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.522 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:53.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:53.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:53.634 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:51.742795+0000 mgr.y (mgr.25421) 669 : cluster 0 pgmap v655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 530 op/s; 2655/64893 objects degraded (4.091%); 12125/64893 objects misplaced (18.685%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T08:59:53.634 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.483072+0000 mon.a (mon.0) 883 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.511134+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.511222+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.515178+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.518381+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: audit 2023-10-09T08:59:52.520594+0000 mon.a (mon.0) 884 : audit 1 from='client.? 172.21.15.16:0/493357236' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: audit 2023-10-09T08:59:52.520990+0000 mon.a (mon.0) 885 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.521085+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.521268+0000 mon.a (mon.0) 886 : cluster 0 osdmap e156: 8 total, 8 up, 6 in 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.523970+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:53 smithi016 bash[75179]: cluster 2023-10-09T08:59:52.526695+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:53.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:53.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:53.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:53.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:53.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:51.742795+0000 mgr.y (mgr.25421) 669 : cluster 0 pgmap v655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 530 op/s; 2655/64893 objects degraded (4.091%); 12125/64893 objects misplaced (18.685%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.483072+0000 mon.a (mon.0) 883 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.511134+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.511222+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.515178+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.518381+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: audit 2023-10-09T08:59:52.520594+0000 mon.a (mon.0) 884 : audit 1 from='client.? 172.21.15.16:0/493357236' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: audit 2023-10-09T08:59:52.520990+0000 mon.a (mon.0) 885 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T08:59:53.907 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.521085+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.908 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.521268+0000 mon.a (mon.0) 886 : cluster 0 osdmap e156: 8 total, 8 up, 6 in 2023-10-09T08:59:53.908 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.523970+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.908 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:53 smithi094 bash[59677]: cluster 2023-10-09T08:59:52.526695+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:53.926 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:53.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:54.029 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:54.062 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:54.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:54.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:54.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:54.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:54.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:54.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:54.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:54.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:54.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:54.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:54.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.507023+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.507420+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.510491+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.510771+0000 mon.a (mon.0) 887 : cluster 0 osdmap e157: 8 total, 8 up, 6 in 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.512346+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.513672+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.515249+0000 osd.0 (osd.0) 8 : cluster 0 7.7 starting backfill to osd.4 from (0'0,0'0] MAX to 155'11547 2023-10-09T08:59:54.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.515466+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.517003+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.518985+0000 osd.1 (osd.1) 5 : cluster 0 7.1 starting backfill to osd.4 from (140'3153,149'9375] MIN to 155'11167 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.521523+0000 osd.1 (osd.1) 6 : cluster 0 7.b starting backfill to osd.4 from (0'0,0'0] MAX to 155'11374 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:53.743247+0000 mgr.y (mgr.25421) 670 : cluster 0 pgmap v658: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 14 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 1.2 GiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 651 op/s; 1186/59301 objects degraded (2.000%); 1893/59301 objects misplaced (3.192%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[75179]: cluster 2023-10-09T08:59:54.476034+0000 mon.a (mon.0) 888 : cluster 3 Health check update: Degraded data redundancy: 2655/64893 objects degraded (4.091%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.507023+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.507420+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.696 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.510491+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.510771+0000 mon.a (mon.0) 887 : cluster 0 osdmap e157: 8 total, 8 up, 6 in 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.512346+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.513672+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.515249+0000 osd.0 (osd.0) 8 : cluster 0 7.7 starting backfill to osd.4 from (0'0,0'0] MAX to 155'11547 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.515466+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.517003+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.518985+0000 osd.1 (osd.1) 5 : cluster 0 7.1 starting backfill to osd.4 from (140'3153,149'9375] MIN to 155'11167 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.521523+0000 osd.1 (osd.1) 6 : cluster 0 7.b starting backfill to osd.4 from (0'0,0'0] MAX to 155'11374 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:53.743247+0000 mgr.y (mgr.25421) 670 : cluster 0 pgmap v658: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 14 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 1.2 GiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 651 op/s; 1186/59301 objects degraded (2.000%); 1893/59301 objects misplaced (3.192%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T08:59:54.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:54 smithi016 bash[77892]: cluster 2023-10-09T08:59:54.476034+0000 mon.a (mon.0) 888 : cluster 3 Health check update: Degraded data redundancy: 2655/64893 objects degraded (4.091%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:54.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:54.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:54.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:54.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:54.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:54.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:54.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:54.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:54.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:54.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.507023+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.507420+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.510491+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.510771+0000 mon.a (mon.0) 887 : cluster 0 osdmap e157: 8 total, 8 up, 6 in 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.512346+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.513672+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.515249+0000 osd.0 (osd.0) 8 : cluster 0 7.7 starting backfill to osd.4 from (0'0,0'0] MAX to 155'11547 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.515466+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.517003+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.518985+0000 osd.1 (osd.1) 5 : cluster 0 7.1 starting backfill to osd.4 from (140'3153,149'9375] MIN to 155'11167 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.521523+0000 osd.1 (osd.1) 6 : cluster 0 7.b starting backfill to osd.4 from (0'0,0'0] MAX to 155'11374 2023-10-09T08:59:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:53.743247+0000 mgr.y (mgr.25421) 670 : cluster 0 pgmap v658: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 14 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 1.2 GiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 651 op/s; 1186/59301 objects degraded (2.000%); 1893/59301 objects misplaced (3.192%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T08:59:54.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:54 smithi094 bash[59677]: cluster 2023-10-09T08:59:54.476034+0000 mon.a (mon.0) 888 : cluster 3 Health check update: Degraded data redundancy: 2655/64893 objects degraded (4.091%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T08:59:55.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:55.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:55.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:55.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:55.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:55.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:55.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:55.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:55.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:55.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:55.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:55.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:55.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:55.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:55.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:55.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:55.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:55.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:55.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:55.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:55.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:55.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[75179]: cluster 2023-10-09T08:59:54.388709+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:55.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[75179]: cluster 2023-10-09T08:59:54.519756+0000 mon.a (mon.0) 889 : cluster 0 osdmap e158: 8 total, 8 up, 6 in 2023-10-09T08:59:55.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[75179]: cluster 2023-10-09T08:59:54.525294+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[75179]: cluster 2023-10-09T08:59:54.536501+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[75179]: cluster 2023-10-09T08:59:54.584200+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[75179]: audit 2023-10-09T08:59:55.489179+0000 mon.a (mon.0) 890 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:55.908 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[77892]: cluster 2023-10-09T08:59:54.388709+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:55.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[77892]: cluster 2023-10-09T08:59:54.519756+0000 mon.a (mon.0) 889 : cluster 0 osdmap e158: 8 total, 8 up, 6 in 2023-10-09T08:59:55.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[77892]: cluster 2023-10-09T08:59:54.525294+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[77892]: cluster 2023-10-09T08:59:54.536501+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[77892]: cluster 2023-10-09T08:59:54.584200+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[77892]: audit 2023-10-09T08:59:55.489179+0000 mon.a (mon.0) 890 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:55.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:55.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:55.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:55 smithi094 bash[59677]: cluster 2023-10-09T08:59:54.388709+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T08:59:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:55 smithi094 bash[59677]: cluster 2023-10-09T08:59:54.519756+0000 mon.a (mon.0) 889 : cluster 0 osdmap e158: 8 total, 8 up, 6 in 2023-10-09T08:59:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:55 smithi094 bash[59677]: cluster 2023-10-09T08:59:54.525294+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:55 smithi094 bash[59677]: cluster 2023-10-09T08:59:54.536501+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:55 smithi094 bash[59677]: cluster 2023-10-09T08:59:54.584200+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:55 smithi094 bash[59677]: audit 2023-10-09T08:59:55.489179+0000 mon.a (mon.0) 890 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:56.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:56.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:56.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:56.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:56.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:56.214 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 08:59:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:08:59:55] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-09T08:59:56.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:56.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:56.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:56.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:56.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:56.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:56.460 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:56.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:56.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:56.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:56.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:56.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:56.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:56.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:56.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:56.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:56 smithi094 bash[59677]: cluster 2023-10-09T08:59:54.709560+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:56.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:56 smithi094 bash[59677]: cluster 2023-10-09T08:59:55.744028+0000 mgr.y (mgr.25421) 671 : cluster 0 pgmap v660: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 14 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 1.2 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 448 op/s; 1186/58962 objects degraded (2.011%); 1893/58962 objects misplaced (3.211%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T08:59:56.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:56.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:56.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:56.860 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:56.892 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T08:59:56.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:56.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T08:59:56.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:57.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:56 smithi016 bash[75179]: cluster 2023-10-09T08:59:54.709560+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:57.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:56 smithi016 bash[75179]: cluster 2023-10-09T08:59:55.744028+0000 mgr.y (mgr.25421) 671 : cluster 0 pgmap v660: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 14 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 1.2 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 448 op/s; 1186/58962 objects degraded (2.011%); 1893/58962 objects misplaced (3.211%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T08:59:57.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:56 smithi016 bash[77892]: cluster 2023-10-09T08:59:54.709560+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T08:59:57.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:56 smithi016 bash[77892]: cluster 2023-10-09T08:59:55.744028+0000 mgr.y (mgr.25421) 671 : cluster 0 pgmap v660: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 14 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 1.2 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 448 op/s; 1186/58962 objects degraded (2.011%); 1893/58962 objects misplaced (3.211%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T08:59:57.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:57.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:57.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:57.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:57.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:57.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:57.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:57.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:57.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:57.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:57.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:57.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:57.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:57.599 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:57.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:57.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:57.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:57.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:57.838 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:57.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:57.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:57.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:57.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:57.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T08:59:58.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:58.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:58.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:58.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:58.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:58.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:58.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:58.272 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:58.301 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T08:59:58.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T08:59:58.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:58.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T08:59:58.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:58.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:58.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:58.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:58.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:58.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:58.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:58.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:58.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T08:59:58.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:58.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T08:59:58.802 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:58 smithi016 bash[75179]: cluster 2023-10-09T08:59:57.744614+0000 mgr.y (mgr.25421) 672 : cluster 0 pgmap v661: 89 pgs: 1 active+remapped+backfilling, 3 remapped+peering, 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+remapped, 72 active+clean; 1.2 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 3092/56787 objects degraded (5.445%); 3467/56787 objects misplaced (6.105%); 12 MiB/s, 189 objects/s recovering 2023-10-09T08:59:58.803 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:58 smithi016 bash[77892]: cluster 2023-10-09T08:59:57.744614+0000 mgr.y (mgr.25421) 672 : cluster 0 pgmap v661: 89 pgs: 1 active+remapped+backfilling, 3 remapped+peering, 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+remapped, 72 active+clean; 1.2 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 3092/56787 objects degraded (5.445%); 3467/56787 objects misplaced (6.105%); 12 MiB/s, 189 objects/s recovering 2023-10-09T08:59:58.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:58.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T08:59:58.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:58.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T08:59:58.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:58 smithi094 bash[59677]: cluster 2023-10-09T08:59:57.744614+0000 mgr.y (mgr.25421) 672 : cluster 0 pgmap v661: 89 pgs: 1 active+remapped+backfilling, 3 remapped+peering, 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+remapped, 72 active+clean; 1.2 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 3092/56787 objects degraded (5.445%); 3467/56787 objects misplaced (6.105%); 12 MiB/s, 189 objects/s recovering 2023-10-09T08:59:59.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:59.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:59.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:59.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T08:59:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T08:59:59.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:59.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:59.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:59.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:59.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T08:59:59.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T08:59:59.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:59.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:59.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:59.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:59.658 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:59.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:59.765 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:59.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T08:59:59.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T08:59:59.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:59.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T08:59:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:59:59.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T08:59:59.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 08:59:59 smithi094 bash[59677]: cluster 2023-10-09T08:59:59.480035+0000 mon.a (mon.0) 891 : cluster 3 Health check update: Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:00.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:00.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:00.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 08:59:59 smithi016 bash[75179]: cluster 2023-10-09T08:59:59.480035+0000 mon.a (mon.0) 891 : cluster 3 Health check update: Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:00.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:00.069 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 08:59:59 smithi016 bash[77892]: cluster 2023-10-09T08:59:59.480035+0000 mon.a (mon.0) 891 : cluster 3 Health check update: Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:00.169 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:00.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:00:00.201 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:00.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:00.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:00.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:00:00.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:00.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:00.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:00.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:00.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:00.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:00.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:00.609 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:00.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:00.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:00.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:00.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:00.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:00.874 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:00.896 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:00.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:00.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:00.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:01.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[75179]: cluster 2023-10-09T08:59:59.745289+0000 mgr.y (mgr.25421) 673 : cluster 0 pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 1.1 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 3092/52710 objects degraded (5.866%); 4395/52710 objects misplaced (8.338%); 10 MiB/s, 192 objects/s recovering 2023-10-09T09:00:01.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[75179]: cluster 2023-10-09T09:00:00.000087+0000 mon.a (mon.0) 892 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded 2023-10-09T09:00:01.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[75179]: cluster 2023-10-09T09:00:00.000127+0000 mon.a (mon.0) 893 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T09:00:01.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[75179]: cluster 2023-10-09T09:00:00.000140+0000 mon.a (mon.0) 894 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded 2023-10-09T09:00:01.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[75179]: cluster 2023-10-09T09:00:00.000153+0000 mon.a (mon.0) 895 : cluster 3 pg 7.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-10-09T09:00:01.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[77892]: cluster 2023-10-09T08:59:59.745289+0000 mgr.y (mgr.25421) 673 : cluster 0 pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 1.1 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 3092/52710 objects degraded (5.866%); 4395/52710 objects misplaced (8.338%); 10 MiB/s, 192 objects/s recovering 2023-10-09T09:00:01.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[77892]: cluster 2023-10-09T09:00:00.000087+0000 mon.a (mon.0) 892 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded 2023-10-09T09:00:01.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[77892]: cluster 2023-10-09T09:00:00.000127+0000 mon.a (mon.0) 893 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T09:00:01.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[77892]: cluster 2023-10-09T09:00:00.000140+0000 mon.a (mon.0) 894 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded 2023-10-09T09:00:01.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:00 smithi016 bash[77892]: cluster 2023-10-09T09:00:00.000153+0000 mon.a (mon.0) 895 : cluster 3 pg 7.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-10-09T09:00:01.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:01.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:01.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:01.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:01.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:01.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:01.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:01.197 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:00 smithi094 bash[59677]: cluster 2023-10-09T08:59:59.745289+0000 mgr.y (mgr.25421) 673 : cluster 0 pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 1.1 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 3092/52710 objects degraded (5.866%); 4395/52710 objects misplaced (8.338%); 10 MiB/s, 192 objects/s recovering 2023-10-09T09:00:01.198 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:00 smithi094 bash[59677]: cluster 2023-10-09T09:00:00.000087+0000 mon.a (mon.0) 892 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded 2023-10-09T09:00:01.198 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:00 smithi094 bash[59677]: cluster 2023-10-09T09:00:00.000127+0000 mon.a (mon.0) 893 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T09:00:01.198 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:00 smithi094 bash[59677]: cluster 2023-10-09T09:00:00.000140+0000 mon.a (mon.0) 894 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 3092/56787 objects degraded (5.445%), 1 pg degraded 2023-10-09T09:00:01.198 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:00 smithi094 bash[59677]: cluster 2023-10-09T09:00:00.000153+0000 mon.a (mon.0) 895 : cluster 3 pg 7.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-10-09T09:00:01.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:01.306 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:01.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:01.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:01.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:01.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:01.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:01.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:01.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:01.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:01.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:01.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:01.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:01.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:01.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:01.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:01.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:02.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:02.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:02.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:02.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:02.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:02.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:02.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:02.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:02.339 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:02.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:02.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:02.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:02.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:02.555 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:02.589 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:02.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:02.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:02.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:02.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:02.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:02.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:02.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:02.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:02.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:02.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:02.984 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: cluster 2023-10-09T09:00:01.746013+0000 mgr.y (mgr.25421) 674 : cluster 0 pgmap v663: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 1.0 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 365 op/s; 3092/50268 objects degraded (6.151%); 4395/50268 objects misplaced (8.743%); 4.3 MiB/s, 96 objects/s recovering 2023-10-09T09:00:02.984 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: audit 2023-10-09T09:00:01.767011+0000 mon.a (mon.0) 896 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:02.984 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: audit 2023-10-09T09:00:01.782012+0000 mon.a (mon.0) 897 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:02.984 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: audit 2023-10-09T09:00:02.053622+0000 mon.a (mon.0) 898 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:02.984 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: audit 2023-10-09T09:00:02.062253+0000 mon.a (mon.0) 899 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:02.984 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: audit 2023-10-09T09:00:02.739567+0000 mon.a (mon.0) 900 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:02.985 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: audit 2023-10-09T09:00:02.740969+0000 mon.a (mon.0) 901 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:02.985 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:02 smithi094 bash[59677]: audit 2023-10-09T09:00:02.749141+0000 mon.a (mon.0) 902 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:03.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:03.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:03.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: cluster 2023-10-09T09:00:01.746013+0000 mgr.y (mgr.25421) 674 : cluster 0 pgmap v663: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 1.0 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 365 op/s; 3092/50268 objects degraded (6.151%); 4395/50268 objects misplaced (8.743%); 4.3 MiB/s, 96 objects/s recovering 2023-10-09T09:00:03.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: audit 2023-10-09T09:00:01.767011+0000 mon.a (mon.0) 896 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: audit 2023-10-09T09:00:01.782012+0000 mon.a (mon.0) 897 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: audit 2023-10-09T09:00:02.053622+0000 mon.a (mon.0) 898 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: audit 2023-10-09T09:00:02.062253+0000 mon.a (mon.0) 899 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: audit 2023-10-09T09:00:02.739567+0000 mon.a (mon.0) 900 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: audit 2023-10-09T09:00:02.740969+0000 mon.a (mon.0) 901 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[75179]: audit 2023-10-09T09:00:02.749141+0000 mon.a (mon.0) 902 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: cluster 2023-10-09T09:00:01.746013+0000 mgr.y (mgr.25421) 674 : cluster 0 pgmap v663: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 1.0 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 365 op/s; 3092/50268 objects degraded (6.151%); 4395/50268 objects misplaced (8.743%); 4.3 MiB/s, 96 objects/s recovering 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: audit 2023-10-09T09:00:01.767011+0000 mon.a (mon.0) 896 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: audit 2023-10-09T09:00:01.782012+0000 mon.a (mon.0) 897 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: audit 2023-10-09T09:00:02.053622+0000 mon.a (mon.0) 898 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: audit 2023-10-09T09:00:02.062253+0000 mon.a (mon.0) 899 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.119 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: audit 2023-10-09T09:00:02.739567+0000 mon.a (mon.0) 900 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:03.120 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: audit 2023-10-09T09:00:02.740969+0000 mon.a (mon.0) 901 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:03.120 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:02 smithi016 bash[77892]: audit 2023-10-09T09:00:02.749141+0000 mon.a (mon.0) 902 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:03.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:03.260 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:03.286 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:03.302 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 4] out_osds: [7, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:00:03.302 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:00:03.302 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T09:00:03.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T09:00:03.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:03.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:03.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:03.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:03.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:03.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:03.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:03.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:03.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:03.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:03.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:03.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:03.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:03.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:04.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:04.111 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:04.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:04.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:04.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:04.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:04.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:04.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:04.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:04.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:04.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:04.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:04.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:04.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:04.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:04.594 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:04.595 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:04.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:04.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:04.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:04.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:04.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:04.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:04.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:04.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:04.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:04.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:05.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:04 smithi016 bash[75179]: cluster 2023-10-09T09:00:03.748122+0000 mgr.y (mgr.25421) 675 : cluster 0 pgmap v664: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 902 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 3092/43326 objects degraded (7.137%); 3634/43326 objects misplaced (8.388%); 3.6 MiB/s, 94 objects/s recovering 2023-10-09T09:00:05.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:04 smithi016 bash[75179]: audit 2023-10-09T09:00:04.228828+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:05.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:04 smithi016 bash[75179]: cluster 2023-10-09T09:00:04.481346+0000 mon.a (mon.0) 904 : cluster 3 Health check update: Degraded data redundancy: 3092/43326 objects degraded (7.137%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:05.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:04 smithi016 bash[77892]: cluster 2023-10-09T09:00:03.748122+0000 mgr.y (mgr.25421) 675 : cluster 0 pgmap v664: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 902 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 3092/43326 objects degraded (7.137%); 3634/43326 objects misplaced (8.388%); 3.6 MiB/s, 94 objects/s recovering 2023-10-09T09:00:05.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:04 smithi016 bash[77892]: audit 2023-10-09T09:00:04.228828+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:05.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:04 smithi016 bash[77892]: cluster 2023-10-09T09:00:04.481346+0000 mon.a (mon.0) 904 : cluster 3 Health check update: Degraded data redundancy: 3092/43326 objects degraded (7.137%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:05.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:05.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:05.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:05.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:05.156 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:05.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:04 smithi094 bash[59677]: cluster 2023-10-09T09:00:03.748122+0000 mgr.y (mgr.25421) 675 : cluster 0 pgmap v664: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 902 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 3092/43326 objects degraded (7.137%); 3634/43326 objects misplaced (8.388%); 3.6 MiB/s, 94 objects/s recovering 2023-10-09T09:00:05.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:04 smithi094 bash[59677]: audit 2023-10-09T09:00:04.228828+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:05.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:04 smithi094 bash[59677]: cluster 2023-10-09T09:00:04.481346+0000 mon.a (mon.0) 904 : cluster 3 Health check update: Degraded data redundancy: 3092/43326 objects degraded (7.137%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:05.183 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:05.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:05.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:05.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:05.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:05.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:05.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:05.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:05.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:05.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:05.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:05.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:05.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:05.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:05.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:05.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:05.846 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:05.870 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:05.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:05.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:05.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:05.989 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:00:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:00:05] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-09T09:00:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:06.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:06.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:06.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:06.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:06.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:06.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:06.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:06.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:06.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:06.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:06.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:06.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:06.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:06.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:06.833 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:06.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:06.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:06.973 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:06 smithi016 bash[75179]: cluster 2023-10-09T09:00:05.748649+0000 mgr.y (mgr.25421) 676 : cluster 0 pgmap v665: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 890 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 3092/42771 objects degraded (7.229%); 3634/42771 objects misplaced (8.496%); 3.2 MiB/s, 84 objects/s recovering 2023-10-09T09:00:06.974 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:06 smithi016 bash[77892]: cluster 2023-10-09T09:00:05.748649+0000 mgr.y (mgr.25421) 676 : cluster 0 pgmap v665: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 890 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 3092/42771 objects degraded (7.229%); 3634/42771 objects misplaced (8.496%); 3.2 MiB/s, 84 objects/s recovering 2023-10-09T09:00:06.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:06.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:07.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:07.106 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:06 smithi094 bash[59677]: cluster 2023-10-09T09:00:05.748649+0000 mgr.y (mgr.25421) 676 : cluster 0 pgmap v665: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 890 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 3092/42771 objects degraded (7.229%); 3634/42771 objects misplaced (8.496%); 3.2 MiB/s, 84 objects/s recovering 2023-10-09T09:00:07.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:07.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:07.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:07.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:07.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:07.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:07.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:07.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:07.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:07.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:07.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:07.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:07.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:07.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:07.644 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:07.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:07.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:07.759 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:07.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:07.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:07.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:07.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:07.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:07.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:07.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:08.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:08.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:08.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:08.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:08.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:08.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:08.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:08.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:08.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:08.436 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:08.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:08.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:08.676 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:08.707 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:08.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:08.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:08.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:08.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:08.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:08.944 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:08 smithi094 bash[59677]: cluster 2023-10-09T09:00:07.749278+0000 mgr.y (mgr.25421) 677 : cluster 0 pgmap v666: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 814 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 3092/39132 objects degraded (7.901%); 3106/39132 objects misplaced (7.937%); 3.0 MiB/s, 78 objects/s recovering 2023-10-09T09:00:08.961 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:08.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:09.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:09.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:09.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:09.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:09.218 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:09.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:09.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:08 smithi016 bash[75179]: cluster 2023-10-09T09:00:07.749278+0000 mgr.y (mgr.25421) 677 : cluster 0 pgmap v666: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 814 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 3092/39132 objects degraded (7.901%); 3106/39132 objects misplaced (7.937%); 3.0 MiB/s, 78 objects/s recovering 2023-10-09T09:00:09.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:08 smithi016 bash[77892]: cluster 2023-10-09T09:00:07.749278+0000 mgr.y (mgr.25421) 677 : cluster 0 pgmap v666: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 814 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 3092/39132 objects degraded (7.901%); 3106/39132 objects misplaced (7.937%); 3.0 MiB/s, 78 objects/s recovering 2023-10-09T09:00:09.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:09.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:09.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:09.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:09.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:09.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:09.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:09.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:09.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:09.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:09.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:09.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:09.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:09.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:09.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:09.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:09.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:09.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:09.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:09 smithi016 bash[75179]: cluster 2023-10-09T09:00:09.482565+0000 mon.a (mon.0) 905 : cluster 3 Health check update: Degraded data redundancy: 3092/39132 objects degraded (7.901%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:09.998 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:09 smithi016 bash[77892]: cluster 2023-10-09T09:00:09.482565+0000 mon.a (mon.0) 905 : cluster 3 Health check update: Degraded data redundancy: 3092/39132 objects degraded (7.901%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:10.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:10.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:10.140 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:09 smithi094 bash[59677]: cluster 2023-10-09T09:00:09.482565+0000 mon.a (mon.0) 905 : cluster 3 Health check update: Degraded data redundancy: 3092/39132 objects degraded (7.901%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:10.157 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:10.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:10.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:10.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:10.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:10.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:10.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:10.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:10.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:10.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:10.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:10.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:10.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:10.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:10.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:10.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:10.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:10.969 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:10.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:11.072 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:11.093 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:11.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:11.196 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:11.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:11.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:11.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:11.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:11.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:11.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:11.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:11.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:11.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:11.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:11.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:11.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:11.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:11.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:11.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:11 smithi094 bash[59677]: cluster 2023-10-09T09:00:09.749947+0000 mgr.y (mgr.25421) 678 : cluster 0 pgmap v667: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 701 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 640 op/s; 3092/33717 objects degraded (9.170%); 2792/33717 objects misplaced (8.281%); 224 KiB/s, 48 objects/s recovering 2023-10-09T09:00:11.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:11.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:11.745 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:11 smithi016 bash[75179]: cluster 2023-10-09T09:00:09.749947+0000 mgr.y (mgr.25421) 678 : cluster 0 pgmap v667: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 701 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 640 op/s; 3092/33717 objects degraded (9.170%); 2792/33717 objects misplaced (8.281%); 224 KiB/s, 48 objects/s recovering 2023-10-09T09:00:11.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:11 smithi016 bash[77892]: cluster 2023-10-09T09:00:09.749947+0000 mgr.y (mgr.25421) 678 : cluster 0 pgmap v667: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 701 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 640 op/s; 3092/33717 objects degraded (9.170%); 2792/33717 objects misplaced (8.281%); 224 KiB/s, 48 objects/s recovering 2023-10-09T09:00:11.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:11.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:11.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:11.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:11.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:12.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:12.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:12.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:12.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:12.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:12.233 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:12.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:12.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:12.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:12.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:12.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:12.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:12.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:12.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:12.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:12.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:12.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:12.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:12.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:12.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:13.008 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:13.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:13.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:13.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:13.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:13.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:13.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:13.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:13.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:13.427 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:13.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:13.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:13.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:13.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:13.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:13.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:13 smithi094 bash[59677]: cluster 2023-10-09T09:00:11.751278+0000 mgr.y (mgr.25421) 679 : cluster 0 pgmap v668: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 654 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 590 op/s; 3092/31440 objects degraded (9.835%); 2792/31440 objects misplaced (8.880%); 0 B/s, 27 objects/s recovering 2023-10-09T09:00:13.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:13.837 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:13.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:13.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:13 smithi016 bash[75179]: cluster 2023-10-09T09:00:11.751278+0000 mgr.y (mgr.25421) 679 : cluster 0 pgmap v668: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 654 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 590 op/s; 3092/31440 objects degraded (9.835%); 2792/31440 objects misplaced (8.880%); 0 B/s, 27 objects/s recovering 2023-10-09T09:00:13.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:13 smithi016 bash[77892]: cluster 2023-10-09T09:00:11.751278+0000 mgr.y (mgr.25421) 679 : cluster 0 pgmap v668: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 654 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 590 op/s; 3092/31440 objects degraded (9.835%); 2792/31440 objects misplaced (8.880%); 0 B/s, 27 objects/s recovering 2023-10-09T09:00:13.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:13.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:14.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:14.098 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:14.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:14.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:14.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:14.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:14.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:14.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:14.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:14.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:14.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:14.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:14.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:14.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:14.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:14.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:14 smithi094 bash[59677]: cluster 2023-10-09T09:00:14.483676+0000 mon.a (mon.0) 906 : cluster 3 Health check update: Degraded data redundancy: 3092/31440 objects degraded (9.835%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:14.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:14.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:14.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:14.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:14.732 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:14 smithi016 bash[75179]: cluster 2023-10-09T09:00:14.483676+0000 mon.a (mon.0) 906 : cluster 3 Health check update: Degraded data redundancy: 3092/31440 objects degraded (9.835%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:14.733 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:14 smithi016 bash[77892]: cluster 2023-10-09T09:00:14.483676+0000 mon.a (mon.0) 906 : cluster 3 Health check update: Degraded data redundancy: 3092/31440 objects degraded (9.835%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:14.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:14.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:14.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:14.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:14.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:14.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:14.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:15.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:15.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:15.152 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:15.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:15.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:15.288 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:15.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:15.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:15.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:15.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:15.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:15.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:15.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:15.671 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:15 smithi016 bash[75179]: cluster 2023-10-09T09:00:13.751819+0000 mgr.y (mgr.25421) 680 : cluster 0 pgmap v669: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 511 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 3092/24555 objects degraded (12.592%); 2028/24555 objects misplaced (8.259%); 0 B/s, 45 objects/s recovering 2023-10-09T09:00:15.671 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:15 smithi016 bash[77892]: cluster 2023-10-09T09:00:13.751819+0000 mgr.y (mgr.25421) 680 : cluster 0 pgmap v669: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 511 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 3092/24555 objects degraded (12.592%); 2028/24555 objects misplaced (8.259%); 0 B/s, 45 objects/s recovering 2023-10-09T09:00:15.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:15.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:15.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:15 smithi094 bash[59677]: cluster 2023-10-09T09:00:13.751819+0000 mgr.y (mgr.25421) 680 : cluster 0 pgmap v669: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 511 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 3092/24555 objects degraded (12.592%); 2028/24555 objects misplaced (8.259%); 0 B/s, 45 objects/s recovering 2023-10-09T09:00:15.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:15.823 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:15.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:15.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:15.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:15.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:15.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:00:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:00:15] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-09T09:00:16.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:16.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:16.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:16.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:16.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:16.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:16.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:16.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:16.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:16.348 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:16.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:16.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:16.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:16.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:16.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:16.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:16.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:16.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:16.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:16 smithi094 bash[59677]: cluster 2023-10-09T09:00:15.752414+0000 mgr.y (mgr.25421) 681 : cluster 0 pgmap v670: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 500 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 535 op/s; 3092/24045 objects degraded (12.859%); 2028/24045 objects misplaced (8.434%); 0 B/s, 32 objects/s recovering 2023-10-09T09:00:16.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:16.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:16 smithi016 bash[75179]: cluster 2023-10-09T09:00:15.752414+0000 mgr.y (mgr.25421) 681 : cluster 0 pgmap v670: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 500 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 535 op/s; 3092/24045 objects degraded (12.859%); 2028/24045 objects misplaced (8.434%); 0 B/s, 32 objects/s recovering 2023-10-09T09:00:16.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:16 smithi016 bash[77892]: cluster 2023-10-09T09:00:15.752414+0000 mgr.y (mgr.25421) 681 : cluster 0 pgmap v670: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 500 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 535 op/s; 3092/24045 objects degraded (12.859%); 2028/24045 objects misplaced (8.434%); 0 B/s, 32 objects/s recovering 2023-10-09T09:00:16.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:16.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:16.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:16.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:16.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:16.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:16.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:17.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:17.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:17.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:17.171 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:17.194 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:17.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:17.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:17.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:17.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:17.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:17.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:17.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:17.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:17.592 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:17.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:17.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:17.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:17.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:17.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:17.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:17.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:17.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:17.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:17.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:18.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:18.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:18.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:18.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:18.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:18.261 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:18.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:18.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:18.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:18.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:18.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:18.488 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:18.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:18.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:18.617 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:18.644 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:18.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:18.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:18.771 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:18.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:18.876 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:18.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:18.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:18.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:19.008 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:19.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:18 smithi094 bash[59677]: cluster 2023-10-09T09:00:17.753033+0000 mgr.y (mgr.25421) 682 : cluster 0 pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 429 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 3092/20637 objects degraded (14.983%); 1547/20637 objects misplaced (7.496%); 0 B/s, 32 objects/s recovering 2023-10-09T09:00:19.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:19.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:19.148 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:18 smithi016 bash[75179]: cluster 2023-10-09T09:00:17.753033+0000 mgr.y (mgr.25421) 682 : cluster 0 pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 429 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 3092/20637 objects degraded (14.983%); 1547/20637 objects misplaced (7.496%); 0 B/s, 32 objects/s recovering 2023-10-09T09:00:19.148 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:18 smithi016 bash[77892]: cluster 2023-10-09T09:00:17.753033+0000 mgr.y (mgr.25421) 682 : cluster 0 pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 429 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 3092/20637 objects degraded (14.983%); 1547/20637 objects misplaced (7.496%); 0 B/s, 32 objects/s recovering 2023-10-09T09:00:19.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:19.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:19.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:19.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:19.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:19.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:19.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:19.538 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:19.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:19.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:19.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:19.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:19.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:19.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:19.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:19.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:20.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:19 smithi016 bash[75179]: audit 2023-10-09T09:00:19.226929+0000 mon.a (mon.0) 907 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:20.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:19 smithi016 bash[75179]: cluster 2023-10-09T09:00:19.484717+0000 mon.a (mon.0) 908 : cluster 3 Health check update: Degraded data redundancy: 3092/20637 objects degraded (14.983%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:20.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:19 smithi016 bash[77892]: audit 2023-10-09T09:00:19.226929+0000 mon.a (mon.0) 907 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:20.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:19 smithi016 bash[77892]: cluster 2023-10-09T09:00:19.484717+0000 mon.a (mon.0) 908 : cluster 3 Health check update: Degraded data redundancy: 3092/20637 objects degraded (14.983%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:20.070 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:20.083 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:19 smithi094 bash[59677]: audit 2023-10-09T09:00:19.226929+0000 mon.a (mon.0) 907 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:20.083 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:19 smithi094 bash[59677]: cluster 2023-10-09T09:00:19.484717+0000 mon.a (mon.0) 908 : cluster 3 Health check update: Degraded data redundancy: 3092/20637 objects degraded (14.983%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:20.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:20.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:20.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:20.206 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:20.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:20.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:20.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:20.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:20.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:20.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:20.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:20.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:20.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:20.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:20.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:20.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:20.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:20.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:20.741 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:20.764 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:20.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:20.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:20.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:20.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:21.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:21.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:21.168 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:21.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:21.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:21.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:21.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:21.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:21.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:21.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:21.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:21.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:21.668 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:21 smithi016 bash[75179]: cluster 2023-10-09T09:00:19.755782+0000 mgr.y (mgr.25421) 683 : cluster 0 pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 315 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 665 op/s; 3092/15153 objects degraded (20.405%); 1248/15153 objects misplaced (8.236%); 0 B/s, 48 objects/s recovering 2023-10-09T09:00:21.669 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:21 smithi016 bash[77892]: cluster 2023-10-09T09:00:19.755782+0000 mgr.y (mgr.25421) 683 : cluster 0 pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 315 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 665 op/s; 3092/15153 objects degraded (20.405%); 1248/15153 objects misplaced (8.236%); 0 B/s, 48 objects/s recovering 2023-10-09T09:00:21.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:21.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:21.807 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:21 smithi094 bash[59677]: cluster 2023-10-09T09:00:19.755782+0000 mgr.y (mgr.25421) 683 : cluster 0 pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 315 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 665 op/s; 3092/15153 objects degraded (20.405%); 1248/15153 objects misplaced (8.236%); 0 B/s, 48 objects/s recovering 2023-10-09T09:00:21.823 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:21.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:21.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:21.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:22.064 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:22.090 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:22.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:22.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:22.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:22.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:22.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:22.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:22.358 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:22.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:22.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:22.389 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:00:22.389 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:00:22.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:22.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:22.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:22.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:22.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:22.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:22.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:22.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:22.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:22.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:22.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:22.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:22.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:22.982 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:23.005 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:23.113 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:23.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:23.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:23.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:23.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:23.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:23.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:23.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:23.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:23.517 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:23 smithi016 bash[75179]: cluster 2023-10-09T09:00:21.756415+0000 mgr.y (mgr.25421) 684 : cluster 0 pgmap v673: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 267 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 3092/12846 objects degraded (24.070%); 1248/12846 objects misplaced (9.715%); 0 B/s, 33 objects/s recovering 2023-10-09T09:00:23.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:23 smithi016 bash[77892]: cluster 2023-10-09T09:00:21.756415+0000 mgr.y (mgr.25421) 684 : cluster 0 pgmap v673: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 267 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 3092/12846 objects degraded (24.070%); 1248/12846 objects misplaced (9.715%); 0 B/s, 33 objects/s recovering 2023-10-09T09:00:23.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:23.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:23.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:23.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:23.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:23.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:23.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:23.932 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:23 smithi094 bash[59677]: cluster 2023-10-09T09:00:21.756415+0000 mgr.y (mgr.25421) 684 : cluster 0 pgmap v673: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 267 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 3092/12846 objects degraded (24.070%); 1248/12846 objects misplaced (9.715%); 0 B/s, 33 objects/s recovering 2023-10-09T09:00:23.955 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:23.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:24.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:24.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:24.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:24.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:24.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:24.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:24.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:24.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:24.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:24.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:24.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:24.497 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:24.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:24.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:24.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:24.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:24 smithi094 bash[59677]: cluster 2023-10-09T09:00:24.485825+0000 mon.a (mon.0) 909 : cluster 3 Health check update: Degraded data redundancy: 3092/12846 objects degraded (24.070%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:24.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:24.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:24.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:24.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:24.766 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:24 smithi016 bash[75179]: cluster 2023-10-09T09:00:24.485825+0000 mon.a (mon.0) 909 : cluster 3 Health check update: Degraded data redundancy: 3092/12846 objects degraded (24.070%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:24.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:24 smithi016 bash[77892]: cluster 2023-10-09T09:00:24.485825+0000 mon.a (mon.0) 909 : cluster 3 Health check update: Degraded data redundancy: 3092/12846 objects degraded (24.070%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:24.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:24.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:24.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:24.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:24.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:24.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:24.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:25.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:25.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:25.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:25.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:25.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:25.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:25.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:25.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:25.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:25.530 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:00:25.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:25.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:25.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:25.605 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 100515 objects 2023-10-09T09:00:25.606 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :172.261 2023-10-09T09:00:25.631 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T09:00:25.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T09:00:25.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:25.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:25.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:25.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:25 smithi016 bash[75179]: cluster 2023-10-09T09:00:23.756976+0000 mgr.y (mgr.25421) 685 : cluster 0 pgmap v674: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 115 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 3092/5544 objects degraded (55.772%); 419/5544 objects misplaced (7.558%); 0 B/s, 49 objects/s recovering 2023-10-09T09:00:25.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:25 smithi016 bash[75179]: audit 2023-10-09T09:00:24.699241+0000 mon.a (mon.0) 910 : audit 1 from='client.? 172.21.15.16:0/1908418123' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:00:25.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:25 smithi016 bash[77892]: cluster 2023-10-09T09:00:23.756976+0000 mgr.y (mgr.25421) 685 : cluster 0 pgmap v674: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 115 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 3092/5544 objects degraded (55.772%); 419/5544 objects misplaced (7.558%); 0 B/s, 49 objects/s recovering 2023-10-09T09:00:25.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:25 smithi016 bash[77892]: audit 2023-10-09T09:00:24.699241+0000 mon.a (mon.0) 910 : audit 1 from='client.? 172.21.15.16:0/1908418123' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:00:25.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:25.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:25.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:25.959 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:25 smithi094 bash[59677]: cluster 2023-10-09T09:00:23.756976+0000 mgr.y (mgr.25421) 685 : cluster 0 pgmap v674: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 115 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 3092/5544 objects degraded (55.772%); 419/5544 objects misplaced (7.558%); 0 B/s, 49 objects/s recovering 2023-10-09T09:00:25.959 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:25 smithi094 bash[59677]: audit 2023-10-09T09:00:24.699241+0000 mon.a (mon.0) 910 : audit 1 from='client.? 172.21.15.16:0/1908418123' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:00:25.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:25.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:25.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:00:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:00:25] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-09T09:00:26.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:26.108 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:26.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:26.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:26.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:26.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:26.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:26.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:26.378 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:26.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:26.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:26.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:26.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:26.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:26.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:26.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:26.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:26.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[75179]: cluster 2023-10-09T09:00:25.511229+0000 mon.a (mon.0) 911 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:00:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[75179]: cluster 2023-10-09T09:00:25.519801+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T09:00:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[75179]: cluster 2023-10-09T09:00:25.520034+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T09:00:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[75179]: audit 2023-10-09T09:00:25.522469+0000 mon.a (mon.0) 912 : audit 1 from='client.? 172.21.15.16:0/1908418123' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:00:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[75179]: cluster 2023-10-09T09:00:25.522545+0000 mon.a (mon.0) 913 : cluster 0 osdmap e159: 8 total, 8 up, 6 in 2023-10-09T09:00:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[75179]: cluster 2023-10-09T09:00:25.757422+0000 mgr.y (mgr.25421) 686 : cluster 0 pgmap v676: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 103 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 634 op/s; 3092/5004 objects degraded (61.791%); 419/5004 objects misplaced (8.373%); 0 B/s, 37 objects/s recovering 2023-10-09T09:00:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[77892]: cluster 2023-10-09T09:00:25.511229+0000 mon.a (mon.0) 911 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:00:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[77892]: cluster 2023-10-09T09:00:25.519801+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T09:00:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[77892]: cluster 2023-10-09T09:00:25.520034+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T09:00:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[77892]: audit 2023-10-09T09:00:25.522469+0000 mon.a (mon.0) 912 : audit 1 from='client.? 172.21.15.16:0/1908418123' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:00:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[77892]: cluster 2023-10-09T09:00:25.522545+0000 mon.a (mon.0) 913 : cluster 0 osdmap e159: 8 total, 8 up, 6 in 2023-10-09T09:00:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:26 smithi016 bash[77892]: cluster 2023-10-09T09:00:25.757422+0000 mgr.y (mgr.25421) 686 : cluster 0 pgmap v676: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 103 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 634 op/s; 3092/5004 objects degraded (61.791%); 419/5004 objects misplaced (8.373%); 0 B/s, 37 objects/s recovering 2023-10-09T09:00:26.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:26.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:26.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:26.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:26.884 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:26 smithi094 bash[59677]: cluster 2023-10-09T09:00:25.511229+0000 mon.a (mon.0) 911 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:00:26.885 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:26 smithi094 bash[59677]: cluster 2023-10-09T09:00:25.519801+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T09:00:26.886 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:26 smithi094 bash[59677]: cluster 2023-10-09T09:00:25.520034+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T09:00:26.887 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:26 smithi094 bash[59677]: audit 2023-10-09T09:00:25.522469+0000 mon.a (mon.0) 912 : audit 1 from='client.? 172.21.15.16:0/1908418123' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:00:26.887 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:26 smithi094 bash[59677]: cluster 2023-10-09T09:00:25.522545+0000 mon.a (mon.0) 913 : cluster 0 osdmap e159: 8 total, 8 up, 6 in 2023-10-09T09:00:26.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:26 smithi094 bash[59677]: cluster 2023-10-09T09:00:25.757422+0000 mgr.y (mgr.25421) 686 : cluster 0 pgmap v676: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped, 82 active+clean; 103 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 634 op/s; 3092/5004 objects degraded (61.791%); 419/5004 objects misplaced (8.373%); 0 B/s, 37 objects/s recovering 2023-10-09T09:00:26.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:26.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:26.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:27.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:27.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:27.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:27.133 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:27.166 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:27.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:27.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:27.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:27.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:27.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:27.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:27.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:27.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:27.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:27.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:27.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:27.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:27.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:27.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:28.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:28.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:28.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:28.191 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:28.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:28.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:28.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:28.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:28.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:28.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:28.561 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:28.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:28.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:28.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:28.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:28.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:28.835 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:28 smithi094 bash[59677]: cluster 2023-10-09T09:00:27.758057+0000 mgr.y (mgr.25421) 687 : cluster 0 pgmap v677: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 50 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 3092/2439 objects degraded (126.773%); 128/2439 objects misplaced (5.248%); 0 B/s, 37 objects/s recovering 2023-10-09T09:00:28.856 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:28.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:28.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:28.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:29.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:29.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:29.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:29.203 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:28 smithi016 bash[75179]: cluster 2023-10-09T09:00:27.758057+0000 mgr.y (mgr.25421) 687 : cluster 0 pgmap v677: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 50 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 3092/2439 objects degraded (126.773%); 128/2439 objects misplaced (5.248%); 0 B/s, 37 objects/s recovering 2023-10-09T09:00:29.203 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:28 smithi016 bash[77892]: cluster 2023-10-09T09:00:27.758057+0000 mgr.y (mgr.25421) 687 : cluster 0 pgmap v677: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 50 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 3092/2439 objects degraded (126.773%); 128/2439 objects misplaced (5.248%); 0 B/s, 37 objects/s recovering 2023-10-09T09:00:29.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:00:29.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:29.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:29.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:29.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:29.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:29.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:29.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:29.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:29.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:29.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:29.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:29.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:29.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:29.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:29.883 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:00:29.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.913 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:00:29.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:29.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:29.944 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:00:29.973 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:00:30.002 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:00:30.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:30.028 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:00:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:29 smithi016 bash[75179]: cluster 2023-10-09T09:00:29.487037+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 3092/2439 objects degraded (126.773%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:30.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:29 smithi016 bash[77892]: cluster 2023-10-09T09:00:29.487037+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 3092/2439 objects degraded (126.773%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:30.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:30.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:30.056 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:00:30.082 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:00:30.084 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.084 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.084 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.084 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.084 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.085 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.085 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.085 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:00:30.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:30.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:29 smithi094 bash[59677]: cluster 2023-10-09T09:00:29.487037+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 3092/2439 objects degraded (126.773%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:30.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:30.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:30.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:30.292 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[75179]: debug 2023-10-09T09:00:30.262+0000 7fa046a4d700 -1 mon.a@0(leader).osd e159 definitely_dead 0 2023-10-09T09:00:30.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:30.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:30.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:30.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:30.554 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:30.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:30.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:30.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:30.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:30.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:30.849 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.6. 2023-10-09T09:00:30.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:30.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:30.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:30.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[75179]: cluster 2023-10-09T09:00:29.758553+0000 mgr.y (mgr.25421) 688 : cluster 0 pgmap v678: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 1.2 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 3092/102 objects degraded (3031.373%); 0 B/s, 68 objects/s recovering 2023-10-09T09:00:30.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[75179]: audit 2023-10-09T09:00:29.852130+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.16:0/2054575778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:00:30.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[75179]: audit 2023-10-09T09:00:30.267037+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.16:0/3265267831' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T09:00:30.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[75179]: audit 2023-10-09T09:00:30.267479+0000 mon.a (mon.0) 915 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T09:00:30.978 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[77892]: cluster 2023-10-09T09:00:29.758553+0000 mgr.y (mgr.25421) 688 : cluster 0 pgmap v678: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 1.2 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 3092/102 objects degraded (3031.373%); 0 B/s, 68 objects/s recovering 2023-10-09T09:00:30.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[77892]: audit 2023-10-09T09:00:29.852130+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.16:0/2054575778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:00:30.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[77892]: audit 2023-10-09T09:00:30.267037+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.16:0/3265267831' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T09:00:30.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:30 smithi016 bash[77892]: audit 2023-10-09T09:00:30.267479+0000 mon.a (mon.0) 915 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T09:00:30.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:30.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:30.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:31.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:31.108 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:30 smithi094 bash[59677]: cluster 2023-10-09T09:00:29.758553+0000 mgr.y (mgr.25421) 688 : cluster 0 pgmap v678: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 1.2 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 3092/102 objects degraded (3031.373%); 0 B/s, 68 objects/s recovering 2023-10-09T09:00:31.108 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:30 smithi094 bash[59677]: audit 2023-10-09T09:00:29.852130+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.16:0/2054575778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:00:31.108 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:30 smithi094 bash[59677]: audit 2023-10-09T09:00:30.267037+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.16:0/3265267831' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T09:00:31.108 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:30 smithi094 bash[59677]: audit 2023-10-09T09:00:30.267479+0000 mon.a (mon.0) 915 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T09:00:31.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:31.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:31.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:31.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:31.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:31.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:31.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:31.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:31.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:31.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:31.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:31.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:31.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:31.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:31.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:31.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:31.734 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:31.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:00:31.761 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:31.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:31.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:31.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:31.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:31.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:31.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:32.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:32.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:32.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:32.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.818744+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.821058+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.822001+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.823225+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: audit 2023-10-09T09:00:30.827382+0000 mon.a (mon.0) 916 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T09:00:32.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.827710+0000 mon.a (mon.0) 917 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-10-09T09:00:32.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.829841+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.829929+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.829982+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.832488+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.136 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.832939+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.136 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.835973+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.136 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:30.836569+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.136 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.822083+0000 mon.a (mon.0) 918 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-10-09T09:00:32.137 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.818744+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.137 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.821058+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.137 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.822001+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.137 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.823225+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.137 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: audit 2023-10-09T09:00:30.827382+0000 mon.a (mon.0) 916 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T09:00:32.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.827710+0000 mon.a (mon.0) 917 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-10-09T09:00:32.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.829841+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.829929+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.829982+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.832488+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.832939+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.835973+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.139 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:30.836569+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.139 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:31 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.822083+0000 mon.a (mon.0) 918 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-10-09T09:00:32.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:32.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.818744+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.821058+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.822001+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.823225+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: audit 2023-10-09T09:00:30.827382+0000 mon.a (mon.0) 916 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.827710+0000 mon.a (mon.0) 917 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.829841+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.829929+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.829982+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.832488+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.832939+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.835973+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:30.836569+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T09:00:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:31 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.822083+0000 mon.a (mon.0) 918 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-10-09T09:00:32.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:32.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:32.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:32.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:32.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:32.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:32.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:32.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:32.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:32.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:32.775 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:32.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:32.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:32.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:32.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:32.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.759146+0000 mgr.y (mgr.25421) 689 : cluster 0 pgmap v680: 89 pgs: 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 3092/60 objects degraded (5153.333%); 0 B/s, 62 objects/s recovering 2023-10-09T09:00:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.819156+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.822455+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.822494+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.822631+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.824914+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.825239+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.826683+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.827573+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.828115+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.829215+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.829931+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.830448+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.831948+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.832904+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.834347+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:31.936827+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[77892]: cluster 2023-10-09T09:00:32.826434+0000 mon.a (mon.0) 919 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-10-09T09:00:33.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.759146+0000 mgr.y (mgr.25421) 689 : cluster 0 pgmap v680: 89 pgs: 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 3092/60 objects degraded (5153.333%); 0 B/s, 62 objects/s recovering 2023-10-09T09:00:33.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.819156+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.822455+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.822494+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.822631+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.824914+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.825239+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.826683+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.827573+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.828115+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.829215+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.829931+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.830448+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.831948+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.832904+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.834347+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:31.936827+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:32 smithi016 bash[75179]: cluster 2023-10-09T09:00:32.826434+0000 mon.a (mon.0) 919 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-10-09T09:00:33.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:33.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:33.114 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T09:00:33.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:33.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:33.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.759146+0000 mgr.y (mgr.25421) 689 : cluster 0 pgmap v680: 89 pgs: 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 3092/60 objects degraded (5153.333%); 0 B/s, 62 objects/s recovering 2023-10-09T09:00:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.819156+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.822455+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.822494+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.822631+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.824914+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.825239+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.826683+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.827573+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.828115+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.829215+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.829931+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.830448+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.831948+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.832904+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.834347+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:31.936827+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T09:00:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:32 smithi094 bash[59677]: cluster 2023-10-09T09:00:32.826434+0000 mon.a (mon.0) 919 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-10-09T09:00:33.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:33.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:33.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:33.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:33.582 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:33.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:33.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:33.735 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:33.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:33.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:33.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:33.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:33.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:33.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:33.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:34.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:34.110 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[75179]: cluster 2023-10-09T09:00:32.821941+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.110 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[75179]: cluster 2023-10-09T09:00:32.828023+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.110 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[75179]: cluster 2023-10-09T09:00:32.830736+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.110 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[75179]: cluster 2023-10-09T09:00:32.832358+0000 osd.1 (osd.1) 7 : cluster 1 7.1 continuing backfill to osd.4 from (145'7595,158'12370] MIN to 158'12370 2023-10-09T09:00:34.110 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[75179]: cluster 2023-10-09T09:00:32.833515+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[75179]: cluster 2023-10-09T09:00:33.037191+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[75179]: cluster 2023-10-09T09:00:33.829961+0000 mon.a (mon.0) 920 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[77892]: cluster 2023-10-09T09:00:32.821941+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[77892]: cluster 2023-10-09T09:00:32.828023+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[77892]: cluster 2023-10-09T09:00:32.830736+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[77892]: cluster 2023-10-09T09:00:32.832358+0000 osd.1 (osd.1) 7 : cluster 1 7.1 continuing backfill to osd.4 from (145'7595,158'12370] MIN to 158'12370 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[77892]: cluster 2023-10-09T09:00:32.833515+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[77892]: cluster 2023-10-09T09:00:33.037191+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.112 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:33 smithi016 bash[77892]: cluster 2023-10-09T09:00:33.829961+0000 mon.a (mon.0) 920 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-10-09T09:00:34.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:34.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:34.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:34.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:34.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:33 smithi094 bash[59677]: cluster 2023-10-09T09:00:32.821941+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:33 smithi094 bash[59677]: cluster 2023-10-09T09:00:32.828023+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:33 smithi094 bash[59677]: cluster 2023-10-09T09:00:32.830736+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:33 smithi094 bash[59677]: cluster 2023-10-09T09:00:32.832358+0000 osd.1 (osd.1) 7 : cluster 1 7.1 continuing backfill to osd.4 from (145'7595,158'12370] MIN to 158'12370 2023-10-09T09:00:34.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:33 smithi094 bash[59677]: cluster 2023-10-09T09:00:32.833515+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:33 smithi094 bash[59677]: cluster 2023-10-09T09:00:33.037191+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:33 smithi094 bash[59677]: cluster 2023-10-09T09:00:33.829961+0000 mon.a (mon.0) 920 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-10-09T09:00:34.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:00:34.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:34.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:34.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:34.361 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:34.389 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:34.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:34.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:34.492 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:34.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:34.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:34.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:34.617 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:34.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:34.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:34.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:34.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:34.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:34.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:34.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:34.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:34.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:34.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: cluster 2023-10-09T09:00:33.037242+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: cluster 2023-10-09T09:00:33.759887+0000 mgr.y (mgr.25421) 690 : cluster 0 pgmap v683: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 3092/60 objects degraded (5153.333%); 96 KiB/s, 218 objects/s recovering 2023-10-09T09:00:34.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: cluster 2023-10-09T09:00:33.832690+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:34.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: cluster 2023-10-09T09:00:33.834845+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:34.898 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: cluster 2023-10-09T09:00:34.218071+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:34.898 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: audit 2023-10-09T09:00:34.226006+0000 mon.a (mon.0) 921 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:34.898 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: cluster 2023-10-09T09:00:34.488145+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 3092/60 objects degraded (5153.333%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:34.898 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[75179]: cluster 2023-10-09T09:00:34.832589+0000 mon.a (mon.0) 923 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-10-09T09:00:34.898 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: cluster 2023-10-09T09:00:33.037242+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:34.898 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: cluster 2023-10-09T09:00:33.759887+0000 mgr.y (mgr.25421) 690 : cluster 0 pgmap v683: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 3092/60 objects degraded (5153.333%); 96 KiB/s, 218 objects/s recovering 2023-10-09T09:00:34.898 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: cluster 2023-10-09T09:00:33.832690+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:34.899 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: cluster 2023-10-09T09:00:33.834845+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:34.899 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: cluster 2023-10-09T09:00:34.218071+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:34.899 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: audit 2023-10-09T09:00:34.226006+0000 mon.a (mon.0) 921 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:34.899 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: cluster 2023-10-09T09:00:34.488145+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 3092/60 objects degraded (5153.333%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:34.899 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:34 smithi016 bash[77892]: cluster 2023-10-09T09:00:34.832589+0000 mon.a (mon.0) 923 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-10-09T09:00:34.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:34.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:34.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:35.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:35.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:35.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:35.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:35.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:35.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: cluster 2023-10-09T09:00:33.037242+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T09:00:35.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: cluster 2023-10-09T09:00:33.759887+0000 mgr.y (mgr.25421) 690 : cluster 0 pgmap v683: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 3092/60 objects degraded (5153.333%); 96 KiB/s, 218 objects/s recovering 2023-10-09T09:00:35.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: cluster 2023-10-09T09:00:33.832690+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:35.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: cluster 2023-10-09T09:00:33.834845+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:35.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: cluster 2023-10-09T09:00:34.218071+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:35.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: audit 2023-10-09T09:00:34.226006+0000 mon.a (mon.0) 921 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: cluster 2023-10-09T09:00:34.488145+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 3092/60 objects degraded (5153.333%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:00:35.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:34 smithi094 bash[59677]: cluster 2023-10-09T09:00:34.832589+0000 mon.a (mon.0) 923 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-10-09T09:00:35.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:35.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:35.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:35.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:35.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:35.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:35.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:35.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:35.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:35.673 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:35.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:35.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:35.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:35.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:35.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:35.950 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:35 smithi094 bash[59677]: cluster 2023-10-09T09:00:34.218149+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:35.950 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:35 smithi094 bash[59677]: cluster 2023-10-09T09:00:34.825873+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:35.951 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:35 smithi094 bash[59677]: cluster 2023-10-09T09:00:34.833652+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:35.951 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:35 smithi094 bash[59677]: cluster 2023-10-09T09:00:34.834658+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:35.966 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:35.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:35.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:35.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:00:35] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.33.4" 2023-10-09T09:00:36.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[75179]: cluster 2023-10-09T09:00:34.218149+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:36.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[75179]: cluster 2023-10-09T09:00:34.825873+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:36.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[75179]: cluster 2023-10-09T09:00:34.833652+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:36.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[75179]: cluster 2023-10-09T09:00:34.834658+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:36.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[77892]: cluster 2023-10-09T09:00:34.218149+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T09:00:36.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[77892]: cluster 2023-10-09T09:00:34.825873+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:36.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[77892]: cluster 2023-10-09T09:00:34.833652+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:36.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:35 smithi016 bash[77892]: cluster 2023-10-09T09:00:34.834658+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T09:00:36.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:36.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:36.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:36.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:36.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:36.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:36.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:36.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:36.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:36.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:36.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:36.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:36.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:36.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:36.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:36.638 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:36.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:36.653 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:36.742 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:36.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:36.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:36.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:36.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:36.903 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:36.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:37.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:36 smithi016 bash[75179]: cluster 2023-10-09T09:00:35.760955+0000 mgr.y (mgr.25421) 691 : cluster 0 pgmap v686: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 3092/60 objects degraded (5153.333%); 145 KiB/s, 2 keys/s, 204 objects/s recovering 2023-10-09T09:00:37.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:36 smithi016 bash[77892]: cluster 2023-10-09T09:00:35.760955+0000 mgr.y (mgr.25421) 691 : cluster 0 pgmap v686: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 3092/60 objects degraded (5153.333%); 145 KiB/s, 2 keys/s, 204 objects/s recovering 2023-10-09T09:00:37.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:37.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:37.139 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:37.159 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:36 smithi094 bash[59677]: cluster 2023-10-09T09:00:35.760955+0000 mgr.y (mgr.25421) 691 : cluster 0 pgmap v686: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 3092/60 objects degraded (5153.333%); 145 KiB/s, 2 keys/s, 204 objects/s recovering 2023-10-09T09:00:37.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:37.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:37.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:37.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:37.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:37.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:37.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:37.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:37.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:37.667 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:37.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:37.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:37.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:37.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:37.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:37.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:37 smithi016 bash[75179]: cluster 2023-10-09T09:00:36.875679+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T09:00:37.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:37 smithi016 bash[75179]: cluster 2023-10-09T09:00:36.876062+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T09:00:37.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:37 smithi016 bash[75179]: cluster 2023-10-09T09:00:36.877673+0000 mon.a (mon.0) 924 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-10-09T09:00:37.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:37 smithi016 bash[77892]: cluster 2023-10-09T09:00:36.875679+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T09:00:37.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:37 smithi016 bash[77892]: cluster 2023-10-09T09:00:36.876062+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T09:00:37.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:37 smithi016 bash[77892]: cluster 2023-10-09T09:00:36.877673+0000 mon.a (mon.0) 924 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-10-09T09:00:37.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:37.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:38.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:38.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:38.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:38.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:38.115 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 4, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:00:38.115 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:00:38.116 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T09:00:38.116 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:00:38.116 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:00:38.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T09:00:38.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:38.205 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:37 smithi094 bash[59677]: cluster 2023-10-09T09:00:36.875679+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T09:00:38.205 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:37 smithi094 bash[59677]: cluster 2023-10-09T09:00:36.876062+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T09:00:38.205 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:37 smithi094 bash[59677]: cluster 2023-10-09T09:00:36.877673+0000 mon.a (mon.0) 924 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-10-09T09:00:38.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:38.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:38.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:38.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:38.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:38.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:38.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:38.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:38.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:38.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:38.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:38.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:38.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:38.679 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:38.722 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:38.750 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:38.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:38.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:38.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:38.883 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:38.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:38.988 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:38.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:38 smithi094 bash[59677]: cluster 2023-10-09T09:00:37.761838+0000 mgr.y (mgr.25421) 692 : cluster 0 pgmap v688: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 85 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 118 KiB/s, 2 keys/s, 170 objects/s recovering 2023-10-09T09:00:38.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:38 smithi094 bash[59677]: cluster 2023-10-09T09:00:37.874889+0000 mon.a (mon.0) 925 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3092/60 objects degraded (5153.333%), 1 pg degraded) 2023-10-09T09:00:38.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:38 smithi094 bash[59677]: cluster 2023-10-09T09:00:37.882201+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T09:00:38.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:38 smithi094 bash[59677]: cluster 2023-10-09T09:00:37.882901+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T09:00:38.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:38 smithi094 bash[59677]: cluster 2023-10-09T09:00:37.885787+0000 mon.a (mon.0) 926 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-10-09T09:00:38.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[75179]: cluster 2023-10-09T09:00:37.761838+0000 mgr.y (mgr.25421) 692 : cluster 0 pgmap v688: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 85 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 118 KiB/s, 2 keys/s, 170 objects/s recovering 2023-10-09T09:00:38.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[75179]: cluster 2023-10-09T09:00:37.874889+0000 mon.a (mon.0) 925 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3092/60 objects degraded (5153.333%), 1 pg degraded) 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[75179]: cluster 2023-10-09T09:00:37.882201+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[75179]: cluster 2023-10-09T09:00:37.882901+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[75179]: cluster 2023-10-09T09:00:37.885787+0000 mon.a (mon.0) 926 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[77892]: cluster 2023-10-09T09:00:37.761838+0000 mgr.y (mgr.25421) 692 : cluster 0 pgmap v688: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 85 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 118 KiB/s, 2 keys/s, 170 objects/s recovering 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[77892]: cluster 2023-10-09T09:00:37.874889+0000 mon.a (mon.0) 925 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3092/60 objects degraded (5153.333%), 1 pg degraded) 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[77892]: cluster 2023-10-09T09:00:37.882201+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[77892]: cluster 2023-10-09T09:00:37.882901+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T09:00:39.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:38 smithi016 bash[77892]: cluster 2023-10-09T09:00:37.885787+0000 mon.a (mon.0) 926 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-10-09T09:00:39.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:39.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:39.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:39.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:39.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:39.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:39.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:39.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:39.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:39.535 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:39.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:39.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:39.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:39.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:39.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:39.908 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:39.942 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:39.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:40.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:40.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:40.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:40.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:40.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:41.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:41 smithi016 bash[77892]: cluster 2023-10-09T09:00:39.762510+0000 mgr.y (mgr.25421) 693 : cluster 0 pgmap v690: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 814 B/s, 1 keys/s, 137 objects/s recovering 2023-10-09T09:00:41.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:41 smithi016 bash[75179]: cluster 2023-10-09T09:00:39.762510+0000 mgr.y (mgr.25421) 693 : cluster 0 pgmap v690: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 814 B/s, 1 keys/s, 137 objects/s recovering 2023-10-09T09:00:41.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:41 smithi094 bash[59677]: cluster 2023-10-09T09:00:39.762510+0000 mgr.y (mgr.25421) 693 : cluster 0 pgmap v690: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 814 B/s, 1 keys/s, 137 objects/s recovering 2023-10-09T09:00:42.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:42.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:42.396 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:42.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:42.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:42.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:42.529 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:42.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:42.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:42.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:42.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:42.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:42.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:42.789 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:42.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:42.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:42.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:42.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:43.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:43.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:43.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:43.633 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:43.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:43.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:43.749 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:43 smithi094 bash[59677]: cluster 2023-10-09T09:00:41.763289+0000 mgr.y (mgr.25421) 694 : cluster 0 pgmap v691: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 697 B/s, 1 keys/s, 288 objects/s recovering 2023-10-09T09:00:43.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:43.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:43.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:43.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:43.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:43.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:43.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:43.965 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:43 smithi016 bash[75179]: cluster 2023-10-09T09:00:41.763289+0000 mgr.y (mgr.25421) 694 : cluster 0 pgmap v691: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 697 B/s, 1 keys/s, 288 objects/s recovering 2023-10-09T09:00:43.965 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:43 smithi016 bash[77892]: cluster 2023-10-09T09:00:41.763289+0000 mgr.y (mgr.25421) 694 : cluster 0 pgmap v691: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 697 B/s, 1 keys/s, 288 objects/s recovering 2023-10-09T09:00:43.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:44.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:44.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:44.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:44.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:44.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:44.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:44.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:44.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:44.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:44.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:44.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:44.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:44.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:44.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:45.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:45.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:45.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:45.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:45.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:45.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:45.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:45.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:45.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:45.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:45.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:45.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:45.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:45.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:45.815 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:45.823 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:45 smithi094 bash[59677]: cluster 2023-10-09T09:00:43.763867+0000 mgr.y (mgr.25421) 695 : cluster 0 pgmap v692: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 0 B/s, 249 objects/s recovering 2023-10-09T09:00:45.837 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:45.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:45.951 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:00:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:00:45] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.33.4" 2023-10-09T09:00:45.952 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:45 smithi016 bash[75179]: cluster 2023-10-09T09:00:43.763867+0000 mgr.y (mgr.25421) 695 : cluster 0 pgmap v692: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 0 B/s, 249 objects/s recovering 2023-10-09T09:00:45.952 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:45 smithi016 bash[77892]: cluster 2023-10-09T09:00:43.763867+0000 mgr.y (mgr.25421) 695 : cluster 0 pgmap v692: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 0 B/s, 249 objects/s recovering 2023-10-09T09:00:45.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:45.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:45.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:46.070 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:46.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:46.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:46.206 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:46.226 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:46.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:46.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:46.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:46.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:46.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:46.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:46.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:46.591 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:46.611 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:46.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:46.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:46.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:46.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:46.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:46.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:46.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:46.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:46.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:47.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:47.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:47.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:47.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:47.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:47.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:47.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:47.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:47.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:47.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:47.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:47.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:47.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:47.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:47 smithi016 bash[75179]: cluster 2023-10-09T09:00:45.764657+0000 mgr.y (mgr.25421) 696 : cluster 0 pgmap v693: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 221 objects/s recovering 2023-10-09T09:00:47.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:47 smithi016 bash[77892]: cluster 2023-10-09T09:00:45.764657+0000 mgr.y (mgr.25421) 696 : cluster 0 pgmap v693: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 221 objects/s recovering 2023-10-09T09:00:47.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:47.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:47.891 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:47.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:47 smithi094 bash[59677]: cluster 2023-10-09T09:00:45.764657+0000 mgr.y (mgr.25421) 696 : cluster 0 pgmap v693: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 221 objects/s recovering 2023-10-09T09:00:47.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:47.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:48.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:48.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:48.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:48.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:48.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:48.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:48.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:48.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:48.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:48.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:48.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:48.551 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[75179]: cluster 2023-10-09T09:00:47.511911+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.551 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[75179]: cluster 2023-10-09T09:00:47.512476+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.551 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[75179]: cluster 2023-10-09T09:00:47.513738+0000 mon.a (mon.0) 927 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-10-09T09:00:48.551 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[75179]: cluster 2023-10-09T09:00:47.520589+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:48.552 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[77892]: cluster 2023-10-09T09:00:47.511911+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[77892]: cluster 2023-10-09T09:00:47.512476+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[77892]: cluster 2023-10-09T09:00:47.513738+0000 mon.a (mon.0) 927 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-10-09T09:00:48.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:48 smithi016 bash[77892]: cluster 2023-10-09T09:00:47.520589+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:48.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:48.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:48.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:48.791 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:48 smithi094 bash[59677]: cluster 2023-10-09T09:00:47.511911+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:48 smithi094 bash[59677]: cluster 2023-10-09T09:00:47.512476+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:48 smithi094 bash[59677]: cluster 2023-10-09T09:00:47.513738+0000 mon.a (mon.0) 927 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-10-09T09:00:48.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:48 smithi094 bash[59677]: cluster 2023-10-09T09:00:47.520589+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T09:00:48.812 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:48.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:48.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:48.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:49.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:49.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:49.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:49.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:49.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:49.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:49.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:49.306 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:49.341 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:49.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:49.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:49.445 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:49.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:49.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:49.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:49.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:49.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:49.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:49.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:49.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:49.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:49 smithi094 bash[59677]: cluster 2023-10-09T09:00:47.765059+0000 mgr.y (mgr.25421) 697 : cluster 0 pgmap v695: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 363 objects/s recovering 2023-10-09T09:00:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:49 smithi094 bash[59677]: cluster 2023-10-09T09:00:48.522588+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:49 smithi094 bash[59677]: cluster 2023-10-09T09:00:48.524075+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:49 smithi094 bash[59677]: cluster 2023-10-09T09:00:48.525315+0000 mon.a (mon.0) 928 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-10-09T09:00:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:49 smithi094 bash[59677]: cluster 2023-10-09T09:00:48.525585+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:49 smithi094 bash[59677]: audit 2023-10-09T09:00:49.231338+0000 mon.a (mon.0) 929 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:49 smithi094 bash[59677]: audit 2023-10-09T09:00:49.232454+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:49.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:49.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:49.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:49.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:49.986 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[75179]: cluster 2023-10-09T09:00:47.765059+0000 mgr.y (mgr.25421) 697 : cluster 0 pgmap v695: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 363 objects/s recovering 2023-10-09T09:00:49.986 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[75179]: cluster 2023-10-09T09:00:48.522588+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.986 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[75179]: cluster 2023-10-09T09:00:48.524075+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.986 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[75179]: cluster 2023-10-09T09:00:48.525315+0000 mon.a (mon.0) 928 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[75179]: cluster 2023-10-09T09:00:48.525585+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[75179]: audit 2023-10-09T09:00:49.231338+0000 mon.a (mon.0) 929 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[75179]: audit 2023-10-09T09:00:49.232454+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[77892]: cluster 2023-10-09T09:00:47.765059+0000 mgr.y (mgr.25421) 697 : cluster 0 pgmap v695: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 363 objects/s recovering 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[77892]: cluster 2023-10-09T09:00:48.522588+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[77892]: cluster 2023-10-09T09:00:48.524075+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[77892]: cluster 2023-10-09T09:00:48.525315+0000 mon.a (mon.0) 928 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[77892]: cluster 2023-10-09T09:00:48.525585+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[77892]: audit 2023-10-09T09:00:49.231338+0000 mon.a (mon.0) 929 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:00:49.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:49 smithi016 bash[77892]: audit 2023-10-09T09:00:49.232454+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:00:50.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:50.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:50.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:50.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:50.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:50.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:50.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:50.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:50.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:50.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:50.494 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:50.522 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:50.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.526365+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.528895+0000 mon.a (mon.0) 931 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-10-09T09:00:50.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.529073+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.530948+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.765663+0000 mgr.y (mgr.25421) 698 : cluster 0 pgmap v698: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 270 objects/s recovering 2023-10-09T09:00:50.626 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:50.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:50.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:50.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:50.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:50.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:50.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.526365+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.528895+0000 mon.a (mon.0) 931 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-10-09T09:00:50.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.529073+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.530948+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:50 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.765663+0000 mgr.y (mgr.25421) 698 : cluster 0 pgmap v698: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 270 objects/s recovering 2023-10-09T09:00:50.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:50.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:50.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:50.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:50 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.526365+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:50 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.528895+0000 mon.a (mon.0) 931 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-10-09T09:00:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:50 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.529073+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:50 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.530948+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:50 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.765663+0000 mgr.y (mgr.25421) 698 : cluster 0 pgmap v698: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 270 objects/s recovering 2023-10-09T09:00:51.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:51.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:51.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:51.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:51.285 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:51.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:51.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:51.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:51.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:51.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:51.586 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:51.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:51.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:51 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.862254+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:51 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.863664+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:51 smithi016 bash[75179]: cluster 2023-10-09T09:00:49.865057+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.698 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:51 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.862254+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.698 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:51 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.863664+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:51 smithi016 bash[77892]: cluster 2023-10-09T09:00:49.865057+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:51.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:51.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:51.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:51.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:51 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.862254+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:51 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.863664+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:51 smithi094 bash[59677]: cluster 2023-10-09T09:00:49.865057+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T09:00:51.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:51.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:52.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:52.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:52.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:52.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:52.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:52.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:52.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:52.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:52.393 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:52.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:52.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:52.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:52.529 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:52.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:52.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:52.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:52.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:52.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:52 smithi094 bash[59677]: cluster 2023-10-09T09:00:51.766714+0000 mgr.y (mgr.25421) 699 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 315 objects/s recovering 2023-10-09T09:00:52.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:52.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:52.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:52.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:52.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:52.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:53.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:52 smithi016 bash[75179]: cluster 2023-10-09T09:00:51.766714+0000 mgr.y (mgr.25421) 699 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 315 objects/s recovering 2023-10-09T09:00:53.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:52 smithi016 bash[77892]: cluster 2023-10-09T09:00:51.766714+0000 mgr.y (mgr.25421) 699 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 315 objects/s recovering 2023-10-09T09:00:53.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:53.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:53.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:53.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:53.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:53.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:53.323 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:53.427 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:53.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:53.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:53.562 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:53.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:53.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:53.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:53.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:53.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:53.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:53.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:53.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:53.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:53.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:54.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:54.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:54.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:54.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:54.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:54.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:54.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:54.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:54.380 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:54.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:54.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:54.517 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:54.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:54.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:54.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:54.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:54.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:54.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:54.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:54.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:54.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:54.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:54.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:54.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:55.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:55.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:55.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:55.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:55.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:55.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:55.282 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:55.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:55.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:55.415 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:55.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:55.526 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_1' removed 2023-10-09T09:00:55.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:55.658 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:55.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:55 smithi094 bash[59677]: cluster 2023-10-09T09:00:53.767429+0000 mgr.y (mgr.25421) 700 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 43 objects/s recovering 2023-10-09T09:00:55.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:55 smithi094 bash[59677]: audit 2023-10-09T09:00:55.055127+0000 mon.a (mon.0) 932 : audit 1 from='client.? 172.21.15.16:0/3721956428' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:00:55.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:55.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:55.823 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:55.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:55.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:55.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:55.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:56.025 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:00:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:00:55] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-09T09:00:56.025 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:55 smithi016 bash[77892]: cluster 2023-10-09T09:00:53.767429+0000 mgr.y (mgr.25421) 700 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 43 objects/s recovering 2023-10-09T09:00:56.026 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:55 smithi016 bash[77892]: audit 2023-10-09T09:00:55.055127+0000 mon.a (mon.0) 932 : audit 1 from='client.? 172.21.15.16:0/3721956428' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:00:56.026 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:55 smithi016 bash[75179]: cluster 2023-10-09T09:00:53.767429+0000 mgr.y (mgr.25421) 700 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 43 objects/s recovering 2023-10-09T09:00:56.026 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:55 smithi016 bash[75179]: audit 2023-10-09T09:00:55.055127+0000 mon.a (mon.0) 932 : audit 1 from='client.? 172.21.15.16:0/3721956428' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:00:56.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:56.096 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:56.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:56.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:56.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:56.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:56.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:56.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:00:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:56.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:00:56.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:56.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:56.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:56.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:56.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:56.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:56.740 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:56.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:56 smithi016 bash[75179]: cluster 2023-10-09T09:00:55.499525+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:56.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:56 smithi016 bash[75179]: audit 2023-10-09T09:00:55.506037+0000 mon.a (mon.0) 933 : audit 1 from='client.? 172.21.15.16:0/3721956428' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:00:56.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:56 smithi016 bash[75179]: cluster 2023-10-09T09:00:55.506339+0000 mon.a (mon.0) 934 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-10-09T09:00:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:56 smithi016 bash[77892]: cluster 2023-10-09T09:00:55.499525+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:56 smithi016 bash[77892]: audit 2023-10-09T09:00:55.506037+0000 mon.a (mon.0) 933 : audit 1 from='client.? 172.21.15.16:0/3721956428' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:00:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:56 smithi016 bash[77892]: cluster 2023-10-09T09:00:55.506339+0000 mon.a (mon.0) 934 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-10-09T09:00:56.773 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:56.775 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:56 smithi094 bash[59677]: cluster 2023-10-09T09:00:55.499525+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:56.775 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:56 smithi094 bash[59677]: audit 2023-10-09T09:00:55.506037+0000 mon.a (mon.0) 933 : audit 1 from='client.? 172.21.15.16:0/3721956428' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:00:56.775 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:56 smithi094 bash[59677]: cluster 2023-10-09T09:00:55.506339+0000 mon.a (mon.0) 934 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-10-09T09:00:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:56.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:56.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:56.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:56.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:57.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:57.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:57.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:57.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:57.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:57.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:57.408 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:57.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:57.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:57.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:57.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:57.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:57 smithi016 bash[75179]: cluster 2023-10-09T09:00:55.767783+0000 mgr.y (mgr.25421) 701 : cluster 0 pgmap v702: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:57.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:57 smithi016 bash[75179]: cluster 2023-10-09T09:00:56.306656+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:57.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:57 smithi016 bash[75179]: cluster 2023-10-09T09:00:56.333924+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:57.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:57 smithi016 bash[77892]: cluster 2023-10-09T09:00:55.767783+0000 mgr.y (mgr.25421) 701 : cluster 0 pgmap v702: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:57.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:57 smithi016 bash[77892]: cluster 2023-10-09T09:00:56.306656+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:57.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:57 smithi016 bash[77892]: cluster 2023-10-09T09:00:56.333924+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:57.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:57.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:57 smithi094 bash[59677]: cluster 2023-10-09T09:00:55.767783+0000 mgr.y (mgr.25421) 701 : cluster 0 pgmap v702: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:57 smithi094 bash[59677]: cluster 2023-10-09T09:00:56.306656+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:57.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:57 smithi094 bash[59677]: cluster 2023-10-09T09:00:56.333924+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:57.818 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:57.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:57.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:00:57.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:57.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:57.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:58.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:58.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:58.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:58.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:58.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:00:58.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:58.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:00:58.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:58.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:58.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:58.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:58.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:58.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:58.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:58.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:00:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:58.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:00:58.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:58 smithi016 bash[77892]: cluster 2023-10-09T09:00:56.734381+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:00:58 smithi016 bash[77892]: cluster 2023-10-09T09:00:57.768457+0000 mgr.y (mgr.25421) 702 : cluster 0 pgmap v703: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:58 smithi016 bash[75179]: cluster 2023-10-09T09:00:56.734381+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:00:58 smithi016 bash[75179]: cluster 2023-10-09T09:00:57.768457+0000 mgr.y (mgr.25421) 702 : cluster 0 pgmap v703: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:58.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:58.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:58.871 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:00:58.871 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:00:58.871 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T09:00:58.871 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T09:00:58.871 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:00:58.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:58.884 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:58 smithi094 bash[59677]: cluster 2023-10-09T09:00:56.734381+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T09:00:58.884 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:00:58 smithi094 bash[59677]: cluster 2023-10-09T09:00:57.768457+0000 mgr.y (mgr.25421) 702 : cluster 0 pgmap v703: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:58.900 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:00:58.901 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:00:58.901 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T09:00:58.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T09:00:58.903 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:58.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:59.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:59.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:59.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:59.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:00:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:00:59.207 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:59.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:59.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:59.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:59.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:59.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:59.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:00:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:00:59.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:59.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:00:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:00:59.797 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:59.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:00:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:00:59.936 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:59.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:00:59.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:00:59.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:00.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:00.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:00.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:00.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:00.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:00.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:00.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:00.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:00.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:00.353 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:00.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:00.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:00.465 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:01:00.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:00.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:00.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:00.493 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:01:00.513 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:01:00.537 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:01:00.561 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:01:00.582 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:01:00.591 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:00.605 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:01:00.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:00.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:00.628 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:01:00.630 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.630 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.631 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.631 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.631 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.631 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.631 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.631 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:01:00.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:00.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:00.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:00.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:00.780 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:01:00.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:00.857 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:00 smithi094 bash[59677]: audit 2023-10-09T09:00:59.767995+0000 mon.a (mon.0) 935 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:00.857 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:00 smithi094 bash[59677]: cluster 2023-10-09T09:00:59.768927+0000 mgr.y (mgr.25421) 703 : cluster 0 pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:00.857 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:00 smithi094 bash[59677]: audit 2023-10-09T09:01:00.078954+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.16:0/302817194' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:00.857 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:00 smithi094 bash[59677]: audit 2023-10-09T09:01:00.079348+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:00.857 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:00 smithi094 bash[59677]: audit 2023-10-09T09:01:00.436722+0000 mon.a (mon.0) 937 : audit 0 from='client.? 172.21.15.16:0/1211938575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:00.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:00.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:00.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:01.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:01.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:01.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[75179]: audit 2023-10-09T09:00:59.767995+0000 mon.a (mon.0) 935 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:01.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[75179]: cluster 2023-10-09T09:00:59.768927+0000 mgr.y (mgr.25421) 703 : cluster 0 pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:01.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[75179]: audit 2023-10-09T09:01:00.078954+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.16:0/302817194' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[75179]: audit 2023-10-09T09:01:00.079348+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[75179]: audit 2023-10-09T09:01:00.436722+0000 mon.a (mon.0) 937 : audit 0 from='client.? 172.21.15.16:0/1211938575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:01.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[77892]: audit 2023-10-09T09:00:59.767995+0000 mon.a (mon.0) 935 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:01.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[77892]: cluster 2023-10-09T09:00:59.768927+0000 mgr.y (mgr.25421) 703 : cluster 0 pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:01.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[77892]: audit 2023-10-09T09:01:00.078954+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.16:0/302817194' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:01.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[77892]: audit 2023-10-09T09:01:00.079348+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:01.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:00 smithi016 bash[77892]: audit 2023-10-09T09:01:00.436722+0000 mon.a (mon.0) 937 : audit 0 from='client.? 172.21.15.16:0/1211938575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:01.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:01.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:01.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:01:01.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:01.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:01.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:01.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:01.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:01.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:01.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:01.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:01.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:01.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:01.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:01.797 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:01.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:01.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:01.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:01.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:01.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:02.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:02.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[75179]: cluster 2023-10-09T09:01:00.764228+0000 mon.a (mon.0) 938 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:01:02.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[75179]: cluster 2023-10-09T09:01:00.764261+0000 mon.a (mon.0) 939 : cluster 1 Cluster is now healthy 2023-10-09T09:01:02.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[75179]: cluster 2023-10-09T09:01:00.772382+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:02.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[75179]: audit 2023-10-09T09:01:00.774689+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:01:02.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[75179]: cluster 2023-10-09T09:01:00.774746+0000 mon.a (mon.0) 941 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-10-09T09:01:02.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[77892]: cluster 2023-10-09T09:01:00.764228+0000 mon.a (mon.0) 938 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:01:02.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[77892]: cluster 2023-10-09T09:01:00.764261+0000 mon.a (mon.0) 939 : cluster 1 Cluster is now healthy 2023-10-09T09:01:02.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[77892]: cluster 2023-10-09T09:01:00.772382+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:02.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[77892]: audit 2023-10-09T09:01:00.774689+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:01:02.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:01 smithi016 bash[77892]: cluster 2023-10-09T09:01:00.774746+0000 mon.a (mon.0) 941 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-10-09T09:01:02.046 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:01 smithi094 bash[59677]: cluster 2023-10-09T09:01:00.764228+0000 mon.a (mon.0) 938 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:01:02.046 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:01 smithi094 bash[59677]: cluster 2023-10-09T09:01:00.764261+0000 mon.a (mon.0) 939 : cluster 1 Cluster is now healthy 2023-10-09T09:01:02.046 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:01 smithi094 bash[59677]: cluster 2023-10-09T09:01:00.772382+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:02.046 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:01 smithi094 bash[59677]: audit 2023-10-09T09:01:00.774689+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:01:02.046 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:01 smithi094 bash[59677]: cluster 2023-10-09T09:01:00.774746+0000 mon.a (mon.0) 941 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-10-09T09:01:02.065 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:02.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:02.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:02.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:02.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:02.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:02.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:02.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:02.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:02.440 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:02.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:02.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:02.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:02.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:02.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:02.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:02.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:02.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:02.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:02.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:02.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:02.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:02.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:03.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:03.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[75179]: cluster 2023-10-09T09:01:00.969044+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[75179]: cluster 2023-10-09T09:01:01.769586+0000 mgr.y (mgr.25421) 704 : cluster 0 pgmap v706: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:03.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[75179]: cluster 2023-10-09T09:01:02.124266+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[75179]: cluster 2023-10-09T09:01:02.124376+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[77892]: cluster 2023-10-09T09:01:00.969044+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[77892]: cluster 2023-10-09T09:01:01.769586+0000 mgr.y (mgr.25421) 704 : cluster 0 pgmap v706: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:03.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[77892]: cluster 2023-10-09T09:01:02.124266+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:02 smithi016 bash[77892]: cluster 2023-10-09T09:01:02.124376+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:03.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:03.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:03.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:03.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:02 smithi094 bash[59677]: cluster 2023-10-09T09:01:00.969044+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:02 smithi094 bash[59677]: cluster 2023-10-09T09:01:01.769586+0000 mgr.y (mgr.25421) 704 : cluster 0 pgmap v706: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:03.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:02 smithi094 bash[59677]: cluster 2023-10-09T09:01:02.124266+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:02 smithi094 bash[59677]: cluster 2023-10-09T09:01:02.124376+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T09:01:03.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:03.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:03.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:03.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:03.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:03.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:03.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:03.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:03.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:03.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:03.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:03.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:03.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:03.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:04.029 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:04.042 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:03 smithi094 bash[59677]: audit 2023-10-09T09:01:02.835621+0000 mon.a (mon.0) 942 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:04.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:04.165 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:04.196 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:04.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:03 smithi016 bash[75179]: audit 2023-10-09T09:01:02.835621+0000 mon.a (mon.0) 942 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:04.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:03 smithi016 bash[77892]: audit 2023-10-09T09:01:02.835621+0000 mon.a (mon.0) 942 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:04.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:04.331 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:04.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:04.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:04.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:04.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:04.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:04.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:04.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:04.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:04.834 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:04.861 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:04.964 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:04.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:04.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:05.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:05.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:05.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:05.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:05.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:05.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:05.363 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:05.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:05.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:05.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:05.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:01:05.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:05 smithi016 bash[75179]: cluster 2023-10-09T09:01:03.770392+0000 mgr.y (mgr.25421) 705 : cluster 0 pgmap v707: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:05.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:05 smithi016 bash[75179]: audit 2023-10-09T09:01:04.230770+0000 mon.a (mon.0) 943 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:05.428 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:05 smithi016 bash[75179]: audit 2023-10-09T09:01:04.232147+0000 mon.a (mon.0) 944 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:05.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:05 smithi016 bash[77892]: cluster 2023-10-09T09:01:03.770392+0000 mgr.y (mgr.25421) 705 : cluster 0 pgmap v707: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:05.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:05 smithi016 bash[77892]: audit 2023-10-09T09:01:04.230770+0000 mon.a (mon.0) 943 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:05.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:05 smithi016 bash[77892]: audit 2023-10-09T09:01:04.232147+0000 mon.a (mon.0) 944 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:05 smithi094 bash[59677]: cluster 2023-10-09T09:01:03.770392+0000 mgr.y (mgr.25421) 705 : cluster 0 pgmap v707: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:05 smithi094 bash[59677]: audit 2023-10-09T09:01:04.230770+0000 mon.a (mon.0) 943 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:05 smithi094 bash[59677]: audit 2023-10-09T09:01:04.232147+0000 mon.a (mon.0) 944 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:05.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:05.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:05.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:05.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:01:05.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:05.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:05.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:05.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:05.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:05.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:05.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:05.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:05.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:05.927 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:05.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:05.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:01:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:01:05] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.33.4" 2023-10-09T09:01:06.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:06.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:06.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:06.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:06.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:06.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:06.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:06.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:06.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:06.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:06.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:06.490 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:01:06.490 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "" 2023-10-09T09:01:06.491 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:01:06.552 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:06.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:06.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:06.686 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:06.712 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:06.816 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:06.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:06.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:06.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:06.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:06.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:07.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:07.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:07.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:07.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:07.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:07.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:07.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:07.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:07.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:07.349 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:07 smithi016 bash[75179]: cluster 2023-10-09T09:01:05.770939+0000 mgr.y (mgr.25421) 706 : cluster 0 pgmap v708: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:07.349 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:07 smithi016 bash[77892]: cluster 2023-10-09T09:01:05.770939+0000 mgr.y (mgr.25421) 706 : cluster 0 pgmap v708: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:07.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:07.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:07.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:07.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:07 smithi094 bash[59677]: cluster 2023-10-09T09:01:05.770939+0000 mgr.y (mgr.25421) 706 : cluster 0 pgmap v708: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:07.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:07.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:07.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:07.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:07.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:07.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:07.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:07.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:07.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:07.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:07.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:07.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:07.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:07.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:08.012 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:08.116 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:08.174 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:08.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:08.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:08.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:08.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:08.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:08.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:08.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:08.586 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:08.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:08.716 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:08.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:08.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:08.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:08.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:08.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:08.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:08.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:09.004 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:09.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:09.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:09.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:09.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:09.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:09.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:09.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:09.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:09.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:09.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:09.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:09 smithi094 bash[59677]: cluster 2023-10-09T09:01:07.771724+0000 mgr.y (mgr.25421) 707 : cluster 0 pgmap v709: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:09 smithi094 bash[59677]: audit 2023-10-09T09:01:09.125887+0000 mon.a (mon.0) 945 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:09 smithi094 bash[59677]: audit 2023-10-09T09:01:09.132560+0000 mon.a (mon.0) 946 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:09.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:09 smithi016 bash[75179]: cluster 2023-10-09T09:01:07.771724+0000 mgr.y (mgr.25421) 707 : cluster 0 pgmap v709: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:09 smithi016 bash[75179]: audit 2023-10-09T09:01:09.125887+0000 mon.a (mon.0) 945 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:09 smithi016 bash[75179]: audit 2023-10-09T09:01:09.132560+0000 mon.a (mon.0) 946 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:09.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:09 smithi016 bash[77892]: cluster 2023-10-09T09:01:07.771724+0000 mgr.y (mgr.25421) 707 : cluster 0 pgmap v709: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:09.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:09 smithi016 bash[77892]: audit 2023-10-09T09:01:09.125887+0000 mon.a (mon.0) 945 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:09.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:09 smithi016 bash[77892]: audit 2023-10-09T09:01:09.132560+0000 mon.a (mon.0) 946 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:09.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:09.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:09.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:09.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:09.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:09.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:09.735 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:09.763 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:09.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:09.867 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:09.902 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:09.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:10.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:10.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:10.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:10.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:10.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:10.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:10.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:10.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:10.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:10.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:10.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:10.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:10.584 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:10.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:10.688 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:10.719 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:10.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:10.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:10.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:10.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:10.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:11.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:11.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:11.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:11.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:11.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:11.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:11.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:11.363 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:11.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:11.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:11.501 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:11.531 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:11.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:11.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:11.566 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:11.567 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:11.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:11.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:11.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:11 smithi094 bash[59677]: cluster 2023-10-09T09:01:09.772249+0000 mgr.y (mgr.25421) 708 : cluster 0 pgmap v710: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:11 smithi094 bash[59677]: audit 2023-10-09T09:01:10.787485+0000 mon.a (mon.0) 947 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:11 smithi094 bash[59677]: audit 2023-10-09T09:01:10.794217+0000 mon.a (mon.0) 948 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:11 smithi094 bash[59677]: audit 2023-10-09T09:01:11.416034+0000 mon.a (mon.0) 949 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:11 smithi094 bash[59677]: audit 2023-10-09T09:01:11.417574+0000 mon.a (mon.0) 950 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:11 smithi094 bash[59677]: audit 2023-10-09T09:01:11.425852+0000 mon.a (mon.0) 951 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:11.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:11.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:11.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[75179]: cluster 2023-10-09T09:01:09.772249+0000 mgr.y (mgr.25421) 708 : cluster 0 pgmap v710: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:11.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[75179]: audit 2023-10-09T09:01:10.787485+0000 mon.a (mon.0) 947 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[75179]: audit 2023-10-09T09:01:10.794217+0000 mon.a (mon.0) 948 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[75179]: audit 2023-10-09T09:01:11.416034+0000 mon.a (mon.0) 949 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[75179]: audit 2023-10-09T09:01:11.417574+0000 mon.a (mon.0) 950 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[75179]: audit 2023-10-09T09:01:11.425852+0000 mon.a (mon.0) 951 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[77892]: cluster 2023-10-09T09:01:09.772249+0000 mgr.y (mgr.25421) 708 : cluster 0 pgmap v710: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[77892]: audit 2023-10-09T09:01:10.787485+0000 mon.a (mon.0) 947 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[77892]: audit 2023-10-09T09:01:10.794217+0000 mon.a (mon.0) 948 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[77892]: audit 2023-10-09T09:01:11.416034+0000 mon.a (mon.0) 949 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:11.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[77892]: audit 2023-10-09T09:01:11.417574+0000 mon.a (mon.0) 950 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:11.790 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:11 smithi016 bash[77892]: audit 2023-10-09T09:01:11.425852+0000 mon.a (mon.0) 951 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:11.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:11.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:11.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:11.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:12.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:12.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:12.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:12.178 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:12.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:12.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:12.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:12.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:12.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:12.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:12.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:12.664 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:12.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:12.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:12.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:12.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:12.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:12.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:12.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:12.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:13.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:13.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:13.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:13.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:13.224 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:13.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:13.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:13.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:13.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:13.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:13.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:13.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:13.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:13.528 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 4, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:01:13.528 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:01:13.528 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 5, 4, 6] 2023-10-09T09:01:13.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T09:01:13.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:13.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:13.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:13.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:13.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:13.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:13 smithi016 bash[75179]: cluster 2023-10-09T09:01:11.772898+0000 mgr.y (mgr.25421) 709 : cluster 0 pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:13 smithi016 bash[77892]: cluster 2023-10-09T09:01:11.772898+0000 mgr.y (mgr.25421) 709 : cluster 0 pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:13.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:13.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:13.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:13.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:13.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:13.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:13 smithi094 bash[59677]: cluster 2023-10-09T09:01:11.772898+0000 mgr.y (mgr.25421) 709 : cluster 0 pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:14.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:14.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:14.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:14.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:14.272 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:14.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:14.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:14.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:14.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:14.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:14.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:14.666 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:14.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:14.770 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:14.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:14.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:14.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:14.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:14.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:15.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:15.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:15.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:15.173 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:15.205 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:15.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:15.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:15.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:15.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:15.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:15.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:15.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:15.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:15.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:15.802 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_2' created 2023-10-09T09:01:15.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:15.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:15.935 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:15 smithi016 bash[75179]: cluster 2023-10-09T09:01:13.773623+0000 mgr.y (mgr.25421) 710 : cluster 0 pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:15.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:15 smithi016 bash[77892]: cluster 2023-10-09T09:01:13.773623+0000 mgr.y (mgr.25421) 710 : cluster 0 pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:15.936 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:01:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:01:15] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.33.4" 2023-10-09T09:01:15.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:15.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:15.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:15.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:16.069 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:16.081 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:15 smithi094 bash[59677]: cluster 2023-10-09T09:01:13.773623+0000 mgr.y (mgr.25421) 710 : cluster 0 pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:16.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:16.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:16.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:16.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:16.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:16.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:16.345 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:16.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:16.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:16.482 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:16.513 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:16.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:16.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:16.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:16.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:16.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:16.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:16.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:16.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:16.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:16.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:16.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:16.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:16.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:17.000 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:17.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:16 smithi094 bash[59677]: audit 2023-10-09T09:01:15.297688+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.16:0/3633959223' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:01:17.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:16 smithi094 bash[59677]: audit 2023-10-09T09:01:15.298455+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:01:17.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:16 smithi094 bash[59677]: cluster 2023-10-09T09:01:15.774113+0000 mgr.y (mgr.25421) 711 : cluster 0 pgmap v713: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:17.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:16 smithi094 bash[59677]: cluster 2023-10-09T09:01:15.794228+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T09:01:17.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:16 smithi094 bash[59677]: audit 2023-10-09T09:01:15.795297+0000 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T09:01:17.014 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:16 smithi094 bash[59677]: cluster 2023-10-09T09:01:15.795385+0000 mon.a (mon.0) 954 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-10-09T09:01:17.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:17.080 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[75179]: audit 2023-10-09T09:01:15.297688+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.16:0/3633959223' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:01:17.080 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[75179]: audit 2023-10-09T09:01:15.298455+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:01:17.080 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[75179]: cluster 2023-10-09T09:01:15.774113+0000 mgr.y (mgr.25421) 711 : cluster 0 pgmap v713: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[75179]: cluster 2023-10-09T09:01:15.794228+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[75179]: audit 2023-10-09T09:01:15.795297+0000 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[75179]: cluster 2023-10-09T09:01:15.795385+0000 mon.a (mon.0) 954 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[77892]: audit 2023-10-09T09:01:15.297688+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.16:0/3633959223' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[77892]: audit 2023-10-09T09:01:15.298455+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[77892]: cluster 2023-10-09T09:01:15.774113+0000 mgr.y (mgr.25421) 711 : cluster 0 pgmap v713: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[77892]: cluster 2023-10-09T09:01:15.794228+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[77892]: audit 2023-10-09T09:01:15.795297+0000 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T09:01:17.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:16 smithi016 bash[77892]: cluster 2023-10-09T09:01:15.795385+0000 mon.a (mon.0) 954 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-10-09T09:01:17.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:17.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:17.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:17.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:17.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:17.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:17.423 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:17.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:17.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:17.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:17.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:17.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:17.788 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:17.816 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:17.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:17.947 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:17.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:18.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:18.076 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:18.077 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[75179]: cluster 2023-10-09T09:01:16.795516+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.077 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[75179]: cluster 2023-10-09T09:01:16.796487+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.077 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[75179]: cluster 2023-10-09T09:01:16.797618+0000 mon.a (mon.0) 955 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-10-09T09:01:18.078 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[75179]: cluster 2023-10-09T09:01:16.798063+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.078 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[75179]: cluster 2023-10-09T09:01:16.802192+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.078 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[75179]: cluster 2023-10-09T09:01:16.802296+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[77892]: cluster 2023-10-09T09:01:16.795516+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[77892]: cluster 2023-10-09T09:01:16.796487+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[77892]: cluster 2023-10-09T09:01:16.797618+0000 mon.a (mon.0) 955 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-10-09T09:01:18.079 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[77892]: cluster 2023-10-09T09:01:16.798063+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.079 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[77892]: cluster 2023-10-09T09:01:16.802192+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.079 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:17 smithi016 bash[77892]: cluster 2023-10-09T09:01:16.802296+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:18.180 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:18.194 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:17 smithi094 bash[59677]: cluster 2023-10-09T09:01:16.795516+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.194 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:17 smithi094 bash[59677]: cluster 2023-10-09T09:01:16.796487+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.194 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:17 smithi094 bash[59677]: cluster 2023-10-09T09:01:16.797618+0000 mon.a (mon.0) 955 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-10-09T09:01:18.195 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:17 smithi094 bash[59677]: cluster 2023-10-09T09:01:16.798063+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.195 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:17 smithi094 bash[59677]: cluster 2023-10-09T09:01:16.802192+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.195 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:17 smithi094 bash[59677]: cluster 2023-10-09T09:01:16.802296+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T09:01:18.212 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:18.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:18.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:18.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:18.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:18.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:18.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:18.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:18.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:18.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:18.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:18.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:18.837 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[75179]: cluster 2023-10-09T09:01:17.774889+0000 mgr.y (mgr.25421) 712 : cluster 0 pgmap v716: 89 pgs: 8 creating+peering, 5 unknown, 76 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:18.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[75179]: cluster 2023-10-09T09:01:17.802796+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:18.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[75179]: cluster 2023-10-09T09:01:17.805581+0000 mon.a (mon.0) 956 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-10-09T09:01:18.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[75179]: cluster 2023-10-09T09:01:17.967464+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:18.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[75179]: cluster 2023-10-09T09:01:18.231488+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:18.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:18.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[77892]: cluster 2023-10-09T09:01:17.774889+0000 mgr.y (mgr.25421) 712 : cluster 0 pgmap v716: 89 pgs: 8 creating+peering, 5 unknown, 76 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:18.841 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[77892]: cluster 2023-10-09T09:01:17.802796+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:18.841 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[77892]: cluster 2023-10-09T09:01:17.805581+0000 mon.a (mon.0) 956 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-10-09T09:01:18.841 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[77892]: cluster 2023-10-09T09:01:17.967464+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:18.841 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:18 smithi016 bash[77892]: cluster 2023-10-09T09:01:18.231488+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:18.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:18.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:18.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:18.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:19.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:19.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:19.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:19.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:19.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:18 smithi094 bash[59677]: cluster 2023-10-09T09:01:17.774889+0000 mgr.y (mgr.25421) 712 : cluster 0 pgmap v716: 89 pgs: 8 creating+peering, 5 unknown, 76 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:19.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:18 smithi094 bash[59677]: cluster 2023-10-09T09:01:17.802796+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:19.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:18 smithi094 bash[59677]: cluster 2023-10-09T09:01:17.805581+0000 mon.a (mon.0) 956 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-10-09T09:01:19.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:18 smithi094 bash[59677]: cluster 2023-10-09T09:01:17.967464+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:19.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:18 smithi094 bash[59677]: cluster 2023-10-09T09:01:18.231488+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:19.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:19.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:19.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:19.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:19.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:19.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:19.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T09:01:19.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:19.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:19.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:19.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:19.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:19.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:19.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:19.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:19.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:20.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:19 smithi016 bash[75179]: cluster 2023-10-09T09:01:18.231499+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:20.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:19 smithi016 bash[75179]: audit 2023-10-09T09:01:19.226592+0000 mon.a (mon.0) 957 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:20.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:19 smithi016 bash[77892]: cluster 2023-10-09T09:01:18.231499+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:20.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:19 smithi016 bash[77892]: audit 2023-10-09T09:01:19.226592+0000 mon.a (mon.0) 957 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:20.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.032 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:19 smithi094 bash[59677]: cluster 2023-10-09T09:01:18.231499+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T09:01:20.032 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:19 smithi094 bash[59677]: audit 2023-10-09T09:01:19.226592+0000 mon.a (mon.0) 957 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:20.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:20.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:20.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:20.291 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.323 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:20.426 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:20.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:20.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:20.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:20.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:20.687 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.716 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:20.820 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.854 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:20.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.991 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:20.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:21.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:21.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:21.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:21.233 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:21.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:21.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:21.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:21.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:21.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:21.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:21.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:21.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:21.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:21.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:21.672 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:21.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:21.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:21.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:21.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:21 smithi016 bash[75179]: cluster 2023-10-09T09:01:19.775308+0000 mgr.y (mgr.25421) 713 : cluster 0 pgmap v718: 89 pgs: 8 creating+peering, 5 unknown, 76 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:21.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:21 smithi016 bash[77892]: cluster 2023-10-09T09:01:19.775308+0000 mgr.y (mgr.25421) 713 : cluster 0 pgmap v718: 89 pgs: 8 creating+peering, 5 unknown, 76 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:21.788 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:21 smithi094 bash[59677]: cluster 2023-10-09T09:01:19.775308+0000 mgr.y (mgr.25421) 713 : cluster 0 pgmap v718: 89 pgs: 8 creating+peering, 5 unknown, 76 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:21.805 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:21.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:21.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:21.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:21.942 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:21.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:21.964 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:01:21.964 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:01:22.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:22.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:22.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:22.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:22.207 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:22.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:22.310 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:22.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:22.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:22.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:22.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:22.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:22.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:22.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:22.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:22.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:22.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:22.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:22.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:22.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:22.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:22.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:22.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:22.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:22.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:23.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:23.103 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:23.136 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:23.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:23.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:23.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:23.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:23.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:23.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:23.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:23.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:23.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:23.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:23.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:23.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:23 smithi094 bash[59677]: cluster 2023-10-09T09:01:21.776141+0000 mgr.y (mgr.25421) 714 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:23.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:23.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:23 smithi016 bash[75179]: cluster 2023-10-09T09:01:21.776141+0000 mgr.y (mgr.25421) 714 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:23.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:23 smithi016 bash[77892]: cluster 2023-10-09T09:01:21.776141+0000 mgr.y (mgr.25421) 714 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:23.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:23.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:23.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:23.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:23.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:23.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:23.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:01:24.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:24.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:24.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:24.158 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:24.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:24.254 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:01:24.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:24.294 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:01:24.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:24.324 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:01:24.365 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:01:24.400 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:01:24.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:24.424 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:01:24.437 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:24.439 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:24.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:24.456 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:01:24.484 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:01:24.486 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.486 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.486 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.486 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.487 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.487 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.487 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.487 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:01:24.524 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:01:24.553 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:24.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:24 smithi016 bash[77892]: audit 2023-10-09T09:01:24.227574+0000 mon.a (mon.0) 958 : audit 0 from='client.? 172.21.15.16:0/2497701447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:24.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:24 smithi016 bash[77892]: audit 2023-10-09T09:01:24.423088+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.16:0/944426087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:01:24.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:24 smithi016 bash[75179]: audit 2023-10-09T09:01:24.227574+0000 mon.a (mon.0) 958 : audit 0 from='client.? 172.21.15.16:0/2497701447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:24.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:24 smithi016 bash[75179]: audit 2023-10-09T09:01:24.423088+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.16:0/944426087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:01:24.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:24.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:24.679 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:24.713 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:24.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:24.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:24.830 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:24 smithi094 bash[59677]: audit 2023-10-09T09:01:24.227574+0000 mon.a (mon.0) 958 : audit 0 from='client.? 172.21.15.16:0/2497701447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:24.830 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:24 smithi094 bash[59677]: audit 2023-10-09T09:01:24.423088+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.16:0/944426087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:01:24.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:24.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:24.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:24.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:25.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:25.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:25.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:25.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:25.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:25.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:25.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:25.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:25.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:25.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:25.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:25.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:25.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:25.625 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:25.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:25.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:25.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[75179]: cluster 2023-10-09T09:01:23.776849+0000 mgr.y (mgr.25421) 715 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[75179]: cluster 2023-10-09T09:01:24.509966+0000 mon.a (mon.0) 960 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:01:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[75179]: cluster 2023-10-09T09:01:24.516078+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[75179]: cluster 2023-10-09T09:01:24.516329+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[75179]: audit 2023-10-09T09:01:24.519545+0000 mon.a (mon.0) 961 : audit 1 from='client.? 172.21.15.16:0/944426087' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:01:25.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[75179]: cluster 2023-10-09T09:01:24.519607+0000 mon.a (mon.0) 962 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-10-09T09:01:25.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[77892]: cluster 2023-10-09T09:01:23.776849+0000 mgr.y (mgr.25421) 715 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:25.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[77892]: cluster 2023-10-09T09:01:24.509966+0000 mon.a (mon.0) 960 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:01:25.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[77892]: cluster 2023-10-09T09:01:24.516078+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:25.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[77892]: cluster 2023-10-09T09:01:24.516329+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:25.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[77892]: audit 2023-10-09T09:01:24.519545+0000 mon.a (mon.0) 961 : audit 1 from='client.? 172.21.15.16:0/944426087' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:01:25.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[77892]: cluster 2023-10-09T09:01:24.519607+0000 mon.a (mon.0) 962 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-10-09T09:01:25.761 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:25.774 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:25 smithi094 bash[59677]: cluster 2023-10-09T09:01:23.776849+0000 mgr.y (mgr.25421) 715 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:25.774 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:25 smithi094 bash[59677]: cluster 2023-10-09T09:01:24.509966+0000 mon.a (mon.0) 960 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:01:25.774 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:25 smithi094 bash[59677]: cluster 2023-10-09T09:01:24.516078+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:25.774 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:25 smithi094 bash[59677]: cluster 2023-10-09T09:01:24.516329+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:25.774 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:25 smithi094 bash[59677]: audit 2023-10-09T09:01:24.519545+0000 mon.a (mon.0) 961 : audit 1 from='client.? 172.21.15.16:0/944426087' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:01:25.774 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:25 smithi094 bash[59677]: cluster 2023-10-09T09:01:24.519607+0000 mon.a (mon.0) 962 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-10-09T09:01:25.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:25.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:25.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:25.926 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:25.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:26.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:01:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:01:25] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-09T09:01:26.029 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:26.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:26.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:26.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:26.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:26.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:26.322 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:26.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:26.425 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:26.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:26.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:26.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:26.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:26.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:26 smithi094 bash[59677]: cluster 2023-10-09T09:01:25.569059+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:26.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:26 smithi016 bash[75179]: cluster 2023-10-09T09:01:25.569059+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:26.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:26 smithi016 bash[77892]: cluster 2023-10-09T09:01:25.569059+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T09:01:26.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:26.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:26.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:27.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:27.109 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:27.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:27.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:27.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:27.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:27.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:27.346 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:27.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:27.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:27.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:27.481 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:27.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:27.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:27.613 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:27.626 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:27 smithi094 bash[59677]: cluster 2023-10-09T09:01:25.777491+0000 mgr.y (mgr.25421) 716 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:27.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:27.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:27.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:27.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:27.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:27.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:27.882 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:27 smithi016 bash[75179]: cluster 2023-10-09T09:01:25.777491+0000 mgr.y (mgr.25421) 716 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:27.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:27 smithi016 bash[77892]: cluster 2023-10-09T09:01:25.777491+0000 mgr.y (mgr.25421) 716 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:27.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:27.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:27.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:27.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:28.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:28.130 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:28.161 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:28.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:28.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:28.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:28.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:28.434 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:28.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:28.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:28.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:28.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:28.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:28.680 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:28 smithi094 bash[59677]: cluster 2023-10-09T09:01:27.778456+0000 mgr.y (mgr.25421) 717 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:28.700 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:28.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:28.742 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:01:28.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:28.811 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:28 smithi016 bash[77892]: cluster 2023-10-09T09:01:27.778456+0000 mgr.y (mgr.25421) 717 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:28.812 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:28 smithi016 bash[75179]: cluster 2023-10-09T09:01:27.778456+0000 mgr.y (mgr.25421) 717 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:28.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:28.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:28.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:28.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:28.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:28.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:29.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:29.090 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:29.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:29.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:29.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:29.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:29.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:29.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:29.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:29.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:29.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:29.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:29.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:29.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:29.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:29.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:29.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:29.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:29.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:29.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:29.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:29.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:29.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:29.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:29.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:30.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:30.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:30.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:30.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:30.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:30.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:30.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:30.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:30.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:30.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:30.648 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:30.674 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:30.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:30.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:30.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:01:30.903 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:30.916 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:30 smithi094 bash[59677]: cluster 2023-10-09T09:01:29.779130+0000 mgr.y (mgr.25421) 718 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:30.916 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:30 smithi094 bash[59677]: audit 2023-10-09T09:01:29.788293+0000 mon.a (mon.0) 963 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:30.930 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:30.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:31.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:01:31.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:30 smithi016 bash[75179]: cluster 2023-10-09T09:01:29.779130+0000 mgr.y (mgr.25421) 718 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:31.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:30 smithi016 bash[75179]: audit 2023-10-09T09:01:29.788293+0000 mon.a (mon.0) 963 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:30 smithi016 bash[77892]: cluster 2023-10-09T09:01:29.779130+0000 mgr.y (mgr.25421) 718 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:31.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:30 smithi016 bash[77892]: audit 2023-10-09T09:01:29.788293+0000 mon.a (mon.0) 963 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:31.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:31.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:31.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:31.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:31.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:31.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:31.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:31.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:31.699 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:31.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:31.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:31.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:31.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:31.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:31.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:32.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:32.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:32.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:32.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:32.240 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:32.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:32.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:32.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:32.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:32.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:32.504 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:32.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:32.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:32.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:32.581 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:01:32.581 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:01:32.581 INFO:teuthology.orchestra.run.smithi094.stdout: "num_ops": 0 2023-10-09T09:01:32.581 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:01:32.633 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:32.661 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:32.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:32.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:32.793 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:32.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:32.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:32.908 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:32 smithi094 bash[59677]: cluster 2023-10-09T09:01:31.780306+0000 mgr.y (mgr.25421) 719 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:32.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:32.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:33.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:33.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:33.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:32 smithi016 bash[75179]: cluster 2023-10-09T09:01:31.780306+0000 mgr.y (mgr.25421) 719 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:33.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:32 smithi016 bash[77892]: cluster 2023-10-09T09:01:31.780306+0000 mgr.y (mgr.25421) 719 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:33.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:33.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:33.155 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:33.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:33.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:33.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:33.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:33.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:33.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:33.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:33.417 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:33.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:33.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:33.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:33.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:01:33.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:33.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:33.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:33.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:33.732 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:33.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:33.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:33.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:33.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:33.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:33.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:33.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:33.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:34.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:34.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:34.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:34.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:34.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:34.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:34.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:34.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:34 smithi016 bash[75179]: debug 2023-10-09T09:01:34.297+0000 7fa046a4d700 -1 mon.a@0(leader).osd e175 definitely_dead 0 2023-10-09T09:01:34.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:34.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:34.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:34.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:34.653 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:34.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:34.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:34.787 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:34.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:34.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:34.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:35.028 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:35.062 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:35.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:35.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:35.166 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:35.199 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:35.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:35.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:35.274 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T09:01:35.275 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.5. 2023-10-09T09:01:35.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:35.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:35.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:35.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:35.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:35.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:35.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: cluster 2023-10-09T09:01:33.781135+0000 mgr.y (mgr.25421) 720 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: audit 2023-10-09T09:01:34.231654+0000 mon.a (mon.0) 964 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: audit 2023-10-09T09:01:34.233152+0000 mon.a (mon.0) 965 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: audit 2023-10-09T09:01:34.303148+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.16:0/4212184679' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:01:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: audit 2023-10-09T09:01:34.303794+0000 mon.a (mon.0) 966 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:01:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: cluster 2023-10-09T09:01:34.304003+0000 mon.a (mon.0) 967 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T09:01:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: audit 2023-10-09T09:01:34.473892+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.16:0/903035595' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:35 smithi094 bash[59677]: audit 2023-10-09T09:01:34.474704+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: cluster 2023-10-09T09:01:33.781135+0000 mgr.y (mgr.25421) 720 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: audit 2023-10-09T09:01:34.231654+0000 mon.a (mon.0) 964 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: audit 2023-10-09T09:01:34.233152+0000 mon.a (mon.0) 965 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: audit 2023-10-09T09:01:34.303148+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.16:0/4212184679' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: audit 2023-10-09T09:01:34.303794+0000 mon.a (mon.0) 966 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: cluster 2023-10-09T09:01:34.304003+0000 mon.a (mon.0) 967 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: audit 2023-10-09T09:01:34.473892+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.16:0/903035595' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:35.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[77892]: audit 2023-10-09T09:01:34.474704+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: cluster 2023-10-09T09:01:33.781135+0000 mgr.y (mgr.25421) 720 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: audit 2023-10-09T09:01:34.231654+0000 mon.a (mon.0) 964 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: audit 2023-10-09T09:01:34.233152+0000 mon.a (mon.0) 965 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: audit 2023-10-09T09:01:34.303148+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.16:0/4212184679' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: audit 2023-10-09T09:01:34.303794+0000 mon.a (mon.0) 966 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: cluster 2023-10-09T09:01:34.304003+0000 mon.a (mon.0) 967 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: audit 2023-10-09T09:01:34.473892+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.16:0/903035595' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:35.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[75179]: audit 2023-10-09T09:01:34.474704+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:35.562 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:35.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:35.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:35.699 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:35.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:35.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:35.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:35.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:35.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:35.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:35.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:35.985 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:01:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:01:35] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.33.4" 2023-10-09T09:01:36.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:36.110 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:36.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:36.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:36.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: audit 2023-10-09T09:01:35.243661+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T09:01:36.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: audit 2023-10-09T09:01:35.243779+0000 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:01:36.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.243869+0000 mon.a (mon.0) 971 : cluster 0 osdmap e176: 8 total, 8 up, 6 in 2023-10-09T09:01:36.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.244477+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.244495+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.246001+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.248437+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.249177+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.251242+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.251394+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.330670+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:36.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:36.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:36.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: audit 2023-10-09T09:01:35.243661+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T09:01:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: audit 2023-10-09T09:01:35.243779+0000 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:01:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.243869+0000 mon.a (mon.0) 971 : cluster 0 osdmap e176: 8 total, 8 up, 6 in 2023-10-09T09:01:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.244477+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.244495+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.246001+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.248437+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.249177+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.251242+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.251394+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:36 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.330670+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: audit 2023-10-09T09:01:35.243661+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: audit 2023-10-09T09:01:35.243779+0000 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.243869+0000 mon.a (mon.0) 971 : cluster 0 osdmap e176: 8 total, 8 up, 6 in 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.244477+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.244495+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.246001+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.248437+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.249177+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.251242+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.251394+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:36 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.330670+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T09:01:36.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:36.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:36.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:36.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:36.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:36.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:36.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:36.925 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:36.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:36.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:37.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:37.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:37.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:37.194 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:37.195 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:37.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:37.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:37.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:37.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:37.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:37.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:37.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:37.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:37.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:37.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:37.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:35.781691+0000 mgr.y (mgr.25421) 721 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:01:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.253460+0000 mon.a (mon.0) 972 : cluster 0 osdmap e177: 8 total, 8 up, 6 in 2023-10-09T09:01:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.253791+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.255839+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.259392+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.260036+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.262400+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.265234+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.571555+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.574514+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.577325+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[75179]: cluster 2023-10-09T09:01:36.580101+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:35.781691+0000 mgr.y (mgr.25421) 721 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.253460+0000 mon.a (mon.0) 972 : cluster 0 osdmap e177: 8 total, 8 up, 6 in 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.253791+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.255839+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.259392+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.260036+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.262400+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.265234+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.571555+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.574514+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.577325+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:37 smithi016 bash[77892]: cluster 2023-10-09T09:01:36.580101+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:37.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:37.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:37.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:37.734 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:35.781691+0000 mgr.y (mgr.25421) 721 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:01:37.734 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.253460+0000 mon.a (mon.0) 972 : cluster 0 osdmap e177: 8 total, 8 up, 6 in 2023-10-09T09:01:37.734 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.253791+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.734 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.255839+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.734 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.259392+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.260036+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.262400+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.265234+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.571555+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.574514+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.577325+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:37 smithi094 bash[59677]: cluster 2023-10-09T09:01:36.580101+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T09:01:37.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:37.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:37.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:37.822 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T09:01:37.824 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:01:37.825 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:01:37.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:37.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-10-09T09:01:37.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:01:37.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_151580 2023-10-09T09:01:37.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:01:37.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:01:37.894 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:37.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:37.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:38.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:38.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:38.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:38.158 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:38.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:38.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:38.300 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:01:38.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:38.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:38.427 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:38.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:38.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:38.563 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:38 smithi016 bash[75179]: cluster 2023-10-09T09:01:38.249457+0000 mon.a (mon.0) 973 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T09:01:38.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:38 smithi016 bash[77892]: cluster 2023-10-09T09:01:38.249457+0000 mon.a (mon.0) 973 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T09:01:38.664 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:38.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:38 smithi094 bash[59677]: cluster 2023-10-09T09:01:38.249457+0000 mon.a (mon.0) 973 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T09:01:38.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:38.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:38.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:38.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 1535 1519 94.9323 94.9375 0.00153404 0.00877747 2023-10-09T09:01:38.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:38.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:38.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:39.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:39.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:39.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:39.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:39.220 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:39.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:39.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:39.353 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:39.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:39.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:39.470 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:39 smithi094 bash[59677]: cluster 2023-10-09T09:01:37.782850+0000 mgr.y (mgr.25421) 722 : cluster 0 pgmap v730: 89 pgs: 7 peering, 82 active+clean; 582 KiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 60 B/s, 0 objects/s recovering 2023-10-09T09:01:39.486 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:39 smithi016 bash[75179]: cluster 2023-10-09T09:01:37.782850+0000 mgr.y (mgr.25421) 722 : cluster 0 pgmap v730: 89 pgs: 7 peering, 82 active+clean; 582 KiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 60 B/s, 0 objects/s recovering 2023-10-09T09:01:39.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:39 smithi016 bash[77892]: cluster 2023-10-09T09:01:37.782850+0000 mgr.y (mgr.25421) 722 : cluster 0 pgmap v730: 89 pgs: 7 peering, 82 active+clean; 582 KiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 60 B/s, 0 objects/s recovering 2023-10-09T09:01:39.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:39.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:39.591 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:39.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:39.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:39.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:39.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:39.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:39.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:39.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 15 3196 3181 99.3991 103.875 0.00696702 0.0100341 2023-10-09T09:01:39.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:39.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:39.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:40.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:40.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:40.118 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:40.144 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:40.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:40.280 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:40.384 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:40.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:40.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:40.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:40.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:40.648 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:40.675 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:40.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:40.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:40.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 5682 5666 118.032 155.312 0.00379412 0.00845435 2023-10-09T09:01:40.916 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:40.941 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:40.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:40.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:41.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:41.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:41.176 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:41.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:41.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:41.332 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:41.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:41.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:41.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:41.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:41.596 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:41.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:41.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:41.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:41.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:41.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:41 smithi016 bash[75179]: cluster 2023-10-09T09:01:39.783501+0000 mgr.y (mgr.25421) 723 : cluster 0 pgmap v731: 89 pgs: 7 peering, 82 active+clean; 5.1 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 576 KiB/s wr, 8 op/s; 60 B/s, 0 objects/s recovering 2023-10-09T09:01:41.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:41 smithi016 bash[77892]: cluster 2023-10-09T09:01:39.783501+0000 mgr.y (mgr.25421) 723 : cluster 0 pgmap v731: 89 pgs: 7 peering, 82 active+clean; 5.1 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 576 KiB/s wr, 8 op/s; 60 B/s, 0 objects/s recovering 2023-10-09T09:01:41.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:41 smithi094 bash[59677]: cluster 2023-10-09T09:01:39.783501+0000 mgr.y (mgr.25421) 723 : cluster 0 pgmap v731: 89 pgs: 7 peering, 82 active+clean; 5.1 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 576 KiB/s wr, 8 op/s; 60 B/s, 0 objects/s recovering 2023-10-09T09:01:41.810 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6] out_osds: [7, 5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:01:41.810 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:01:41.811 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 6] 2023-10-09T09:01:41.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T09:01:41.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:41.859 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:41.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 7969 7953 124.255 142.938 0.00466257 0.00803334 2023-10-09T09:01:41.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:41.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:41.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:42.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:42.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:42.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:42.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:42.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:42.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:42.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:42.380 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:42.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:42.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:42.510 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:42.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:42.614 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:42.623 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:42 smithi094 bash[59677]: cluster 2023-10-09T09:01:41.784238+0000 mgr.y (mgr.25421) 724 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 295 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 37 MiB/s wr, 587 op/s; 69 B/s, 0 objects/s recovering 2023-10-09T09:01:42.624 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:42 smithi094 bash[59677]: cluster 2023-10-09T09:01:42.516807+0000 mon.a (mon.0) 974 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T09:01:42.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:42.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:42.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:42.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:42.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:42.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:42.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:42.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:42.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 9467 9451 118.127 93.625 0.00336785 0.00845325 2023-10-09T09:01:42.896 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:42.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:42.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:43.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:42 smithi016 bash[75179]: cluster 2023-10-09T09:01:41.784238+0000 mgr.y (mgr.25421) 724 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 295 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 37 MiB/s wr, 587 op/s; 69 B/s, 0 objects/s recovering 2023-10-09T09:01:43.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:42 smithi016 bash[75179]: cluster 2023-10-09T09:01:42.516807+0000 mon.a (mon.0) 974 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T09:01:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:42 smithi016 bash[77892]: cluster 2023-10-09T09:01:41.784238+0000 mgr.y (mgr.25421) 724 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 295 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 37 MiB/s wr, 587 op/s; 69 B/s, 0 objects/s recovering 2023-10-09T09:01:43.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:42 smithi016 bash[77892]: cluster 2023-10-09T09:01:42.516807+0000 mon.a (mon.0) 974 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T09:01:43.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:43.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:43.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:43.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:43.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:43.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:43.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:43.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:43.417 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:43.445 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:43.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:43.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:43.572 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:43.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:43.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:43.603 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:01:43.603 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:01:43.603 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:01:43.603 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:01:43.603 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:01:43.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:43.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:43.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:43.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 11543 11527 120.062 129.75 0.0071819 0.00831819 2023-10-09T09:01:43.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:44.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:44.110 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:44.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:44.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:44.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:44.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:44.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:44.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:44.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:44.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:44.669 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:44.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:44.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:44.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 13583 13567 121.123 127.5 0.00436359 0.00824742 2023-10-09T09:01:44.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:44.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:44.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:44.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:01:45.040 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:45.076 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:45.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:45.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:45.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:45.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:45.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:45.310 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:45.339 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:45.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:45.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:45.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:45.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:45.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:45.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:45.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:45.604 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:45 smithi094 bash[59677]: cluster 2023-10-09T09:01:43.784841+0000 mgr.y (mgr.25421) 725 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 423 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 49 MiB/s wr, 791 op/s; 64 B/s, 0 objects/s recovering 2023-10-09T09:01:45.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:45.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:45.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:45.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:45.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:45 smithi016 bash[75179]: cluster 2023-10-09T09:01:43.784841+0000 mgr.y (mgr.25421) 725 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 423 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 49 MiB/s wr, 791 op/s; 64 B/s, 0 objects/s recovering 2023-10-09T09:01:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:45 smithi016 bash[77892]: cluster 2023-10-09T09:01:43.784841+0000 mgr.y (mgr.25421) 725 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 423 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 49 MiB/s wr, 791 op/s; 64 B/s, 0 objects/s recovering 2023-10-09T09:01:45.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:45.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:45.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:45.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 15614 15598 121.848 126.938 0.00838307 0.00819767 2023-10-09T09:01:45.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:45.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:45.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:45.985 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:45.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:01:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:01:45] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.33.4" 2023-10-09T09:01:46.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:46.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:46.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:46.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:46.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:46.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:46.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:46.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:46.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:46.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:46.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:46.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:46.604 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:46.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:46.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:46.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:46.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:46.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 17424 17408 120.878 113.125 0.00304815 0.00826439 2023-10-09T09:01:46.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:46.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:46.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:47.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:47.414 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:47.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:47.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:47.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:47.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:47.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:47.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:47.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:47.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:47.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:47 smithi016 bash[75179]: cluster 2023-10-09T09:01:45.785473+0000 mgr.y (mgr.25421) 726 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 544 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 54 MiB/s wr, 869 op/s; 55 B/s, 0 objects/s recovering 2023-10-09T09:01:47.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:47 smithi016 bash[77892]: cluster 2023-10-09T09:01:45.785473+0000 mgr.y (mgr.25421) 726 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 544 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 54 MiB/s wr, 869 op/s; 55 B/s, 0 objects/s recovering 2023-10-09T09:01:47.795 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:47.805 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:47 smithi094 bash[59677]: cluster 2023-10-09T09:01:45.785473+0000 mgr.y (mgr.25421) 726 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 544 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 54 MiB/s wr, 869 op/s; 55 B/s, 0 objects/s recovering 2023-10-09T09:01:47.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:47.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:47.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 19415 19399 121.233 124.438 0.00759926 0.00824105 2023-10-09T09:01:47.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:47.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:47.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:48.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:48.081 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:48.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:48.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:48.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:48.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:48.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:48.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:48.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:48.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:48.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:48.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:48.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:48.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:48.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:48.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:48.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:48.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:48.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:48.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:48.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:48.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:48.838 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:48.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:48.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:48.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 21264 21248 120.716 115.562 0.00706335 0.0082745 2023-10-09T09:01:48.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:48.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:49.103 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:49.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:49.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:49.230 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:49.258 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:49.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:49.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:49.489 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:49.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:49.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:49.619 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:49.631 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:49 smithi094 bash[59677]: cluster 2023-10-09T09:01:47.786598+0000 mgr.y (mgr.25421) 727 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 54 B/s, 0 objects/s recovering 2023-10-09T09:01:49.631 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:49 smithi094 bash[59677]: audit 2023-10-09T09:01:49.227923+0000 mon.a (mon.0) 975 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:49.631 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:49 smithi094 bash[59677]: audit 2023-10-09T09:01:49.228423+0000 mon.a (mon.0) 976 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T09:01:49.648 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:49.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:49.761 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:01:49.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:49.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:49 smithi016 bash[75179]: cluster 2023-10-09T09:01:47.786598+0000 mgr.y (mgr.25421) 727 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 54 B/s, 0 objects/s recovering 2023-10-09T09:01:49.776 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:49 smithi016 bash[75179]: audit 2023-10-09T09:01:49.227923+0000 mon.a (mon.0) 975 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:49.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:49 smithi016 bash[75179]: audit 2023-10-09T09:01:49.228423+0000 mon.a (mon.0) 976 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T09:01:49.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:49 smithi016 bash[77892]: cluster 2023-10-09T09:01:47.786598+0000 mgr.y (mgr.25421) 727 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 54 B/s, 0 objects/s recovering 2023-10-09T09:01:49.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:49 smithi016 bash[77892]: audit 2023-10-09T09:01:49.227923+0000 mon.a (mon.0) 975 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:01:49.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:49 smithi016 bash[77892]: audit 2023-10-09T09:01:49.228423+0000 mon.a (mon.0) 976 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T09:01:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:49.878 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:49.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 22902 22886 119.187 102.375 0.00221069 0.0083816 2023-10-09T09:01:49.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:49.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:49.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:50.007 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:50.032 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:50.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:50.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:50.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:50.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:50.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:50.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:50.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:50.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:50.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:50.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:50.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:50.650 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:50.674 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:50.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:50.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:50.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:50.802 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:50.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:50 smithi094 bash[59677]: audit 2023-10-09T09:01:49.532622+0000 mon.a (mon.0) 977 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T09:01:50.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:50 smithi094 bash[59677]: cluster 2023-10-09T09:01:49.532785+0000 mon.a (mon.0) 978 : cluster 0 osdmap e178: 8 total, 8 up, 6 in 2023-10-09T09:01:50.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:50 smithi094 bash[59677]: cluster 2023-10-09T09:01:49.536847+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:50 smithi094 bash[59677]: cluster 2023-10-09T09:01:49.538910+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:50 smithi094 bash[59677]: cluster 2023-10-09T09:01:49.541738+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:50 smithi094 bash[59677]: cluster 2023-10-09T09:01:49.787050+0000 mgr.y (mgr.25421) 728 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s; 6 B/s, 0 objects/s recovering 2023-10-09T09:01:50.805 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:50 smithi094 bash[59677]: audit 2023-10-09T09:01:49.807246+0000 mon.a (mon.0) 979 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:50.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:50.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:50.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 24450 24434 117.46 96.75 0.00341953 0.00850661 2023-10-09T09:01:50.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:50.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[75179]: audit 2023-10-09T09:01:49.532622+0000 mon.a (mon.0) 977 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[75179]: cluster 2023-10-09T09:01:49.532785+0000 mon.a (mon.0) 978 : cluster 0 osdmap e178: 8 total, 8 up, 6 in 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[75179]: cluster 2023-10-09T09:01:49.536847+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[75179]: cluster 2023-10-09T09:01:49.538910+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[75179]: cluster 2023-10-09T09:01:49.541738+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[75179]: cluster 2023-10-09T09:01:49.787050+0000 mgr.y (mgr.25421) 728 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s; 6 B/s, 0 objects/s recovering 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[75179]: audit 2023-10-09T09:01:49.807246+0000 mon.a (mon.0) 979 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[77892]: audit 2023-10-09T09:01:49.532622+0000 mon.a (mon.0) 977 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[77892]: cluster 2023-10-09T09:01:49.532785+0000 mon.a (mon.0) 978 : cluster 0 osdmap e178: 8 total, 8 up, 6 in 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[77892]: cluster 2023-10-09T09:01:49.536847+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[77892]: cluster 2023-10-09T09:01:49.538910+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.916 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[77892]: cluster 2023-10-09T09:01:49.541738+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T09:01:50.916 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[77892]: cluster 2023-10-09T09:01:49.787050+0000 mgr.y (mgr.25421) 728 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s; 6 B/s, 0 objects/s recovering 2023-10-09T09:01:50.916 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:50 smithi016 bash[77892]: audit 2023-10-09T09:01:49.807246+0000 mon.a (mon.0) 979 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:01:50.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:50.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:50.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:51.046 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:51.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:51.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:51.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:51.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:51.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:51.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:51.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:51.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:01:51.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:51.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:01:51.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:51.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:51.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:51.577 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:51.602 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:51.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:51.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:51.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:51 smithi094 bash[59677]: cluster 2023-10-09T09:01:50.541974+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:51 smithi094 bash[59677]: cluster 2023-10-09T09:01:50.549919+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:51 smithi094 bash[59677]: cluster 2023-10-09T09:01:50.553347+0000 mon.a (mon.0) 980 : cluster 0 osdmap e179: 8 total, 8 up, 6 in 2023-10-09T09:01:51.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:51 smithi094 bash[59677]: cluster 2023-10-09T09:01:50.553478+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:51 smithi094 bash[59677]: cluster 2023-10-09T09:01:50.556476+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:51 smithi094 bash[59677]: cluster 2023-10-09T09:01:50.651240+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.715 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:51 smithi094 bash[59677]: cluster 2023-10-09T09:01:50.651322+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.728 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:51.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:51.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:51.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:51.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[77892]: cluster 2023-10-09T09:01:50.541974+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[77892]: cluster 2023-10-09T09:01:50.549919+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[77892]: cluster 2023-10-09T09:01:50.553347+0000 mon.a (mon.0) 980 : cluster 0 osdmap e179: 8 total, 8 up, 6 in 2023-10-09T09:01:51.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[77892]: cluster 2023-10-09T09:01:50.553478+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[77892]: cluster 2023-10-09T09:01:50.556476+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[77892]: cluster 2023-10-09T09:01:50.651240+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[77892]: cluster 2023-10-09T09:01:50.651322+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[75179]: cluster 2023-10-09T09:01:50.541974+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[75179]: cluster 2023-10-09T09:01:50.549919+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[75179]: cluster 2023-10-09T09:01:50.553347+0000 mon.a (mon.0) 980 : cluster 0 osdmap e179: 8 total, 8 up, 6 in 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[75179]: cluster 2023-10-09T09:01:50.553478+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[75179]: cluster 2023-10-09T09:01:50.556476+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.845 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[75179]: cluster 2023-10-09T09:01:50.651240+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.846 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:51 smithi016 bash[75179]: cluster 2023-10-09T09:01:50.651322+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T09:01:51.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:51.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:51.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:51.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 25926 25910 115.659 92.25 0.00594673 0.00863351 2023-10-09T09:01:51.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:51.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:51.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:52.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:52.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:52.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:52.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:52.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:52.288 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:01:52.343 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:01:52.377 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:01:52.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:52.413 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:01:52.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:52.447 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:01:52.479 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:01:52.511 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:01:52.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:52.546 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:01:52.553 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.554 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.554 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.554 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.554 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.554 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.554 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.554 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:01:52.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:52.575 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:01:52.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:52.691 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:52.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[75179]: cluster 2023-10-09T09:01:51.787586+0000 mgr.y (mgr.25421) 729 : cluster 0 pgmap v739: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T09:01:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[75179]: audit 2023-10-09T09:01:52.240181+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.16:0/3861832010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:52.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[75179]: audit 2023-10-09T09:01:52.534097+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.16:0/184576366' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:52.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[75179]: audit 2023-10-09T09:01:52.534778+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:52.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[77892]: cluster 2023-10-09T09:01:51.787586+0000 mgr.y (mgr.25421) 729 : cluster 0 pgmap v739: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T09:01:52.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[77892]: audit 2023-10-09T09:01:52.240181+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.16:0/3861832010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:52.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[77892]: audit 2023-10-09T09:01:52.534097+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.16:0/184576366' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:52.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:52 smithi016 bash[77892]: audit 2023-10-09T09:01:52.534778+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:52.795 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:52.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:52 smithi094 bash[59677]: cluster 2023-10-09T09:01:51.787586+0000 mgr.y (mgr.25421) 729 : cluster 0 pgmap v739: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T09:01:52.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:52 smithi094 bash[59677]: audit 2023-10-09T09:01:52.240181+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.16:0/3861832010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:01:52.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:52 smithi094 bash[59677]: audit 2023-10-09T09:01:52.534097+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.16:0/184576366' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:52.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:52 smithi094 bash[59677]: audit 2023-10-09T09:01:52.534778+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:01:52.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:52.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 27160 27144 113.09 77.125 0.00329188 0.00883524 2023-10-09T09:01:52.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:52.953 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:52.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:53.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:53.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:53.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:53.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:53.194 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:53.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:53.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:53.324 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:53.348 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:53.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:53.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:53.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:53.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:53.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:53.578 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:53.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:53.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:53.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:53.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:53.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.543929+0000 mon.a (mon.0) 983 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:01:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.555292+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: audit 2023-10-09T09:01:52.560509+0000 mon.a (mon.0) 984 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:01:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.560915+0000 mon.a (mon.0) 985 : cluster 0 osdmap e180: 8 total, 8 up, 6 in 2023-10-09T09:01:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.565476+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.573203+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.575328+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.577732+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:53 smithi094 bash[59677]: cluster 2023-10-09T09:01:52.580042+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:53.842 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.543929+0000 mon.a (mon.0) 983 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:01:53.842 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.555292+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: audit 2023-10-09T09:01:52.560509+0000 mon.a (mon.0) 984 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.560915+0000 mon.a (mon.0) 985 : cluster 0 osdmap e180: 8 total, 8 up, 6 in 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.565476+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.573203+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.575328+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.577732+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[75179]: cluster 2023-10-09T09:01:52.580042+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.543929+0000 mon.a (mon.0) 983 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:01:53.843 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.555292+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: audit 2023-10-09T09:01:52.560509+0000 mon.a (mon.0) 984 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:01:53.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.560915+0000 mon.a (mon.0) 985 : cluster 0 osdmap e180: 8 total, 8 up, 6 in 2023-10-09T09:01:53.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.565476+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.573203+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.575328+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.577732+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:53 smithi016 bash[77892]: cluster 2023-10-09T09:01:52.580042+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:53.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:53.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:53.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 28809 28793 112.462 103.062 0.00955957 0.00888485 2023-10-09T09:01:53.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:53.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:53.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:54.101 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:54.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:54.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:54.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:54.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:54.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:54.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:54.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:54.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:54.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:54.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:54.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:54.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:54.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:54 smithi016 bash[75179]: cluster 2023-10-09T09:01:53.335006+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:54.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:54 smithi016 bash[75179]: cluster 2023-10-09T09:01:53.788934+0000 mgr.y (mgr.25421) 730 : cluster 0 pgmap v741: 89 pgs: 1 peering, 88 active+clean; 1.5 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:01:54.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:54 smithi016 bash[77892]: cluster 2023-10-09T09:01:53.335006+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:54.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:54 smithi016 bash[77892]: cluster 2023-10-09T09:01:53.788934+0000 mgr.y (mgr.25421) 730 : cluster 0 pgmap v741: 89 pgs: 1 peering, 88 active+clean; 1.5 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:01:54.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:54.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:54.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 30422 30406 111.776 100.812 0.0104255 0.00894031 2023-10-09T09:01:54.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:54.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:54.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:54.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:54.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:54 smithi094 bash[59677]: cluster 2023-10-09T09:01:53.335006+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T09:01:54.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:54 smithi094 bash[59677]: cluster 2023-10-09T09:01:53.788934+0000 mgr.y (mgr.25421) 730 : cluster 0 pgmap v741: 89 pgs: 1 peering, 88 active+clean; 1.5 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:01:55.023 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:55.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:55.272 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:55.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:55.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:55.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:55.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:55.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:55.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.696 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:55.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:55.801 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:55.836 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:55.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 31969 31953 110.938 96.6875 0.0209982 0.0090083 2023-10-09T09:01:55.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:55.972 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:55.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:55.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:55.976 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:01:55.976 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:01:56.047 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:01:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:01:55] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.33.4" 2023-10-09T09:01:56.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:56.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:56.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:56.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:56.238 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:56.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:56.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:56.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:56.367 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:56.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:56.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:56.470 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:56.493 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:56.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:56.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:56.628 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:56.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:56.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:56.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:56.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:56.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:56.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:56.858 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:56.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 33426 33410 109.891 91.0625 0.0056087 0.00909354 2023-10-09T09:01:56.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:56.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:56.988 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:56.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:56 smithi094 bash[59677]: cluster 2023-10-09T09:01:55.789522+0000 mgr.y (mgr.25421) 731 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T09:01:56.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:56 smithi016 bash[75179]: cluster 2023-10-09T09:01:55.789522+0000 mgr.y (mgr.25421) 731 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T09:01:57.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:56 smithi016 bash[77892]: cluster 2023-10-09T09:01:55.789522+0000 mgr.y (mgr.25421) 731 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T09:01:57.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:57.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:57.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:57.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:57.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:57.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:57.362 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:01:57.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:57.430 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:57.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:57.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:57.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:57.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:57.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:57.797 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:57.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:57.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:57.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:01:57.880774+0000 min lat: 0.00137305 max lat: 0.404889 avg lat: 0.00917588 2023-10-09T09:01:57.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:01:57.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 34869 34853 108.906 90.1875 0.00272463 0.00917588 2023-10-09T09:01:57.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:57.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:57.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:57.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:58.061 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:58.089 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:58.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:58.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:58.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:58.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:58.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:58.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:58.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:58.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:58.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:58.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:58.508 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:58.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:58.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:58.639 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:58.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:58.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:58.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:01:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:58.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:01:58.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:58.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 36350 36334 108.127 92.5625 0.0162504 0.00924174 2023-10-09T09:01:58.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:58.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:58.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:58.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:01:58 smithi094 bash[59677]: cluster 2023-10-09T09:01:57.790363+0000 mgr.y (mgr.25421) 732 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T09:01:59.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:59.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:01:58 smithi016 bash[75179]: cluster 2023-10-09T09:01:57.790363+0000 mgr.y (mgr.25421) 732 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T09:01:59.019 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:01:58 smithi016 bash[77892]: cluster 2023-10-09T09:01:57.790363+0000 mgr.y (mgr.25421) 732 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T09:01:59.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:01:59.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:01:59.150 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:59.175 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:01:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:01:59.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:59.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:01:59.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:01:59.408 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:59.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:01:59.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:01:59.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:59.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:59.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:59.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:01:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:01:59.809 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:59.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:59.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:01:59.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 15 37835 37820 107.433 92.875 0.00554288 0.00930079 2023-10-09T09:01:59.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:59.970 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:01:59.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:01:59.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:00.073 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:00.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:00.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:00.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:00.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:00.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:00.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:00.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:00.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:00.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:00.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:00.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:00.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:00.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:00.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:00.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:00.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:00.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:00.736 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:00 smithi094 bash[59677]: cluster 2023-10-09T09:01:59.790814+0000 mgr.y (mgr.25421) 733 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T09:02:00.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:00.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:00.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:00.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:02:00.846 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:00 smithi016 bash[75179]: cluster 2023-10-09T09:01:59.790814+0000 mgr.y (mgr.25421) 733 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T09:02:00.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:00 smithi016 bash[77892]: cluster 2023-10-09T09:01:59.790814+0000 mgr.y (mgr.25421) 733 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T09:02:00.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:00.881 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 39127 39111 106.27 80.6875 0.0146826 0.00940196 2023-10-09T09:02:00.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:00.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:00.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:01.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:01.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:01.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:01.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:01.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:01.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:01.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.419 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:01 smithi016 bash[75179]: debug 2023-10-09T09:02:01.249+0000 7fa046a4d700 -1 mon.a@0(leader).osd e180 definitely_dead 0 2023-10-09T09:02:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:01.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:01.596 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.1. 2023-10-09T09:02:01.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:02:01.671 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:01 smithi016 bash[75179]: audit 2023-10-09T09:02:01.252280+0000 mon.a (mon.0) 986 : audit 1 from='client.? 172.21.15.16:0/2736676106' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T09:02:01.672 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:01 smithi016 bash[75179]: cluster 2023-10-09T09:02:01.252596+0000 mon.a (mon.0) 987 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T09:02:01.672 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:01 smithi016 bash[77892]: audit 2023-10-09T09:02:01.252280+0000 mon.a (mon.0) 986 : audit 1 from='client.? 172.21.15.16:0/2736676106' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T09:02:01.672 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:01 smithi016 bash[77892]: cluster 2023-10-09T09:02:01.252596+0000 mon.a (mon.0) 987 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T09:02:01.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:01.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:01.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:01.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:01.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 39953 39937 103.993 51.625 0.00396552 0.00950131 2023-10-09T09:02:01.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:01.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:01.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:01.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:01 smithi094 bash[59677]: audit 2023-10-09T09:02:01.252280+0000 mon.a (mon.0) 986 : audit 1 from='client.? 172.21.15.16:0/2736676106' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T09:02:01.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:01 smithi094 bash[59677]: cluster 2023-10-09T09:02:01.252596+0000 mon.a (mon.0) 987 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T09:02:02.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:02.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:02.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:02.173 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:02:02.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:02.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:02.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:02.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:02.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:02.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:02.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:02.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:02.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:02.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:02.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:02.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:02.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:02.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:02.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:02.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:02.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:02.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:02.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 39953 39937 99.8327 0 - 0.00950131 2023-10-09T09:02:02.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:02.910 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[77892]: audit 2023-10-09T09:02:01.585984+0000 mon.a (mon.0) 988 : audit 1 from='client.? 172.21.15.16:0/2736676106' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[77892]: cluster 2023-10-09T09:02:01.589978+0000 mon.a (mon.0) 989 : cluster 0 osdmap e181: 8 total, 8 up, 5 in 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[77892]: cluster 2023-10-09T09:02:01.596092+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[77892]: cluster 2023-10-09T09:02:01.675286+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[77892]: cluster 2023-10-09T09:02:01.677432+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[77892]: cluster 2023-10-09T09:02:01.753360+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[77892]: cluster 2023-10-09T09:02:01.791448+0000 mgr.y (mgr.25421) 734 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 2.2 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[75179]: audit 2023-10-09T09:02:01.585984+0000 mon.a (mon.0) 988 : audit 1 from='client.? 172.21.15.16:0/2736676106' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[75179]: cluster 2023-10-09T09:02:01.589978+0000 mon.a (mon.0) 989 : cluster 0 osdmap e181: 8 total, 8 up, 5 in 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[75179]: cluster 2023-10-09T09:02:01.596092+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[75179]: cluster 2023-10-09T09:02:01.675286+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[75179]: cluster 2023-10-09T09:02:01.677432+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[75179]: cluster 2023-10-09T09:02:01.753360+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:02 smithi016 bash[75179]: cluster 2023-10-09T09:02:01.791448+0000 mgr.y (mgr.25421) 734 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 2.2 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:02:02.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:02.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:02.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:02 smithi094 bash[59677]: audit 2023-10-09T09:02:01.585984+0000 mon.a (mon.0) 988 : audit 1 from='client.? 172.21.15.16:0/2736676106' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T09:02:02.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:02 smithi094 bash[59677]: cluster 2023-10-09T09:02:01.589978+0000 mon.a (mon.0) 989 : cluster 0 osdmap e181: 8 total, 8 up, 5 in 2023-10-09T09:02:02.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:02 smithi094 bash[59677]: cluster 2023-10-09T09:02:01.596092+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:02 smithi094 bash[59677]: cluster 2023-10-09T09:02:01.675286+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:02 smithi094 bash[59677]: cluster 2023-10-09T09:02:01.677432+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:02 smithi094 bash[59677]: cluster 2023-10-09T09:02:01.753360+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T09:02:02.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:02 smithi094 bash[59677]: cluster 2023-10-09T09:02:01.791448+0000 mgr.y (mgr.25421) 734 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 2.2 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:02:03.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:03.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:03.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:03.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:03.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:03.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:03.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:03.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:03.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:03.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:03.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:03.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:03.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[75179]: cluster 2023-10-09T09:02:02.603002+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[75179]: cluster 2023-10-09T09:02:02.603199+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[75179]: cluster 2023-10-09T09:02:02.605820+0000 mon.a (mon.0) 990 : cluster 0 osdmap e182: 8 total, 8 up, 5 in 2023-10-09T09:02:03.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[75179]: cluster 2023-10-09T09:02:02.606085+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[75179]: cluster 2023-10-09T09:02:02.608150+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[75179]: cluster 2023-10-09T09:02:02.611604+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[75179]: cluster 2023-10-09T09:02:02.738773+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[77892]: cluster 2023-10-09T09:02:02.603002+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[77892]: cluster 2023-10-09T09:02:02.603199+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[77892]: cluster 2023-10-09T09:02:02.605820+0000 mon.a (mon.0) 990 : cluster 0 osdmap e182: 8 total, 8 up, 5 in 2023-10-09T09:02:03.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[77892]: cluster 2023-10-09T09:02:02.606085+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[77892]: cluster 2023-10-09T09:02:02.608150+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[77892]: cluster 2023-10-09T09:02:02.611604+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:03 smithi016 bash[77892]: cluster 2023-10-09T09:02:02.738773+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:03.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:03.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:03.847 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:03.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 39993 39977 96.0891 1.25 0.00943029 0.00988113 2023-10-09T09:02:03.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:03.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:03.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:03.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:03 smithi094 bash[59677]: cluster 2023-10-09T09:02:02.603002+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:03 smithi094 bash[59677]: cluster 2023-10-09T09:02:02.603199+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:03 smithi094 bash[59677]: cluster 2023-10-09T09:02:02.605820+0000 mon.a (mon.0) 990 : cluster 0 osdmap e182: 8 total, 8 up, 5 in 2023-10-09T09:02:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:03 smithi094 bash[59677]: cluster 2023-10-09T09:02:02.606085+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:03 smithi094 bash[59677]: cluster 2023-10-09T09:02:02.608150+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:03 smithi094 bash[59677]: cluster 2023-10-09T09:02:02.611604+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:03 smithi094 bash[59677]: cluster 2023-10-09T09:02:02.738773+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T09:02:04.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:04.115 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:04.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:04.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:04.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:04.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:04.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:04.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:04.382 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:04.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:04.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:04.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:04.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:04.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:04.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:04.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:04.638 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:04.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:04.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:04.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:04.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:04.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:04.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:03.613147+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:03.615563+0000 mon.a (mon.0) 991 : cluster 0 osdmap e183: 8 total, 8 up, 5 in 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:03.616788+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:03.619138+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:03.623368+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:03.624547+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:03.791984+0000 mgr.y (mgr.25421) 735 : cluster 0 pgmap v749: 89 pgs: 2 unknown, 22 peering, 65 active+clean; 2.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: audit 2023-10-09T09:02:04.227639+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:04.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[75179]: cluster 2023-10-09T09:02:04.525913+0000 mon.a (mon.0) 993 : cluster 0 osdmap e184: 8 total, 8 up, 5 in 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:03.613147+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:03.615563+0000 mon.a (mon.0) 991 : cluster 0 osdmap e183: 8 total, 8 up, 5 in 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:03.616788+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:03.619138+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:03.623368+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:03.624547+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:03.791984+0000 mgr.y (mgr.25421) 735 : cluster 0 pgmap v749: 89 pgs: 2 unknown, 22 peering, 65 active+clean; 2.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: audit 2023-10-09T09:02:04.227639+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:04.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:04 smithi016 bash[77892]: cluster 2023-10-09T09:02:04.525913+0000 mon.a (mon.0) 993 : cluster 0 osdmap e184: 8 total, 8 up, 5 in 2023-10-09T09:02:04.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 39997 39981 92.5395 0.25 0.00488085 0.00988089 2023-10-09T09:02:04.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:04.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:04.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:03.613147+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:03.615563+0000 mon.a (mon.0) 991 : cluster 0 osdmap e183: 8 total, 8 up, 5 in 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:03.616788+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:03.619138+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:03.623368+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:03.624547+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:03.791984+0000 mgr.y (mgr.25421) 735 : cluster 0 pgmap v749: 89 pgs: 2 unknown, 22 peering, 65 active+clean; 2.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: audit 2023-10-09T09:02:04.227639+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:04 smithi094 bash[59677]: cluster 2023-10-09T09:02:04.525913+0000 mon.a (mon.0) 993 : cluster 0 osdmap e184: 8 total, 8 up, 5 in 2023-10-09T09:02:04.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:05.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:05.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:05.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:05.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:05.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:05.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:05.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:05.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:05.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:05.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:05.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:05.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:05.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:05.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:05.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:05.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:05.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 39997 39981 89.2345 0 - 0.00988089 2023-10-09T09:02:05.915 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:05 smithi094 bash[59677]: cluster 2023-10-09T09:02:04.517774+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:05.915 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:05 smithi094 bash[59677]: cluster 2023-10-09T09:02:04.525633+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:05.915 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:05 smithi094 bash[59677]: cluster 2023-10-09T09:02:04.612327+0000 mon.a (mon.0) 994 : cluster 3 Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T09:02:05.915 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:05 smithi094 bash[59677]: audit 2023-10-09T09:02:04.825271+0000 mon.a (mon.0) 995 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:05.915 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:05 smithi094 bash[59677]: cluster 2023-10-09T09:02:05.522778+0000 mon.a (mon.0) 996 : cluster 0 osdmap e185: 8 total, 8 up, 5 in 2023-10-09T09:02:05.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:05.927 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:02:05] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.33.4" 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[75179]: cluster 2023-10-09T09:02:04.517774+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[75179]: cluster 2023-10-09T09:02:04.525633+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[75179]: cluster 2023-10-09T09:02:04.612327+0000 mon.a (mon.0) 994 : cluster 3 Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[75179]: audit 2023-10-09T09:02:04.825271+0000 mon.a (mon.0) 995 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[75179]: cluster 2023-10-09T09:02:05.522778+0000 mon.a (mon.0) 996 : cluster 0 osdmap e185: 8 total, 8 up, 5 in 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[77892]: cluster 2023-10-09T09:02:04.517774+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[77892]: cluster 2023-10-09T09:02:04.525633+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:05.928 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[77892]: cluster 2023-10-09T09:02:04.612327+0000 mon.a (mon.0) 994 : cluster 3 Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T09:02:05.929 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[77892]: audit 2023-10-09T09:02:04.825271+0000 mon.a (mon.0) 995 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:05.929 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:05 smithi016 bash[77892]: cluster 2023-10-09T09:02:05.522778+0000 mon.a (mon.0) 996 : cluster 0 osdmap e185: 8 total, 8 up, 5 in 2023-10-09T09:02:05.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:05.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:05.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:06.047 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:06.072 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:06.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:06.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:06.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:06.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:06.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:06.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:06.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:06.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:06.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:06.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:06.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:06.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[75179]: cluster 2023-10-09T09:02:05.253653+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:06.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[75179]: cluster 2023-10-09T09:02:05.253747+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:06.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[75179]: cluster 2023-10-09T09:02:05.519517+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:06.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[75179]: cluster 2023-10-09T09:02:05.519973+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:06.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[75179]: cluster 2023-10-09T09:02:05.792570+0000 mgr.y (mgr.25421) 736 : cluster 0 pgmap v752: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 unknown, 22 peering, 64 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s; 2530/116799 objects degraded (2.166%); 16 KiB/s, 1 objects/s recovering 2023-10-09T09:02:06.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[77892]: cluster 2023-10-09T09:02:05.253653+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:06.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[77892]: cluster 2023-10-09T09:02:05.253747+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:06.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[77892]: cluster 2023-10-09T09:02:05.519517+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:06.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[77892]: cluster 2023-10-09T09:02:05.519973+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:06.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:06 smithi016 bash[77892]: cluster 2023-10-09T09:02:05.792570+0000 mgr.y (mgr.25421) 736 : cluster 0 pgmap v752: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 unknown, 22 peering, 64 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s; 2530/116799 objects degraded (2.166%); 16 KiB/s, 1 objects/s recovering 2023-10-09T09:02:06.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:06.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:06.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:06.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:06.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 40009 39993 86.1833 0.375 0.00354161 0.0103179 2023-10-09T09:02:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:06.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:06.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:06.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:06 smithi094 bash[59677]: cluster 2023-10-09T09:02:05.253653+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T09:02:06.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:06 smithi094 bash[59677]: cluster 2023-10-09T09:02:05.253747+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T09:02:06.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:06 smithi094 bash[59677]: cluster 2023-10-09T09:02:05.519517+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:06.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:06 smithi094 bash[59677]: cluster 2023-10-09T09:02:05.519973+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:06.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:06 smithi094 bash[59677]: cluster 2023-10-09T09:02:05.792570+0000 mgr.y (mgr.25421) 736 : cluster 0 pgmap v752: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 unknown, 22 peering, 64 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s; 2530/116799 objects degraded (2.166%); 16 KiB/s, 1 objects/s recovering 2023-10-09T09:02:07.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:07.111 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:07.112 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:07.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:07.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:07.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:07.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:07.373 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:07.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:07.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:07.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:07.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:07.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:07.660 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:02:07.661 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:02:07.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:07.692 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:07 smithi094 bash[59677]: cluster 2023-10-09T09:02:06.239613+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:07.692 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:07 smithi094 bash[59677]: cluster 2023-10-09T09:02:06.239724+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:07.692 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:07 smithi094 bash[59677]: cluster 2023-10-09T09:02:06.621142+0000 mon.a (mon.0) 997 : cluster 3 Health check failed: Degraded data redundancy: 2530/116799 objects degraded (2.166%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:02:07.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:07.793 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:07.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 40581 40565 84.5021 35.75 0.0219974 0.0118272 2023-10-09T09:02:07.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:07.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:07 smithi016 bash[75179]: cluster 2023-10-09T09:02:06.239613+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:07.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:07 smithi016 bash[75179]: cluster 2023-10-09T09:02:06.239724+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:07.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:07 smithi016 bash[75179]: cluster 2023-10-09T09:02:06.621142+0000 mon.a (mon.0) 997 : cluster 3 Health check failed: Degraded data redundancy: 2530/116799 objects degraded (2.166%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:02:07.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:07 smithi016 bash[77892]: cluster 2023-10-09T09:02:06.239613+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:07.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:07 smithi016 bash[77892]: cluster 2023-10-09T09:02:06.239724+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T09:02:07.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:07 smithi016 bash[77892]: cluster 2023-10-09T09:02:06.621142+0000 mon.a (mon.0) 997 : cluster 3 Health check failed: Degraded data redundancy: 2530/116799 objects degraded (2.166%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:02:07.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:07.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:07.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:08.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:08.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:08.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:08.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:08.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:08.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:08.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:08.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:08.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:08.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:08.455 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:08.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:08.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:08.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:08.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:08.584 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6] out_osds: [7, 5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:02:08.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:02:08.619 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:08.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:08.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:08.722 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:08.732 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:08 smithi094 bash[59677]: cluster 2023-10-09T09:02:07.794397+0000 mgr.y (mgr.25421) 737 : cluster 0 pgmap v753: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 17 MiB/s wr, 270 op/s; 15126/120102 objects degraded (12.594%); 4275/120102 objects misplaced (3.559%); 7.3 MiB/s, 115 objects/s recovering 2023-10-09T09:02:08.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:08.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:08.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:08.869 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:08 smithi016 bash[75179]: cluster 2023-10-09T09:02:07.794397+0000 mgr.y (mgr.25421) 737 : cluster 0 pgmap v753: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 17 MiB/s wr, 270 op/s; 15126/120102 objects degraded (12.594%); 4275/120102 objects misplaced (3.559%); 7.3 MiB/s, 115 objects/s recovering 2023-10-09T09:02:08.869 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:08 smithi016 bash[77892]: cluster 2023-10-09T09:02:07.794397+0000 mgr.y (mgr.25421) 737 : cluster 0 pgmap v753: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 17 MiB/s wr, 270 op/s; 15126/120102 objects degraded (12.594%); 4275/120102 objects misplaced (3.559%); 7.3 MiB/s, 115 objects/s recovering 2023-10-09T09:02:08.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 41609 41593 83.8486 64.25 0.0115041 0.0119194 2023-10-09T09:02:08.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:08.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:08.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:08.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:02:09.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:09.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:09.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:09.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:09.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:09.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:09.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:09.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:09.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:09.428 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:09.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:09.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:09.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:09.653 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:09 smithi094 bash[59677]: cluster 2023-10-09T09:02:08.628862+0000 mon.a (mon.0) 998 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T09:02:09.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:09.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:09.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:09.807 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:09 smithi016 bash[75179]: cluster 2023-10-09T09:02:08.628862+0000 mon.a (mon.0) 998 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T09:02:09.808 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:09 smithi016 bash[77892]: cluster 2023-10-09T09:02:08.628862+0000 mon.a (mon.0) 998 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T09:02:09.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:09.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 42580 42564 83.1246 60.6875 0.019283 0.0120175 2023-10-09T09:02:09.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:09.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:09.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:09.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:10.057 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:10.083 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:10.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:10.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:10.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:10.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:10.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:10.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:10.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:10.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:10.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:10.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:10.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:10.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:10.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:10.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:10.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:10.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:10.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:10.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:10.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:10 smithi016 bash[75179]: cluster 2023-10-09T09:02:09.794906+0000 mgr.y (mgr.25421) 738 : cluster 0 pgmap v754: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 14 MiB/s wr, 226 op/s; 15126/120102 objects degraded (12.594%); 4275/120102 objects misplaced (3.559%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:02:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:10 smithi016 bash[77892]: cluster 2023-10-09T09:02:09.794906+0000 mgr.y (mgr.25421) 738 : cluster 0 pgmap v754: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 14 MiB/s wr, 226 op/s; 15126/120102 objects degraded (12.594%); 4275/120102 objects misplaced (3.559%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:02:10.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:10.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:10.884 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 43508 43492 82.3586 58 0.00601404 0.0121343 2023-10-09T09:02:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:10.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:10.988 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:10.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:10 smithi094 bash[59677]: cluster 2023-10-09T09:02:09.794906+0000 mgr.y (mgr.25421) 738 : cluster 0 pgmap v754: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 14 MiB/s wr, 226 op/s; 15126/120102 objects degraded (12.594%); 4275/120102 objects misplaced (3.559%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:02:11.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:11.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:11.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:11.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:11.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:11.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:11.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:11.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:11.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:11.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:11.884 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 15 44551 44536 81.8553 65.25 0.0273955 0.0122065 2023-10-09T09:02:11.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:11 smithi016 bash[75179]: cluster 2023-10-09T09:02:11.846920+0000 mon.a (mon.0) 999 : cluster 3 Health check update: Degraded data redundancy: 15126/123699 objects degraded (12.228%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:11.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:11 smithi016 bash[75179]: audit 2023-10-09T09:02:11.888369+0000 mon.a (mon.0) 1000 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:11.974 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:11 smithi016 bash[77892]: cluster 2023-10-09T09:02:11.846920+0000 mon.a (mon.0) 999 : cluster 3 Health check update: Degraded data redundancy: 15126/123699 objects degraded (12.228%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:11.974 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:11 smithi016 bash[77892]: audit 2023-10-09T09:02:11.888369+0000 mon.a (mon.0) 1000 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:11.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:11.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:12.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:12.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:12.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:12.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:11 smithi094 bash[59677]: cluster 2023-10-09T09:02:11.846920+0000 mon.a (mon.0) 999 : cluster 3 Health check update: Degraded data redundancy: 15126/123699 objects degraded (12.228%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:12.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:11 smithi094 bash[59677]: audit 2023-10-09T09:02:11.888369+0000 mon.a (mon.0) 1000 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:12.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:12.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:12.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:12.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:12.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:12.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:12.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:12.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:12.553 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:12.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:12.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:12.658 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:12.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:12.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:12.789 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:12.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:12.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:12.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:12.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 45510 45494 81.2271 59.875 0.0115146 0.0123042 2023-10-09T09:02:12.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:12.955 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:12.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:12.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:13.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:13.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:13.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:13.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:13.210 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:12 smithi016 bash[75179]: cluster 2023-10-09T09:02:11.795629+0000 mgr.y (mgr.25421) 739 : cluster 0 pgmap v755: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 20 MiB/s wr, 324 op/s; 15126/123699 objects degraded (12.228%); 3908/123699 objects misplaced (3.159%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T09:02:13.211 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:12 smithi016 bash[77892]: cluster 2023-10-09T09:02:11.795629+0000 mgr.y (mgr.25421) 739 : cluster 0 pgmap v755: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 20 MiB/s wr, 324 op/s; 15126/123699 objects degraded (12.228%); 3908/123699 objects misplaced (3.159%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T09:02:13.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:13.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:13.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:12 smithi094 bash[59677]: cluster 2023-10-09T09:02:11.795629+0000 mgr.y (mgr.25421) 739 : cluster 0 pgmap v755: 89 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 20 MiB/s wr, 324 op/s; 15126/123699 objects degraded (12.228%); 3908/123699 objects misplaced (3.159%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T09:02:13.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:13.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:13.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:13.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:13.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:13.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:13.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:13.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:13.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:13.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:13.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:13.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:13.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:13.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:13.826 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:02:13.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:13.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 46572 46556 80.8144 66.375 0.0138875 0.0123673 2023-10-09T09:02:13.899 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:13.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:14.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:14.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:14.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:14.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:14.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:14.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:14.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:14.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:14.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:14.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:14.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:14.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:14.564 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:14.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:14.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:14.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:14.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:14.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:14.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:14.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 47505 47489 80.206 58.3125 0.00898363 0.0124591 2023-10-09T09:02:14.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:14.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:14.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:14.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:15.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:15.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:15.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:15.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:15.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:15.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:15.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:15.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:15.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:15.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:15.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:15.424 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:15.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:15.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:15.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:15.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:15.625 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:15 smithi016 bash[75179]: cluster 2023-10-09T09:02:13.796532+0000 mgr.y (mgr.25421) 740 : cluster 0 pgmap v756: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 38 MiB/s wr, 605 op/s; 14628/132738 objects degraded (11.020%); 4227/132738 objects misplaced (3.184%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:02:15.625 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:15 smithi016 bash[75179]: audit 2023-10-09T09:02:14.842065+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:15.625 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:15 smithi016 bash[77892]: cluster 2023-10-09T09:02:13.796532+0000 mgr.y (mgr.25421) 740 : cluster 0 pgmap v756: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 38 MiB/s wr, 605 op/s; 14628/132738 objects degraded (11.020%); 4227/132738 objects misplaced (3.184%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:02:15.625 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:15 smithi016 bash[77892]: audit 2023-10-09T09:02:14.842065+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:15.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:15.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:15.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:15.788 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:15.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:15 smithi094 bash[59677]: cluster 2023-10-09T09:02:13.796532+0000 mgr.y (mgr.25421) 740 : cluster 0 pgmap v756: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 38 MiB/s wr, 605 op/s; 14628/132738 objects degraded (11.020%); 4227/132738 objects misplaced (3.184%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:02:15.792 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:15 smithi094 bash[59677]: audit 2023-10-09T09:02:14.842065+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:15.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:15.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 48513 48497 79.7531 63 0.0207867 0.0125313 2023-10-09T09:02:15.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:15.918 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:15.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:02:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:02:15] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.33.4" 2023-10-09T09:02:16.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:16.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:16.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:16.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:16.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:16.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:16.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:16.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:16.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:16.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:16.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:16.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:16.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:16.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:16.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:16.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:16.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:16.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:16.834 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:16.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:16.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:16.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:16.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 49539 49523 79.3522 64.125 0.0150247 0.0125958 2023-10-09T09:02:16.971 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.001 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:17.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:17.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:17.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:17.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:17.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:17.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:17.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:17.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:17.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:17.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:17.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:17.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:17.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:17.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:17.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:17.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:17 smithi094 bash[59677]: cluster 2023-10-09T09:02:15.797360+0000 mgr.y (mgr.25421) 741 : cluster 0 pgmap v757: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 42 MiB/s wr, 677 op/s; 14628/137676 objects degraded (10.625%); 4137/137676 objects misplaced (3.005%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:02:17.766 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:17.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:17.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:02:17.884687+0000 min lat: 0.00137305 max lat: 5.82908 avg lat: 0.0126682 2023-10-09T09:02:17.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:02:17.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 50517 50501 78.8964 61.125 0.0249759 0.0126682 2023-10-09T09:02:17.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:17.918 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:17 smithi016 bash[75179]: cluster 2023-10-09T09:02:15.797360+0000 mgr.y (mgr.25421) 741 : cluster 0 pgmap v757: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 42 MiB/s wr, 677 op/s; 14628/137676 objects degraded (10.625%); 4137/137676 objects misplaced (3.005%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:02:17.919 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:17 smithi016 bash[77892]: cluster 2023-10-09T09:02:15.797360+0000 mgr.y (mgr.25421) 741 : cluster 0 pgmap v757: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 42 MiB/s wr, 677 op/s; 14628/137676 objects degraded (10.625%); 4137/137676 objects misplaced (3.005%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:02:17.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:17.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:18.037 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:18.082 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:18.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:18.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:18.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:18.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:18.325 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:18.350 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:18.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:18.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:18.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:18.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:18.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:18.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:18.600 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:18.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:18.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:18.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:18.730 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:18.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:18.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:18.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:18.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 51252 51236 78.0925 45.9375 0.0116519 0.0127981 2023-10-09T09:02:18.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:18.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:18.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:19.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:19.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:19.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:19.152 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:19.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:19.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:19.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:19.389 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:19.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:19.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[75179]: cluster 2023-10-09T09:02:17.798075+0000 mgr.y (mgr.25421) 742 : cluster 0 pgmap v758: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 14628/146079 objects degraded (10.014%); 3853/146079 objects misplaced (2.638%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[75179]: audit 2023-10-09T09:02:18.505003+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[75179]: audit 2023-10-09T09:02:18.514035+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[75179]: cluster 2023-10-09T09:02:18.529066+0000 mon.a (mon.0) 1004 : cluster 3 Health check update: Degraded data redundancy: 14628/146079 objects degraded (10.014%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[75179]: audit 2023-10-09T09:02:19.231942+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[75179]: audit 2023-10-09T09:02:19.237629+0000 mon.a (mon.0) 1006 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[77892]: cluster 2023-10-09T09:02:17.798075+0000 mgr.y (mgr.25421) 742 : cluster 0 pgmap v758: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 14628/146079 objects degraded (10.014%); 3853/146079 objects misplaced (2.638%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T09:02:19.530 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[77892]: audit 2023-10-09T09:02:18.505003+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.531 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[77892]: audit 2023-10-09T09:02:18.514035+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.531 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[77892]: cluster 2023-10-09T09:02:18.529066+0000 mon.a (mon.0) 1004 : cluster 3 Health check update: Degraded data redundancy: 14628/146079 objects degraded (10.014%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:19.531 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[77892]: audit 2023-10-09T09:02:19.231942+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.531 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:19 smithi016 bash[77892]: audit 2023-10-09T09:02:19.237629+0000 mon.a (mon.0) 1006 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:19.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:19.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:19.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:19 smithi094 bash[59677]: cluster 2023-10-09T09:02:17.798075+0000 mgr.y (mgr.25421) 742 : cluster 0 pgmap v758: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 14628/146079 objects degraded (10.014%); 3853/146079 objects misplaced (2.638%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T09:02:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:19 smithi094 bash[59677]: audit 2023-10-09T09:02:18.505003+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:19 smithi094 bash[59677]: audit 2023-10-09T09:02:18.514035+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:19 smithi094 bash[59677]: cluster 2023-10-09T09:02:18.529066+0000 mon.a (mon.0) 1004 : cluster 3 Health check update: Degraded data redundancy: 14628/146079 objects degraded (10.014%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:19 smithi094 bash[59677]: audit 2023-10-09T09:02:19.231942+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:19.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:19 smithi094 bash[59677]: audit 2023-10-09T09:02:19.237629+0000 mon.a (mon.0) 1006 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:19.685 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:19.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.838 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:19.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:19.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 52128 52112 77.5366 54.75 0.0140585 0.0128899 2023-10-09T09:02:19.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:19.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:19.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:20.071 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:20.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:20.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:20.128 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:02:20.128 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:02:20.211 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:20.242 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:20.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:20.345 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:20.371 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:20.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:20.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:20.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:20.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:20.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:20.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:20.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:20.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:20.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:20.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:20.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:20.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:20.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 53083 53067 77.1214 59.6875 0.0148524 0.0129586 2023-10-09T09:02:20.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:20.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:20.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:20.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:21.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:21.076 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:02:21.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:21.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:21.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:21.125 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:02:21.181 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:02:21.196 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:21.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:21.221 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:02:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:21.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:21.260 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:02:21.309 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:02:21.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:21.349 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:02:21.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:21.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:21.399 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:02:21.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.408 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.408 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.408 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.408 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.408 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:02:21.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:02:21.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:21.546 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:21.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:21.552 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:02:21.603 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[75179]: cluster 2023-10-09T09:02:19.798542+0000 mgr.y (mgr.25421) 743 : cluster 0 pgmap v759: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 14628/146079 objects degraded (10.014%); 3853/146079 objects misplaced (2.638%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:02:21.604 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[75179]: audit 2023-10-09T09:02:21.018257+0000 mon.a (mon.0) 1007 : audit 0 from='client.? 172.21.15.16:0/1401501368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:02:21.604 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[75179]: audit 2023-10-09T09:02:21.391003+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.16:0/2039437443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:02:21.604 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[75179]: audit 2023-10-09T09:02:21.391687+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:02:21.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[77892]: cluster 2023-10-09T09:02:19.798542+0000 mgr.y (mgr.25421) 743 : cluster 0 pgmap v759: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 14628/146079 objects degraded (10.014%); 3853/146079 objects misplaced (2.638%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:02:21.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[77892]: audit 2023-10-09T09:02:21.018257+0000 mon.a (mon.0) 1007 : audit 0 from='client.? 172.21.15.16:0/1401501368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:02:21.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[77892]: audit 2023-10-09T09:02:21.391003+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.16:0/2039437443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:02:21.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:21 smithi016 bash[77892]: audit 2023-10-09T09:02:21.391687+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:02:21.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:21.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:21.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:21.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:21.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:21.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:21.888 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 53845 53829 76.4509 47.625 0.011944 0.0130719 2023-10-09T09:02:21.918 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:21.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:21 smithi094 bash[59677]: cluster 2023-10-09T09:02:19.798542+0000 mgr.y (mgr.25421) 743 : cluster 0 pgmap v759: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 14628/146079 objects degraded (10.014%); 3853/146079 objects misplaced (2.638%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:02:21.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:21 smithi094 bash[59677]: audit 2023-10-09T09:02:21.018257+0000 mon.a (mon.0) 1007 : audit 0 from='client.? 172.21.15.16:0/1401501368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:02:21.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:21 smithi094 bash[59677]: audit 2023-10-09T09:02:21.391003+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.16:0/2039437443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:02:21.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:21 smithi094 bash[59677]: audit 2023-10-09T09:02:21.391687+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:02:21.944 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:21.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:21.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:22.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:22.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:22.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:22.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:22.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:22.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:22.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:22.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:22.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:22.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:22.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:22.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:22.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:22.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:22.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:22.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[75179]: cluster 2023-10-09T09:02:21.522103+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:02:22.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[75179]: cluster 2023-10-09T09:02:21.530610+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T09:02:22.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[75179]: audit 2023-10-09T09:02:21.538208+0000 mon.a (mon.0) 1010 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:02:22.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[75179]: cluster 2023-10-09T09:02:21.538508+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e186: 8 total, 8 up, 5 in 2023-10-09T09:02:22.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[75179]: cluster 2023-10-09T09:02:21.799544+0000 mgr.y (mgr.25421) 744 : cluster 0 pgmap v761: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s; 14628/154830 objects degraded (9.448%); 3360/154830 objects misplaced (2.170%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:02:22.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[75179]: audit 2023-10-09T09:02:22.175642+0000 mon.a (mon.0) 1012 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:22.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[75179]: audit 2023-10-09T09:02:22.185408+0000 mon.a (mon.0) 1013 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:22.598 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:22.608 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:22 smithi094 bash[59677]: cluster 2023-10-09T09:02:21.522103+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:02:22.608 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:22 smithi094 bash[59677]: cluster 2023-10-09T09:02:21.530610+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T09:02:22.608 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:22 smithi094 bash[59677]: audit 2023-10-09T09:02:21.538208+0000 mon.a (mon.0) 1010 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:02:22.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:22 smithi094 bash[59677]: cluster 2023-10-09T09:02:21.538508+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e186: 8 total, 8 up, 5 in 2023-10-09T09:02:22.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:22 smithi094 bash[59677]: cluster 2023-10-09T09:02:21.799544+0000 mgr.y (mgr.25421) 744 : cluster 0 pgmap v761: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s; 14628/154830 objects degraded (9.448%); 3360/154830 objects misplaced (2.170%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:02:22.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:22 smithi094 bash[59677]: audit 2023-10-09T09:02:22.175642+0000 mon.a (mon.0) 1012 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:22.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:22 smithi094 bash[59677]: audit 2023-10-09T09:02:22.185408+0000 mon.a (mon.0) 1013 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:22.624 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:22.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:22.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:22.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[77892]: cluster 2023-10-09T09:02:21.522103+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:02:22.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[77892]: cluster 2023-10-09T09:02:21.530610+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T09:02:22.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[77892]: audit 2023-10-09T09:02:21.538208+0000 mon.a (mon.0) 1010 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:02:22.736 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[77892]: cluster 2023-10-09T09:02:21.538508+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e186: 8 total, 8 up, 5 in 2023-10-09T09:02:22.736 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[77892]: cluster 2023-10-09T09:02:21.799544+0000 mgr.y (mgr.25421) 744 : cluster 0 pgmap v761: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s; 14628/154830 objects degraded (9.448%); 3360/154830 objects misplaced (2.170%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:02:22.736 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[77892]: audit 2023-10-09T09:02:22.175642+0000 mon.a (mon.0) 1012 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:22.736 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:22 smithi016 bash[77892]: audit 2023-10-09T09:02:22.185408+0000 mon.a (mon.0) 1013 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:22.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:22.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:22.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:22.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:22.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 54628 54612 75.8394 48.9375 0.128536 0.0131699 2023-10-09T09:02:22.890 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:22.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:22.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:23.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:23.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:23.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:23.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:23.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:23.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:23.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:23.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:23.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:23.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:23.661 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:23.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:23.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:23.815 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:23 smithi016 bash[75179]: audit 2023-10-09T09:02:22.811431+0000 mon.a (mon.0) 1014 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:02:23.815 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:23 smithi016 bash[75179]: audit 2023-10-09T09:02:22.812222+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:02:23.815 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:23 smithi016 bash[75179]: audit 2023-10-09T09:02:22.817439+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:23.815 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:23 smithi016 bash[77892]: audit 2023-10-09T09:02:22.811431+0000 mon.a (mon.0) 1014 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:02:23.815 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:23 smithi016 bash[77892]: audit 2023-10-09T09:02:22.812222+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:02:23.815 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:23 smithi016 bash[77892]: audit 2023-10-09T09:02:22.817439+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:23.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:23.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:23.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 55402 55386 75.2423 48.375 0.00768398 0.013284 2023-10-09T09:02:23.946 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:23.955 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:23 smithi094 bash[59677]: audit 2023-10-09T09:02:22.811431+0000 mon.a (mon.0) 1014 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:02:23.955 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:23 smithi094 bash[59677]: audit 2023-10-09T09:02:22.812222+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:02:23.955 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:23 smithi094 bash[59677]: audit 2023-10-09T09:02:22.817439+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:02:23.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:23.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:23.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:24.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:24.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:24.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:24.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:24.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:24.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:24.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:24.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:24.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:24.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:24.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:24.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:24.523 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:24.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:24.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:24.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:24 smithi016 bash[75179]: cluster 2023-10-09T09:02:23.800279+0000 mgr.y (mgr.25421) 745 : cluster 0 pgmap v762: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 56 MiB/s wr, 892 op/s; 14628/159531 objects degraded (9.169%); 3360/159531 objects misplaced (2.106%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T09:02:24.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:24 smithi016 bash[75179]: cluster 2023-10-09T09:02:24.519203+0000 mon.a (mon.0) 1017 : cluster 3 Health check update: Degraded data redundancy: 14628/154830 objects degraded (9.448%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:24.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:24 smithi016 bash[77892]: cluster 2023-10-09T09:02:23.800279+0000 mgr.y (mgr.25421) 745 : cluster 0 pgmap v762: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 56 MiB/s wr, 892 op/s; 14628/159531 objects degraded (9.169%); 3360/159531 objects misplaced (2.106%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T09:02:24.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:24 smithi016 bash[77892]: cluster 2023-10-09T09:02:24.519203+0000 mon.a (mon.0) 1017 : cluster 3 Health check update: Degraded data redundancy: 14628/154830 objects degraded (9.448%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:24.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:24.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:24.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:24.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:24.765 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:24 smithi094 bash[59677]: cluster 2023-10-09T09:02:23.800279+0000 mgr.y (mgr.25421) 745 : cluster 0 pgmap v762: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 56 MiB/s wr, 892 op/s; 14628/159531 objects degraded (9.169%); 3360/159531 objects misplaced (2.106%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T09:02:24.765 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:24 smithi094 bash[59677]: cluster 2023-10-09T09:02:24.519203+0000 mon.a (mon.0) 1017 : cluster 3 Health check update: Degraded data redundancy: 14628/154830 objects degraded (9.448%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:24.780 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:24.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:24.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:24.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:24.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 56384 56368 74.9471 61.375 0.0251707 0.0133362 2023-10-09T09:02:24.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:24.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:24.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:25.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:25.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:25.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:25.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:25.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:25.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:25.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:25.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:25.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:25.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:25.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:25.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:25.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:25.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:25.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:25.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:25.857 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:25.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:25.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 15 57286 57271 74.5614 56.4375 0.0106503 0.0134049 2023-10-09T09:02:25.924 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:02:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:02:25] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.33.4" 2023-10-09T09:02:25.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:25.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:25.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:26.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:26.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:26.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:26.232 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:26.257 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:02:26.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:26.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:26.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:26.362 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:26.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:26.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:26.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:26.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:26.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:26.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:26.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:26.662 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:26.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:26.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:26.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:26.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:26.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:26.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:26.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 58268 58252 74.2909 61.3125 0.0179585 0.0134538 2023-10-09T09:02:26.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:26.906 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:26 smithi094 bash[59677]: cluster 2023-10-09T09:02:25.800923+0000 mgr.y (mgr.25421) 746 : cluster 0 pgmap v763: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 57 MiB/s wr, 915 op/s; 14628/165159 objects degraded (8.857%); 3193/165159 objects misplaced (1.933%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:02:26.928 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:26.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:27.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:27.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:26 smithi016 bash[75179]: cluster 2023-10-09T09:02:25.800923+0000 mgr.y (mgr.25421) 746 : cluster 0 pgmap v763: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 57 MiB/s wr, 915 op/s; 14628/165159 objects degraded (8.857%); 3193/165159 objects misplaced (1.933%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:02:27.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:26 smithi016 bash[77892]: cluster 2023-10-09T09:02:25.800923+0000 mgr.y (mgr.25421) 746 : cluster 0 pgmap v763: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 57 MiB/s wr, 915 op/s; 14628/165159 objects degraded (8.857%); 3193/165159 objects misplaced (1.933%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:02:27.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:27.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:27.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:27.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:27.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:27.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:27.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:27.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:27.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:27.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:27.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:27.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:27.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:27.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:27.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:27.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:02:27.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:27.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:27.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:27.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:27.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:27.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 59189 59173 73.9563 57.5625 0.0231572 0.0135129 2023-10-09T09:02:27.907 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:27.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:27.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:27.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:02:28.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:28.036 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:28.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:28.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:28.166 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:28.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:28.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:28.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:28.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:28.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:28.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:28.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:28.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:28.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:28.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:28.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:28.800 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:28.827 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:28.877 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:02:28.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 60042 60026 73.5514 53.3125 0.0246623 0.0135888 2023-10-09T09:02:28.934 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:28.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:28 smithi094 bash[59677]: cluster 2023-10-09T09:02:27.808614+0000 mgr.y (mgr.25421) 747 : cluster 0 pgmap v764: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 55 MiB/s wr, 881 op/s; 14628/172566 objects degraded (8.477%); 2868/172566 objects misplaced (1.662%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T09:02:28.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:28 smithi094 bash[59677]: audit 2023-10-09T09:02:28.572741+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.16:0/60315820' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:02:28.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:28 smithi094 bash[59677]: audit 2023-10-09T09:02:28.573906+0000 mon.a (mon.0) 1018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:02:28.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:28.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:29.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:29.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:29.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:29.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:28 smithi016 bash[75179]: cluster 2023-10-09T09:02:27.808614+0000 mgr.y (mgr.25421) 747 : cluster 0 pgmap v764: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 55 MiB/s wr, 881 op/s; 14628/172566 objects degraded (8.477%); 2868/172566 objects misplaced (1.662%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T09:02:29.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:28 smithi016 bash[75179]: audit 2023-10-09T09:02:28.572741+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.16:0/60315820' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:02:29.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:28 smithi016 bash[75179]: audit 2023-10-09T09:02:28.573906+0000 mon.a (mon.0) 1018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:02:29.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:28 smithi016 bash[77892]: cluster 2023-10-09T09:02:27.808614+0000 mgr.y (mgr.25421) 747 : cluster 0 pgmap v764: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 55 MiB/s wr, 881 op/s; 14628/172566 objects degraded (8.477%); 2868/172566 objects misplaced (1.662%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T09:02:29.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:28 smithi016 bash[77892]: audit 2023-10-09T09:02:28.572741+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.16:0/60315820' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:02:29.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:28 smithi016 bash[77892]: audit 2023-10-09T09:02:28.573906+0000 mon.a (mon.0) 1018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:02:29.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:29.249 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:29.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:29.357 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:29.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:29.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:29.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:29.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:29.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:29.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:29.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:29.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:29.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:29.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:29.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:29.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:29.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:29.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 60966 60950 73.2475 57.75 0.0277746 0.0136451 2023-10-09T09:02:29.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:29.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:29.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:30.056 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:30.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:30.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:30.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:30.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:29 smithi094 bash[59677]: cluster 2023-10-09T09:02:28.866552+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T09:02:30.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:29 smithi094 bash[59677]: cluster 2023-10-09T09:02:28.867259+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T09:02:30.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:29 smithi094 bash[59677]: audit 2023-10-09T09:02:28.869791+0000 mon.a (mon.0) 1019 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:02:30.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:29 smithi094 bash[59677]: cluster 2023-10-09T09:02:28.869867+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e187: 8 total, 8 up, 5 in 2023-10-09T09:02:30.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:29 smithi094 bash[59677]: cluster 2023-10-09T09:02:29.520303+0000 mon.a (mon.0) 1021 : cluster 3 Health check update: Degraded data redundancy: 14628/172566 objects degraded (8.477%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:30.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:30.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[75179]: cluster 2023-10-09T09:02:28.866552+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T09:02:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[75179]: cluster 2023-10-09T09:02:28.867259+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T09:02:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[75179]: audit 2023-10-09T09:02:28.869791+0000 mon.a (mon.0) 1019 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:02:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[75179]: cluster 2023-10-09T09:02:28.869867+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e187: 8 total, 8 up, 5 in 2023-10-09T09:02:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[75179]: cluster 2023-10-09T09:02:29.520303+0000 mon.a (mon.0) 1021 : cluster 3 Health check update: Degraded data redundancy: 14628/172566 objects degraded (8.477%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:30.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[77892]: cluster 2023-10-09T09:02:28.866552+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T09:02:30.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[77892]: cluster 2023-10-09T09:02:28.867259+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T09:02:30.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[77892]: audit 2023-10-09T09:02:28.869791+0000 mon.a (mon.0) 1019 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:02:30.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[77892]: cluster 2023-10-09T09:02:28.869867+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e187: 8 total, 8 up, 5 in 2023-10-09T09:02:30.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:29 smithi016 bash[77892]: cluster 2023-10-09T09:02:29.520303+0000 mon.a (mon.0) 1021 : cluster 3 Health check update: Degraded data redundancy: 14628/172566 objects degraded (8.477%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:30.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:30.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:02:30.347 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:30.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:30.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:30.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:30.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:30.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:30.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:30.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:30.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:30.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:30.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:30.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:30.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:30.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:30.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:30.886 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 61884 61868 72.9479 57.375 0.0178582 0.0137019 2023-10-09T09:02:30.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:30.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:30.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:31.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:31.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:31.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:31.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:31.193 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:31.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:31.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:31.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:31.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:31.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:31.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:31.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:31.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:31.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:31.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:31.592 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:31 smithi016 bash[75179]: cluster 2023-10-09T09:02:29.809177+0000 mgr.y (mgr.25421) 748 : cluster 0 pgmap v766: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s; 14628/172566 objects degraded (8.477%); 2868/172566 objects misplaced (1.662%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T09:02:31.593 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:31 smithi016 bash[77892]: cluster 2023-10-09T09:02:29.809177+0000 mgr.y (mgr.25421) 748 : cluster 0 pgmap v766: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s; 14628/172566 objects degraded (8.477%); 2868/172566 objects misplaced (1.662%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T09:02:31.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:31.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:31.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:31.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:31.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:31.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:31.856 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:31 smithi094 bash[59677]: cluster 2023-10-09T09:02:29.809177+0000 mgr.y (mgr.25421) 748 : cluster 0 pgmap v766: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s; 14628/172566 objects degraded (8.477%); 2868/172566 objects misplaced (1.662%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T09:02:31.888 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 62799 62783 72.654 57.1875 0.0150744 0.0137549 2023-10-09T09:02:31.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:31.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:32.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:32.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:32.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:32.134 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:02:32.134 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T09:02:32.134 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T09:02:32.134 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:02:32.134 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.134 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.44690.0:116224 6.d 6:be6a79b8:::benchmark_data_smithi016_105291_object15097:head [delete] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:52:39.068938+0000", 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 593.06102185099996, 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.102569588, 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.44690", 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2821329149", 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 116224 2023-10-09T09:02:32.135 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.068938+0000", 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.136 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.137 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.068938+0000", 2023-10-09T09:02:32.137 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:02:32.137 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.137 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.137 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.068936+0000", 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0449999999999997e-06 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.068942+0000", 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7849999999999997e-06 2023-10-09T09:02:32.138 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.068951+0000", 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6029999999999998e-06 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.068956+0000", 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.8167999999999999e-05 2023-10-09T09:02:32.139 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.068984+0000", 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018835799999999999 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.069172+0000", 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.6519000000000002e-05 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.140 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.069269+0000", 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0041160210000000001 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.073385+0000", 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0022244700000000001 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.141 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.075609+0000", 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.095766156000000005 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.171375+0000", 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9120000000000002e-05 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.142 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.171445+0000", 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.2644999999999994e-05 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:52:39.171507+0000", 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.102569588 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.143 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:66900 7.0 7:0ce9f13f:::benchmark_data_smithi016_128518_object66899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:56:52.637607+0000", 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 339.49235226000002, 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.17648983800000001, 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 66900 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.144 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637607+0000", 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637607+0000", 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.145 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637606+0000", 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.8046e-05 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637654+0000", 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4820999999999999e-05 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.146 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637668+0000", 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6489999999999996e-06 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637674+0000", 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.1902000000000001e-05 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.147 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637706+0000", 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000113435 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637819+0000", 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000116319 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.637936+0000", 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020563460000000001 2023-10-09T09:02:32.148 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.639992+0000", 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000398312 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.640390+0000", 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.173600436 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.149 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.813991+0000", 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8968999999999996e-05 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.814060+0000", 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.7240999999999998e-05 2023-10-09T09:02:32.150 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:56:52.814097+0000", 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.17648983800000001 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:103039 7.0 7:0c1e609b:::benchmark_data_smithi016_128518_object2521:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T09:02:32.151 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:57:36.732391+0000", 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 295.397568498, 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.153877707, 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.152 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 103039 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732391+0000", 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.153 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732391+0000", 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732389+0000", 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0129000000000001e-05 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.154 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732399+0000", 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0557999999999999e-05 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732420+0000", 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0125999999999999e-05 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732430+0000", 2023-10-09T09:02:32.155 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4845999999999994e-05 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732515+0000", 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00021034800000000001 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732725+0000", 2023-10-09T09:02:32.156 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016084000000000001 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.732886+0000", 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0036426409999999998 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.736528+0000", 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0031918010000000002 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.157 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.739720+0000", 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14632896400000001 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.886049+0000", 2023-10-09T09:02:32.158 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000129355 2023-10-09T09:02:32.159 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.159 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.159 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.159 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.886179+0000", 2023-10-09T09:02:32.159 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.0198000000000006e-05 2023-10-09T09:02:32.159 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.159 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.886269+0000", 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.153877707 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:103062 7.0 7:03469984:::benchmark_data_smithi016_128518_object2544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:57:36.754899+0000", 2023-10-09T09:02:32.160 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 295.37506058399998, 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.131787084, 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 103062 2023-10-09T09:02:32.161 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.754899+0000", 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.162 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.163 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.754899+0000", 2023-10-09T09:02:32.163 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:02:32.163 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.163 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.163 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.163 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.754897+0000", 2023-10-09T09:02:32.163 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1869999999999997e-06 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.754905+0000", 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4374e-05 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.754920+0000", 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.764e-06 2023-10-09T09:02:32.164 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.754926+0000", 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0436000000000002e-05 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.754987+0000", 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00011799299999999999 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.165 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.755105+0000", 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000113165 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.755218+0000", 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003192697 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.758411+0000", 2023-10-09T09:02:32.166 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0241999999999996e-05 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.758481+0000", 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.12793658699999999 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.886418+0000", 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000171094 2023-10-09T09:02:32.167 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.886589+0000", 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7316000000000001e-05 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.886686+0000", 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.131787084 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.168 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:103070 7.0 7:04973025:::benchmark_data_smithi016_128518_object2552:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:57:36.770395+0000", 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 295.359564299, 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.116782074, 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.169 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 103070 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770395+0000", 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.170 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770395+0000", 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770393+0000", 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.3500000000000003e-06 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770402+0000", 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6648000000000001e-05 2023-10-09T09:02:32.171 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.172 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.172 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.172 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770419+0000", 2023-10-09T09:02:32.172 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.2869999999999999e-06 2023-10-09T09:02:32.172 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.172 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.172 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.173 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770426+0000", 2023-10-09T09:02:32.173 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.2680000000000001e-05 2023-10-09T09:02:32.173 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.173 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.173 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.173 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770509+0000", 2023-10-09T09:02:32.173 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017845799999999999 2023-10-09T09:02:32.174 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.174 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.174 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.174 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770688+0000", 2023-10-09T09:02:32.174 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019382700000000001 2023-10-09T09:02:32.174 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.175 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.175 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.175 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.770881+0000", 2023-10-09T09:02:32.175 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0041652829999999997 2023-10-09T09:02:32.175 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.175 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.175 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.176 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.775047+0000", 2023-10-09T09:02:32.176 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000102753 2023-10-09T09:02:32.176 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.176 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.176 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.176 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.775149+0000", 2023-10-09T09:02:32.176 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.11168547 2023-10-09T09:02:32.177 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.177 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.177 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.177 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.886835+0000", 2023-10-09T09:02:32.177 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00022096 2023-10-09T09:02:32.177 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.177 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.178 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.178 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.887056+0000", 2023-10-09T09:02:32.178 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012143500000000001 2023-10-09T09:02:32.178 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.178 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.178 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:36.887177+0000", 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.116782074 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.179 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:110768 7.0 7:0c8dff71:::benchmark_data_smithi016_128518_object10250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e144)", 2023-10-09T09:02:32.180 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:57:47.381921+0000", 2023-10-09T09:02:32.180 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 284.74803874999998, 2023-10-09T09:02:32.180 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.185219575, 2023-10-09T09:02:32.180 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.180 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.180 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.181 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.181 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.181 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 110768 2023-10-09T09:02:32.181 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.181 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.181 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.181 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.182 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.381921+0000", 2023-10-09T09:02:32.182 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.182 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.182 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.182 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.182 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.381921+0000", 2023-10-09T09:02:32.183 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:02:32.183 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.183 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.183 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.183 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.381919+0000", 2023-10-09T09:02:32.183 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.7960000000000001e-06 2023-10-09T09:02:32.183 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.184 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.184 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.184 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.381925+0000", 2023-10-09T09:02:32.184 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0835999999999999e-05 2023-10-09T09:02:32.184 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.184 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.184 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.185 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.381936+0000", 2023-10-09T09:02:32.185 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5170000000000002e-06 2023-10-09T09:02:32.185 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.185 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.185 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.185 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.381943+0000", 2023-10-09T09:02:32.185 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.702e-05 2023-10-09T09:02:32.186 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.186 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.186 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.186 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.381990+0000", 2023-10-09T09:02:32.186 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000139667 2023-10-09T09:02:32.186 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.186 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.187 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.187 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.382130+0000", 2023-10-09T09:02:32.187 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000132247 2023-10-09T09:02:32.187 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.187 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.187 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.187 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.382262+0000", 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0039027689999999999 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.386165+0000", 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0038940289999999998 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.390059+0000", 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.176947365 2023-10-09T09:02:32.188 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.567007+0000", 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.2725000000000005e-05 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.567089+0000", 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1100999999999997e-05 2023-10-09T09:02:32.189 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:47.567140+0000", 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.185219575 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:112402 7.0 7:04c4bd31:::benchmark_data_smithi016_128518_object11884:head [delete] snapc 0=[] ondisk+write+known_if_redirected e144)", 2023-10-09T09:02:32.190 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:57:49.833180+0000", 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 282.29677950000001, 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.107826537, 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 112402 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.191 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833180+0000", 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833180+0000", 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.192 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.193 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.193 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833178+0000", 2023-10-09T09:02:32.193 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.3140000000000004e-06 2023-10-09T09:02:32.193 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.193 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.193 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.193 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833186+0000", 2023-10-09T09:02:32.194 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4375e-05 2023-10-09T09:02:32.194 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.194 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.194 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.194 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833201+0000", 2023-10-09T09:02:32.194 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0729999999999999e-06 2023-10-09T09:02:32.194 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833208+0000", 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.2002999999999997e-05 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833250+0000", 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000115383 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.195 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.196 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.196 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833365+0000", 2023-10-09T09:02:32.196 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016195200000000001 2023-10-09T09:02:32.196 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.196 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.196 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.196 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.833527+0000", 2023-10-09T09:02:32.197 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0076462539999999999 2023-10-09T09:02:32.197 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.197 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.197 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.197 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.841173+0000", 2023-10-09T09:02:32.197 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018332299999999999 2023-10-09T09:02:32.197 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.198 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.198 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.198 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.841357+0000", 2023-10-09T09:02:32.198 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.099497428999999998 2023-10-09T09:02:32.198 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.198 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.198 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.199 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.940854+0000", 2023-10-09T09:02:32.199 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.5622000000000001e-05 2023-10-09T09:02:32.199 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.199 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.199 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.199 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.940940+0000", 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.6792999999999996e-05 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.941007+0000", 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.107826537 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.200 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:112407 7.0 7:0f38b078:::benchmark_data_smithi016_128518_object11889:head [delete] snapc 0=[] ondisk+write+known_if_redirected e144)", 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:57:49.837548+0000", 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 282.29241161900001, 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.110799289, 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.201 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 112407 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837548+0000", 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.202 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837548+0000", 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837546+0000", 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3670000000000002e-06 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837553+0000", 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3521e-05 2023-10-09T09:02:32.203 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837566+0000", 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.263e-06 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837573+0000", 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1873999999999998e-05 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.204 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837624+0000", 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014752400000000001 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837772+0000", 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017099899999999999 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.205 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.837943+0000", 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033521739999999999 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.841295+0000", 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0040988229999999997 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.206 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.845394+0000", 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.102774462 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.948168+0000", 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00011732500000000001 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.207 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.948286+0000", 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.1493000000000002e-05 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:57:49.948347+0000", 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.110799289 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.208 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:137578 7.0 7:0a28b6e5:::benchmark_data_smithi016_128518_object37060:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:58:30.903436+0000", 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 241.22652320700001, 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.105571318, 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 137578 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.209 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903436+0000", 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903436+0000", 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.210 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903434+0000", 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8029999999999997e-06 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903442+0000", 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5996999999999999e-05 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.211 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903458+0000", 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3929999999999996e-06 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903464+0000", 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5373e-05 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903520+0000", 2023-10-09T09:02:32.212 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000181029 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903701+0000", 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017811899999999999 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.903879+0000", 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.013736194 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.213 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.917615+0000", 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00032403400000000001 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:30.917939+0000", 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.090933723999999994 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.214 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:31.008873+0000", 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3591999999999996e-05 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:31.008946+0000", 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.1245000000000001e-05 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.215 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:31.009008+0000", 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.105571318 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:144534 7.0 7:05ef0852:::benchmark_data_smithi016_128518_object44016:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:58:43.173205+0000", 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 228.95675404299999, 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.13452626100000001, 2023-10-09T09:02:32.216 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 144534 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173205+0000", 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.217 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173205+0000", 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999971 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173202+0000", 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5378999999999999e-05 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.218 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173228+0000", 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9646000000000001e-05 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173247+0000", 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.9680000000000001e-06 2023-10-09T09:02:32.219 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173257+0000", 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8609999999999995e-05 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173326+0000", 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017434800000000001 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.220 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173500+0000", 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000182506 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.173683+0000", 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0044919419999999996 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.221 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.178175+0000", 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000113968 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.178289+0000", 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.12934960300000001 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.307638+0000", 2023-10-09T09:02:32.222 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2559999999999998e-05 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.307691+0000", 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.0815000000000001e-05 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:58:43.307732+0000", 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.13452626100000001 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.223 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:172009 7.0 7:08981066:::benchmark_data_smithi016_128518_object71491:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:59:35.711458+0000", 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 176.41850192699999, 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.11114241, 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.224 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 172009 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711458+0000", 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711458+0000", 2023-10-09T09:02:32.225 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999967 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711454+0000", 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7269999999999999e-05 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711472+0000", 2023-10-09T09:02:32.226 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6117000000000001e-05 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711498+0000", 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1420000000000001e-05 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711509+0000", 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3063000000000003e-05 2023-10-09T09:02:32.227 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711572+0000", 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000118498 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711691+0000", 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012131200000000001 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.228 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.711812+0000", 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028884280000000002 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.714700+0000", 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.012256454 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.229 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.726957+0000", 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.095319740999999999 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.822277+0000", 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00020911399999999999 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.230 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.822486+0000", 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000114276 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:35.822600+0000", 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.11114241 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.231 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:177369 7.0 7:08872293:::benchmark_data_smithi016_128518_object76851:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:59:45.132308+0000", 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 166.99765108700001, 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.244597228, 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.232 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 177369 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132308+0000", 2023-10-09T09:02:32.233 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132308+0000", 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999967 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.234 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132305+0000", 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7557000000000001e-05 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132333+0000", 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9385000000000001e-05 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.235 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132392+0000", 2023-10-09T09:02:32.236 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5248e-05 2023-10-09T09:02:32.236 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.236 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.236 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.236 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132407+0000", 2023-10-09T09:02:32.236 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012917699999999999 2023-10-09T09:02:32.236 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.237 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.237 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.237 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132537+0000", 2023-10-09T09:02:32.237 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018752999999999999 2023-10-09T09:02:32.237 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.237 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.237 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.238 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132724+0000", 2023-10-09T09:02:32.238 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00023032200000000001 2023-10-09T09:02:32.238 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.238 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.238 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.238 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.132954+0000", 2023-10-09T09:02:32.239 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0051024249999999998 2023-10-09T09:02:32.239 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.239 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.239 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.239 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.138057+0000", 2023-10-09T09:02:32.240 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002449185 2023-10-09T09:02:32.240 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.240 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.240 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.240 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.140506+0000", 2023-10-09T09:02:32.240 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.23624030100000001 2023-10-09T09:02:32.240 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.241 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.241 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.241 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.376746+0000", 2023-10-09T09:02:32.241 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.6176999999999998e-05 2023-10-09T09:02:32.241 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.241 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.376843+0000", 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3081000000000001e-05 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.376906+0000", 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.244597228 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.242 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35034.0:177381 7.0 7:03319fae:::benchmark_data_smithi016_128518_object76863:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T08:59:45.152863+0000", 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 166.97709696999999, 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.24956768900000001, 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.243 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35034", 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 177381 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.152863+0000", 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.244 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.152863+0000", 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.152861+0000", 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.2799999999999992e-06 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.245 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.152870+0000", 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7187999999999999e-05 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.152887+0000", 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4574e-05 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.152902+0000", 2023-10-09T09:02:32.246 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.7404000000000001e-05 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.152959+0000", 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017268700000000001 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.153132+0000", 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018993699999999999 2023-10-09T09:02:32.247 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.153322+0000", 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0040661359999999997 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.157388+0000", 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.8787999999999999e-05 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.248 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.157417+0000", 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.24489302900000001 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.402310+0000", 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5228e-05 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:02:32.249 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.402365+0000", 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5282000000000003e-05 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T08:59:45.402430+0000", 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.24956768900000001 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.250 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.34125.0:46 2.2 2.7c339972 (undecoded) ondisk+read+known_if_redirected+??? e168)", 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:01:49.712885+0000", 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 42.417074862, 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00027139599999999997, 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.34125", 2023-10-09T09:02:32.251 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2408130123", 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 46 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:01:49.712885+0000", 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.252 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:01:49.712885+0000", 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999971 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:01:49.712882+0000", 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4239e-05 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.253 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:01:49.712896+0000", 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7775000000000001e-05 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:01:49.712924+0000", 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3186999999999999e-05 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.254 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:01:49.712937+0000", 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00021926899999999999 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:01:49.713156+0000", 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00027139599999999997 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.255 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:32.256 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:32.256 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:02:32.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:32.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:32.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:32.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:32.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:32.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:32.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:32.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:32.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:32.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:32.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:32.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:32.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:32.680 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:32 smithi094 bash[59677]: cluster 2023-10-09T09:02:31.809951+0000 mgr.y (mgr.25421) 749 : cluster 0 pgmap v767: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s; 14628/181002 objects degraded (8.082%); 2321/181002 objects misplaced (1.282%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:32.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:32.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:32.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:32.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:32.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:32 smithi016 bash[75179]: cluster 2023-10-09T09:02:31.809951+0000 mgr.y (mgr.25421) 749 : cluster 0 pgmap v767: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s; 14628/181002 objects degraded (8.082%); 2321/181002 objects misplaced (1.282%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:32.819 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:32 smithi016 bash[77892]: cluster 2023-10-09T09:02:31.809951+0000 mgr.y (mgr.25421) 749 : cluster 0 pgmap v767: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s; 14628/181002 objects degraded (8.082%); 2321/181002 objects misplaced (1.282%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:32.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:32.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:32.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:32.888 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 63699 63683 72.3557 56.25 0.0152491 0.0138132 2023-10-09T09:02:32.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:32.968 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:32.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:32.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:33.073 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:33.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:33.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:33.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:33.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:33.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:33.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:02:33.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:33.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:33.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:33.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:33.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:33.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:33.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:33.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:33.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:33.662 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:33.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:33.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:33.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:33.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:33.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:33.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:33.883 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:33.884 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:33.888 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 64597 64581 72.0657 56.125 0.0104085 0.0138676 2023-10-09T09:02:33.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:33.921 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:33.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:34.024 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:34.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:34.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:34.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:34.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.319 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:34.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:34.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:34.557 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:34.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:34.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:34.687 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:34.717 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:34.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:34.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:34.888 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 65513 65497 71.8057 57.25 0.0243727 0.0139195 2023-10-09T09:02:34.906 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:34.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.961 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:34 smithi094 bash[59677]: cluster 2023-10-09T09:02:33.810635+0000 mgr.y (mgr.25421) 750 : cluster 0 pgmap v768: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 55 MiB/s wr, 885 op/s; 14628/186129 objects degraded (7.859%); 2321/186129 objects misplaced (1.247%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:34.961 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:34 smithi094 bash[59677]: audit 2023-10-09T09:02:34.228194+0000 mon.a (mon.0) 1022 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:34.961 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:34 smithi094 bash[59677]: cluster 2023-10-09T09:02:34.521810+0000 mon.a (mon.0) 1023 : cluster 3 Health check update: Degraded data redundancy: 14628/186129 objects degraded (7.859%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:34.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:34.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:35.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:35.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:35.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:34 smithi016 bash[75179]: cluster 2023-10-09T09:02:33.810635+0000 mgr.y (mgr.25421) 750 : cluster 0 pgmap v768: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 55 MiB/s wr, 885 op/s; 14628/186129 objects degraded (7.859%); 2321/186129 objects misplaced (1.247%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:35.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:34 smithi016 bash[75179]: audit 2023-10-09T09:02:34.228194+0000 mon.a (mon.0) 1022 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:35.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:34 smithi016 bash[75179]: cluster 2023-10-09T09:02:34.521810+0000 mon.a (mon.0) 1023 : cluster 3 Health check update: Degraded data redundancy: 14628/186129 objects degraded (7.859%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:35.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:34 smithi016 bash[77892]: cluster 2023-10-09T09:02:33.810635+0000 mgr.y (mgr.25421) 750 : cluster 0 pgmap v768: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 55 MiB/s wr, 885 op/s; 14628/186129 objects degraded (7.859%); 2321/186129 objects misplaced (1.247%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:35.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:34 smithi016 bash[77892]: audit 2023-10-09T09:02:34.228194+0000 mon.a (mon.0) 1022 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:35.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:34 smithi016 bash[77892]: cluster 2023-10-09T09:02:34.521810+0000 mon.a (mon.0) 1023 : cluster 3 Health check update: Degraded data redundancy: 14628/186129 objects degraded (7.859%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:35.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:35.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:35.357 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:35.381 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:35.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:35.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:35.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:35.510 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:35.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:35.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:35.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:35.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:35.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:35.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:35.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.890 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 66439 66423 71.5644 57.875 0.0213518 0.0139661 2023-10-09T09:02:35.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:35.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:35.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:02:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:02:35] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.33.4" 2023-10-09T09:02:36.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:36.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:36.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:36.153 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:36.196 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:36.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:36.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:36.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:36.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:36.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:36.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:36.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:36.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:36.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:36.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:36.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:36.829 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:36.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:36.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:36.889 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 67289 67273 71.2518 53.125 0.0184519 0.0140278 2023-10-09T09:02:36.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:36.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:36.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:36.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:37.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:37.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:36 smithi094 bash[59677]: cluster 2023-10-09T09:02:35.811419+0000 mgr.y (mgr.25421) 751 : cluster 0 pgmap v769: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 56 MiB/s wr, 899 op/s; 14628/192165 objects degraded (7.612%); 2154/192165 objects misplaced (1.121%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:37.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:37.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:37.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:37.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:36 smithi016 bash[75179]: cluster 2023-10-09T09:02:35.811419+0000 mgr.y (mgr.25421) 751 : cluster 0 pgmap v769: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 56 MiB/s wr, 899 op/s; 14628/192165 objects degraded (7.612%); 2154/192165 objects misplaced (1.121%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:37.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:36 smithi016 bash[77892]: cluster 2023-10-09T09:02:35.811419+0000 mgr.y (mgr.25421) 751 : cluster 0 pgmap v769: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 56 MiB/s wr, 899 op/s; 14628/192165 objects degraded (7.612%); 2154/192165 objects misplaced (1.121%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T09:02:37.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:37.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:37.363 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:37.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:37.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:37.494 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:37.533 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:37.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:37.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:37.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:37.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:37.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:37.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:37.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:37.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:37.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:37.889 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:02:37.888943+0000 min lat: 0.00137305 max lat: 5.82908 avg lat: 0.0140743 2023-10-09T09:02:37.890 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:02:37.890 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 68197 68181 71.01 56.75 0.0101 0.0140743 2023-10-09T09:02:37.896 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:37.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:37.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:38.027 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:38.051 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:38.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:38.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:38.169 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:02:38.191 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:38.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:38.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:38.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:38.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:38.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:38.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:38.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:38.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:38.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:38.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:38.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:38.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:38.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:38.890 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 68484 68468 70.14 17.9375 0.00527578 0.0140579 2023-10-09T09:02:38.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:38.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:38.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:38.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:39.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.109 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:39.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:39.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:39.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.812308+0000 mgr.y (mgr.25421) 752 : cluster 0 pgmap v770: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 57 MiB/s wr, 912 op/s; 14628/199953 objects degraded (7.316%); 1742/199953 objects misplaced (0.871%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T09:02:39.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.884640+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e188: 8 total, 8 up, 5 in 2023-10-09T09:02:39.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.885732+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.887628+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.889142+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.889558+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.891120+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.891138+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[77892]: cluster 2023-10-09T09:02:37.892641+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.812308+0000 mgr.y (mgr.25421) 752 : cluster 0 pgmap v770: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 57 MiB/s wr, 912 op/s; 14628/199953 objects degraded (7.316%); 1742/199953 objects misplaced (0.871%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.884640+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e188: 8 total, 8 up, 5 in 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.885732+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.887628+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.889142+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.889558+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.891120+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.891138+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:38 smithi016 bash[75179]: cluster 2023-10-09T09:02:37.892641+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.812308+0000 mgr.y (mgr.25421) 752 : cluster 0 pgmap v770: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 57 MiB/s wr, 912 op/s; 14628/199953 objects degraded (7.316%); 1742/199953 objects misplaced (0.871%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T09:02:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.884640+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e188: 8 total, 8 up, 5 in 2023-10-09T09:02:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.885732+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.887628+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.889142+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.889558+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.891120+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.891138+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:38 smithi094 bash[59677]: cluster 2023-10-09T09:02:37.892641+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T09:02:39.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:39.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:39.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:39.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:39.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:39.486 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.527 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:39.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:39.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:39.672 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:39.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:39.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.803 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:39.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:39.890 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 69289 69273 69.8201 50.3125 0.0298431 0.0143145 2023-10-09T09:02:39.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.934 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:39.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:40.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:40.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:40.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[77892]: cluster 2023-10-09T09:02:38.883534+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[77892]: cluster 2023-10-09T09:02:38.884293+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[77892]: cluster 2023-10-09T09:02:38.887773+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e189: 8 total, 8 up, 5 in 2023-10-09T09:02:40.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[77892]: cluster 2023-10-09T09:02:38.889263+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[77892]: cluster 2023-10-09T09:02:39.523030+0000 mon.a (mon.0) 1026 : cluster 3 Health check update: Degraded data redundancy: 14628/199953 objects degraded (7.316%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:40.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[75179]: cluster 2023-10-09T09:02:38.883534+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[75179]: cluster 2023-10-09T09:02:38.884293+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[75179]: cluster 2023-10-09T09:02:38.887773+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e189: 8 total, 8 up, 5 in 2023-10-09T09:02:40.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[75179]: cluster 2023-10-09T09:02:38.889263+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:39 smithi016 bash[75179]: cluster 2023-10-09T09:02:39.523030+0000 mon.a (mon.0) 1026 : cluster 3 Health check update: Degraded data redundancy: 14628/199953 objects degraded (7.316%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:40.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:40.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:40.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:40.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:40.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:40.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:39 smithi094 bash[59677]: cluster 2023-10-09T09:02:38.883534+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:39 smithi094 bash[59677]: cluster 2023-10-09T09:02:38.884293+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:39 smithi094 bash[59677]: cluster 2023-10-09T09:02:38.887773+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e189: 8 total, 8 up, 5 in 2023-10-09T09:02:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:39 smithi094 bash[59677]: cluster 2023-10-09T09:02:38.889263+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T09:02:40.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:39 smithi094 bash[59677]: cluster 2023-10-09T09:02:39.523030+0000 mon.a (mon.0) 1026 : cluster 3 Health check update: Degraded data redundancy: 14628/199953 objects degraded (7.316%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:40.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:40.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:40.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:40.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:40.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:40.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:40.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:40.581 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:40.616 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:40.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:40.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:40.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:40.753 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:40.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:40.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:40.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 70108 70092 69.5238 51.1875 0.0266161 0.0143782 2023-10-09T09:02:40.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:40.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:40.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:40.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:41.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:41.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:41.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:41.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:41.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:41.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:41.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:41.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:41.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:41.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:41.549 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:41.579 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:41.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:41.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:41.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:41.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:41.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:41 smithi016 bash[75179]: cluster 2023-10-09T09:02:39.812837+0000 mgr.y (mgr.25421) 753 : cluster 0 pgmap v773: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 789 op/s; 14628/199953 objects degraded (7.316%); 1742/199953 objects misplaced (0.871%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T09:02:41.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:41 smithi016 bash[77892]: cluster 2023-10-09T09:02:39.812837+0000 mgr.y (mgr.25421) 753 : cluster 0 pgmap v773: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 789 op/s; 14628/199953 objects degraded (7.316%); 1742/199953 objects misplaced (0.871%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T09:02:41.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:41.825 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:41 smithi094 bash[59677]: cluster 2023-10-09T09:02:39.812837+0000 mgr.y (mgr.25421) 753 : cluster 0 pgmap v773: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 789 op/s; 14628/199953 objects degraded (7.316%); 1742/199953 objects misplaced (0.871%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T09:02:41.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:41.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:41.890 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 70938 70922 69.2479 51.875 0.0310646 0.0144342 2023-10-09T09:02:41.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:41.978 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:41.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:41.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:42.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:42.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:42.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:42.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:42.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:42.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:42.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:42.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:42.377 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:42.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:42.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:42.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:42.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:42.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:42.619 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:42.626 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:42 smithi094 bash[59677]: cluster 2023-10-09T09:02:41.813628+0000 mgr.y (mgr.25421) 754 : cluster 0 pgmap v774: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 54 MiB/s wr, 861 op/s; 12098/206811 objects degraded (5.850%); 3792/206811 objects misplaced (1.834%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T09:02:42.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:42.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:42.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:42.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:42 smithi016 bash[75179]: cluster 2023-10-09T09:02:41.813628+0000 mgr.y (mgr.25421) 754 : cluster 0 pgmap v774: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 54 MiB/s wr, 861 op/s; 12098/206811 objects degraded (5.850%); 3792/206811 objects misplaced (1.834%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T09:02:42.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:42 smithi016 bash[77892]: cluster 2023-10-09T09:02:41.813628+0000 mgr.y (mgr.25421) 754 : cluster 0 pgmap v774: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 54 MiB/s wr, 861 op/s; 12098/206811 objects degraded (5.850%); 3792/206811 objects misplaced (1.834%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T09:02:42.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:42.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:42.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:42.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:42.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 71828 71812 69.0383 55.625 0.0251507 0.0144779 2023-10-09T09:02:42.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:42.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:43.024 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:43.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:43.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:43.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:43.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:43.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:43.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:43.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:43.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:43.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:43.417 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:43.470 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:43.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:43.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:43.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:43.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:43.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:43.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:43.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:43.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:43.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:43.861 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:43.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:43.890 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 72633 72617 68.7545 50.3125 0.0112812 0.0145385 2023-10-09T09:02:43.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:43.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:43.999 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:02:44.000 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:02:44.000 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.000 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45122.0:72709 8.4 e189/176 8:2b8e745b:::benchmark_data_smithi016_151580_object72708:head v 189'4545, mlcod=189'4543)", 2023-10-09T09:02:44.000 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.992344+0000", 2023-10-09T09:02:44.000 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.002474094, 2023-10-09T09:02:44.001 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0025048140000000002, 2023-10-09T09:02:44.001 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.001 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:02:44.001 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.001 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.001 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.002 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992344+0000", 2023-10-09T09:02:44.002 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.002 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.002 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.002 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.002 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992344+0000", 2023-10-09T09:02:44.003 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:02:44.003 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.003 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.003 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.003 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992343+0000", 2023-10-09T09:02:44.003 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.4973e-05 2023-10-09T09:02:44.004 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.004 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.004 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.004 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992368+0000", 2023-10-09T09:02:44.004 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4230000000000002e-06 2023-10-09T09:02:44.005 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.005 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.005 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.005 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992375+0000", 2023-10-09T09:02:44.005 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.2549999999999999e-06 2023-10-09T09:02:44.005 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.006 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.006 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.006 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992379+0000", 2023-10-09T09:02:44.006 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.8610999999999999e-05 2023-10-09T09:02:44.006 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.006 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992428+0000", 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.9592000000000001e-05 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992457+0000", 2023-10-09T09:02:44.007 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000113564 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop_reply(client.45122.0:72706 8.5 e189/172)", 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.992389+0000", 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0024289979999999999, 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0025289850000000001, 2023-10-09T09:02:44.008 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.009 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:02:44.009 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.009 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.009 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.009 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992389+0000", 2023-10-09T09:02:44.009 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.010 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.010 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.010 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.010 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992389+0000", 2023-10-09T09:02:44.010 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:02:44.010 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.011 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.011 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.011 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992388+0000", 2023-10-09T09:02:44.011 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.689e-06 2023-10-09T09:02:44.011 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.011 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.012 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.012 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992391+0000", 2023-10-09T09:02:44.012 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5550000000000002e-06 2023-10-09T09:02:44.012 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.012 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.012 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.013 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992397+0000", 2023-10-09T09:02:44.013 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.5269999999999999e-06 2023-10-09T09:02:44.013 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.013 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.013 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.014 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992400+0000", 2023-10-09T09:02:44.014 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1462e-05 2023-10-09T09:02:44.014 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.014 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.014 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.015 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.015 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.015 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:72701 8.5 8:ae460006:::benchmark_data_smithi016_151580_object72700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e189)", 2023-10-09T09:02:44.015 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.980252+0000", 2023-10-09T09:02:44.015 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.014565221, 2023-10-09T09:02:44.015 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.014726654, 2023-10-09T09:02:44.016 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.016 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:02:44.016 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:44.016 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:02:44.016 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:02:44.016 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 72701 2023-10-09T09:02:44.017 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.017 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.017 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.017 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.017 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980252+0000", 2023-10-09T09:02:44.017 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.018 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.018 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.018 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.018 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980252+0000", 2023-10-09T09:02:44.018 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:02:44.019 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.019 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.019 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.019 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980252+0000", 2023-10-09T09:02:44.019 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9726000000000003e-05 2023-10-09T09:02:44.019 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.020 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.020 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.020 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980321+0000", 2023-10-09T09:02:44.020 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0213e-05 2023-10-09T09:02:44.020 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.020 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.020 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980332+0000", 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9449999999999998e-06 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980338+0000", 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.0080000000000002e-05 2023-10-09T09:02:44.021 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.022 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.022 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:44.022 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980388+0000", 2023-10-09T09:02:44.022 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015399899999999999 2023-10-09T09:02:44.022 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.022 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.022 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:44.023 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980542+0000", 2023-10-09T09:02:44.023 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018593400000000001 2023-10-09T09:02:44.023 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.023 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.023 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:02:44.023 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.980728+0000", 2023-10-09T09:02:44.023 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029482279999999998 2023-10-09T09:02:44.024 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.024 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.024 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:44.024 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.983676+0000", 2023-10-09T09:02:44.024 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00035886099999999998 2023-10-09T09:02:44.024 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.025 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.025 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:44.025 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.984035+0000", 2023-10-09T09:02:44.025 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0083037700000000002 2023-10-09T09:02:44.025 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.025 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.025 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:44.026 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992338+0000", 2023-10-09T09:02:44.026 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.012085912000000001 2023-10-09T09:02:44.026 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.026 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.026 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.026 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.027 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.027 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45122.0:72704 8.0 e189/182 8:071ee2c5:::benchmark_data_smithi016_151580_object72703:head v 189'4470, mlcod=177'345)", 2023-10-09T09:02:44.027 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.983812+0000", 2023-10-09T09:02:44.027 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.011005951, 2023-10-09T09:02:44.027 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.011266343, 2023-10-09T09:02:44.027 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.027 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.983812+0000", 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.983812+0000", 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.028 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.983811+0000", 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4752e-05 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.983866+0000", 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1576000000000001e-05 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.029 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.030 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.030 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.983877+0000", 2023-10-09T09:02:44.030 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.8869999999999997e-06 2023-10-09T09:02:44.030 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.030 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.030 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.030 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.983883+0000", 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0048241400000000002 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.988707+0000", 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5505000000000001e-05 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.988733+0000", 2023-10-09T09:02:44.031 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0049210340000000003 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:72706 8.5 8:abfdefca:::benchmark_data_smithi016_151580_object72705:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e189)", 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.985379+0000", 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0094388779999999995, 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0097746940000000004, 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.032 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 72706 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985379+0000", 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.033 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985379+0000", 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985377+0000", 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9262000000000003e-05 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.034 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985436+0000", 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5014999999999999e-05 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985451+0000", 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.7300000000000005e-06 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.035 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985459+0000", 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1926999999999999e-05 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985511+0000", 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016757600000000001 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985679+0000", 2023-10-09T09:02:44.036 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000225212 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.985904+0000", 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0032821700000000001 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.989186+0000", 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.4482e-05 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.037 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.989221+0000", 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003841728 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45122.0:72703 8.7 e189/182 8:e93d3cbc:::benchmark_data_smithi016_151580_object72702:head v 189'4644, mlcod=178'1446)", 2023-10-09T09:02:44.038 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.987702+0000", 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0071157429999999999, 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0075549670000000001, 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987702+0000", 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.039 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987702+0000", 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987701+0000", 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.9926999999999999e-05 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987741+0000", 2023-10-09T09:02:44.040 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2216e-05 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987753+0000", 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3640000000000004e-06 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987759+0000", 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6765000000000003e-05 2023-10-09T09:02:44.041 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987816+0000", 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5880999999999998e-05 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.987842+0000", 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014005500000000001 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.042 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45122.0:72711 8.f e189/182)", 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.990995+0000", 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0038230429999999999, 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0043821240000000003, 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.043 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.990995+0000", 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.990995+0000", 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.990994+0000", 2023-10-09T09:02:44.044 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6928000000000003e-05 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.991031+0000", 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3193999999999999e-05 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.991044+0000", 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.2919999999999998e-06 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.045 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.991051+0000", 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6332999999999997e-05 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45122.0:72708 8.2 e189/176 8:4c294456:::benchmark_data_smithi016_151580_object72707:head v 189'4620, mlcod=189'4618)", 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:43.992290+0000", 2023-10-09T09:02:44.046 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0025272290000000002, 2023-10-09T09:02:44.047 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0031539900000000002, 2023-10-09T09:02:44.047 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:02:44.047 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:02:44.047 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:02:44.047 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.047 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:02:44.048 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992290+0000", 2023-10-09T09:02:44.048 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:02:44.048 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.048 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.048 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:02:44.048 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992290+0000", 2023-10-09T09:02:44.048 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:02:44.049 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.049 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.049 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:02:44.049 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992290+0000", 2023-10-09T09:02:44.049 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.8436e-05 2023-10-09T09:02:44.049 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.050 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.050 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:02:44.050 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992319+0000", 2023-10-09T09:02:44.050 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.2459999999999997e-06 2023-10-09T09:02:44.050 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.050 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992326+0000", 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.5879999999999999e-06 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992330+0000", 2023-10-09T09:02:44.051 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.1559e-05 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992372+0000", 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.2619000000000001e-05 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:02:44.052 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:02:44.053 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:43.992395+0000", 2023-10-09T09:02:44.053 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000104104 2023-10-09T09:02:44.053 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.053 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:02:44.053 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.053 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:02:44.053 INFO:teuthology.orchestra.run.smithi094.stdout: ], 2023-10-09T09:02:44.054 INFO:teuthology.orchestra.run.smithi094.stdout: "num_ops": 8 2023-10-09T09:02:44.054 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:02:44.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:44.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:44.156 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:44.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:44.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:44.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:44.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:44.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:44.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:44.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:44.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:44.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:44.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:44.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:44.612 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:44.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:44.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:44.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:44.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:44.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:44.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:44.875 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:44 smithi094 bash[59677]: cluster 2023-10-09T09:02:43.814414+0000 mgr.y (mgr.25421) 755 : cluster 0 pgmap v775: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 780 op/s; 12098/210915 objects degraded (5.736%); 3792/210915 objects misplaced (1.798%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T09:02:44.875 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:44 smithi094 bash[59677]: cluster 2023-10-09T09:02:44.524355+0000 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 12098/210915 objects degraded (5.736%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:44.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:44.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:44.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 73471 73455 68.5099 52.375 0.0116989 0.0145874 2023-10-09T09:02:44.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:45.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:45.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:02:45.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:45.133 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:44 smithi016 bash[75179]: cluster 2023-10-09T09:02:43.814414+0000 mgr.y (mgr.25421) 755 : cluster 0 pgmap v775: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 780 op/s; 12098/210915 objects degraded (5.736%); 3792/210915 objects misplaced (1.798%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T09:02:45.133 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:44 smithi016 bash[75179]: cluster 2023-10-09T09:02:44.524355+0000 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 12098/210915 objects degraded (5.736%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:45.133 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:44 smithi016 bash[77892]: cluster 2023-10-09T09:02:43.814414+0000 mgr.y (mgr.25421) 755 : cluster 0 pgmap v775: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 780 op/s; 12098/210915 objects degraded (5.736%); 3792/210915 objects misplaced (1.798%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T09:02:45.133 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:44 smithi016 bash[77892]: cluster 2023-10-09T09:02:44.524355+0000 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 12098/210915 objects degraded (5.736%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:45.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:45.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:45.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:45.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:45.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:45.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:45.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:45.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:45.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:45.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:45.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:45.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:45.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:45.740 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:02:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:02:45] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.33.4" 2023-10-09T09:02:45.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:45.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:45.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 74371 74355 68.3296 56.25 0.0245201 0.0146288 2023-10-09T09:02:45.930 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:45.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:45.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:46.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:46.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:46.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:46.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:46.196 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:46.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:46.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:46.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:46.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:46.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:46.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:46.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:46.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:46.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:46.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:46.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:46.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:46.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:46.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:46.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:46.881 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:46 smithi094 bash[59677]: cluster 2023-10-09T09:02:45.815156+0000 mgr.y (mgr.25421) 756 : cluster 0 pgmap v776: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 684 op/s; 12098/216381 objects degraded (5.591%); 3489/216381 objects misplaced (1.612%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:02:46.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:46 smithi016 bash[75179]: cluster 2023-10-09T09:02:45.815156+0000 mgr.y (mgr.25421) 756 : cluster 0 pgmap v776: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 684 op/s; 12098/216381 objects degraded (5.591%); 3489/216381 objects misplaced (1.612%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:02:46.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:46 smithi016 bash[77892]: cluster 2023-10-09T09:02:45.815156+0000 mgr.y (mgr.25421) 756 : cluster 0 pgmap v776: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 684 op/s; 12098/216381 objects degraded (5.591%); 3489/216381 objects misplaced (1.612%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:02:46.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:46.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 75238 75222 68.1245 54.1875 0.00563718 0.0146682 2023-10-09T09:02:46.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:46.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:46.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:47.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:47.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:47.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:47.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:47.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:47.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:47.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:47.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:47.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:47.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:47.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:47.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:47.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:47.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:47.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:47.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:47.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:47.758 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:47.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:47.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:47.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:47.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:47.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:47.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 76094 76078 67.9141 53.5 0.0812428 0.0147104 2023-10-09T09:02:47.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:48.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:48.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:48.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:48.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:48.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:48.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:48.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.416 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:48.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:48.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:48.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:48.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:48.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:48.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:48.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:48.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:48.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:48.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:48.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 76980 76964 67.7375 55.375 0.0468168 0.0147556 2023-10-09T09:02:48.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:48.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:48.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:49.056 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:49.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:49.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:49.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:49.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:48 smithi016 bash[75179]: cluster 2023-10-09T09:02:47.815850+0000 mgr.y (mgr.25421) 757 : cluster 0 pgmap v777: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 50 MiB/s wr, 794 op/s; 12098/223617 objects degraded (5.410%); 3489/223617 objects misplaced (1.560%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:02:49.216 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:48 smithi016 bash[77892]: cluster 2023-10-09T09:02:47.815850+0000 mgr.y (mgr.25421) 757 : cluster 0 pgmap v777: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 50 MiB/s wr, 794 op/s; 12098/223617 objects degraded (5.410%); 3489/223617 objects misplaced (1.560%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:02:49.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:49.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:49.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:49.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:48 smithi094 bash[59677]: cluster 2023-10-09T09:02:47.815850+0000 mgr.y (mgr.25421) 757 : cluster 0 pgmap v777: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 50 MiB/s wr, 794 op/s; 12098/223617 objects degraded (5.410%); 3489/223617 objects misplaced (1.560%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:02:49.334 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:49.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:49.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:49.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:49.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:49.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:49.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:49.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:49.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:49.698 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:02:49.727 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:02:49.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:49.788 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:02:49.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:49.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:49.850 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:02:49.892 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:02:49.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 77738 77722 67.4546 47.375 0.0322314 0.0148166 2023-10-09T09:02:49.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:49.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:49.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:49.934 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:02:49.942 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:02:49.969 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:02:50.004 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:02:50.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.015 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.015 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.015 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.015 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.015 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:02:50.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:50.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:49 smithi016 bash[75179]: audit 2023-10-09T09:02:49.231049+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:50.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:49 smithi016 bash[75179]: cluster 2023-10-09T09:02:49.527601+0000 mon.a (mon.0) 1029 : cluster 3 Health check update: Degraded data redundancy: 12098/223617 objects degraded (5.410%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:50.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:49 smithi016 bash[75179]: audit 2023-10-09T09:02:49.671911+0000 mon.a (mon.0) 1030 : audit 0 from='client.? 172.21.15.16:0/2246255208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:02:50.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:49 smithi016 bash[77892]: audit 2023-10-09T09:02:49.231049+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:50.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:49 smithi016 bash[77892]: cluster 2023-10-09T09:02:49.527601+0000 mon.a (mon.0) 1029 : cluster 3 Health check update: Degraded data redundancy: 12098/223617 objects degraded (5.410%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:50.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:49 smithi016 bash[77892]: audit 2023-10-09T09:02:49.671911+0000 mon.a (mon.0) 1030 : audit 0 from='client.? 172.21.15.16:0/2246255208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:02:50.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:50.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:50.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:50.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:49 smithi094 bash[59677]: audit 2023-10-09T09:02:49.231049+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:02:50.173 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:49 smithi094 bash[59677]: cluster 2023-10-09T09:02:49.527601+0000 mon.a (mon.0) 1029 : cluster 3 Health check update: Degraded data redundancy: 12098/223617 objects degraded (5.410%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:50.173 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:49 smithi094 bash[59677]: audit 2023-10-09T09:02:49.671911+0000 mon.a (mon.0) 1030 : audit 0 from='client.? 172.21.15.16:0/2246255208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:02:50.190 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:50.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:50.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:50.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:50.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:50.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:50.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:50.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:50.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:50.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:50.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:50.728 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:50.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:50.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:50.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:50.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 78604 78588 67.272 54.125 0.0136111 0.0148573 2023-10-09T09:02:50.899 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:02:50.904 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:02:50.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:50 smithi094 bash[59677]: cluster 2023-10-09T09:02:49.816460+0000 mgr.y (mgr.25421) 758 : cluster 0 pgmap v778: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 12098/223617 objects degraded (5.410%); 3489/223617 objects misplaced (1.560%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:02:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:50 smithi094 bash[59677]: audit 2023-10-09T09:02:49.948489+0000 mon.a (mon.0) 1031 : audit 1 from='client.? 172.21.15.16:0/2648626413' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:02:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:50 smithi094 bash[59677]: audit 2023-10-09T09:02:50.092581+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.16:0/2041992623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:02:50.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:50 smithi094 bash[59677]: audit 2023-10-09T09:02:50.093730+0000 mon.a (mon.0) 1032 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:02:50.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:50.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:51.097 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:51.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:51.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:51.221 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:51.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:51.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[75179]: cluster 2023-10-09T09:02:49.816460+0000 mgr.y (mgr.25421) 758 : cluster 0 pgmap v778: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 12098/223617 objects degraded (5.410%); 3489/223617 objects misplaced (1.560%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:02:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[75179]: audit 2023-10-09T09:02:49.948489+0000 mon.a (mon.0) 1031 : audit 1 from='client.? 172.21.15.16:0/2648626413' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:02:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[75179]: audit 2023-10-09T09:02:50.092581+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.16:0/2041992623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:02:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[75179]: audit 2023-10-09T09:02:50.093730+0000 mon.a (mon.0) 1032 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:02:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[77892]: cluster 2023-10-09T09:02:49.816460+0000 mgr.y (mgr.25421) 758 : cluster 0 pgmap v778: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 12098/223617 objects degraded (5.410%); 3489/223617 objects misplaced (1.560%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:02:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[77892]: audit 2023-10-09T09:02:49.948489+0000 mon.a (mon.0) 1031 : audit 1 from='client.? 172.21.15.16:0/2648626413' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:02:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[77892]: audit 2023-10-09T09:02:50.092581+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.16:0/2041992623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:02:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:50 smithi016 bash[77892]: audit 2023-10-09T09:02:50.093730+0000 mon.a (mon.0) 1032 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:02:51.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:51.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:51.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:51.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:51.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:51.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:51.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:51.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:51.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:51.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:51.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:51.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:51.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:51.753 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:51.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:51.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:51.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:51.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:51.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:51.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 78711 78695 66.4533 6.6875 0.00358141 0.0148612 2023-10-09T09:02:51.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:52.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:52.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:52.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:52.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:52.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:50.884588+0000 mon.a (mon.0) 1033 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:50.887854+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: audit 2023-10-09T09:02:50.890169+0000 mon.a (mon.0) 1034 : audit 1 from='client.? 172.21.15.16:0/2648626413' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: audit 2023-10-09T09:02:50.890426+0000 mon.a (mon.0) 1035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:50.890790+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e190: 8 total, 8 up, 5 in 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:50.893425+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:50.900060+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.181796+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.182917+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.183936+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:51 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.184923+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:52.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:50.884588+0000 mon.a (mon.0) 1033 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:02:52.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:50.887854+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: audit 2023-10-09T09:02:50.890169+0000 mon.a (mon.0) 1034 : audit 1 from='client.? 172.21.15.16:0/2648626413' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: audit 2023-10-09T09:02:50.890426+0000 mon.a (mon.0) 1035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:50.890790+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e190: 8 total, 8 up, 5 in 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:50.893425+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:50.900060+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.181796+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.182917+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.183936+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.184923+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:50.884588+0000 mon.a (mon.0) 1033 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:50.887854+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: audit 2023-10-09T09:02:50.890169+0000 mon.a (mon.0) 1034 : audit 1 from='client.? 172.21.15.16:0/2648626413' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: audit 2023-10-09T09:02:50.890426+0000 mon.a (mon.0) 1035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:50.890790+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e190: 8 total, 8 up, 5 in 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:50.893425+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:50.900060+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.181796+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.182917+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.183936+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.184923+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:52.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:52.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:52.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:52.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:52.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:52.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:52.640 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:52.660 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:52.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:52.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:52.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:52.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 78818 78802 65.6565 6.6875 0.00443059 0.0150096 2023-10-09T09:02:52.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:52.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:52.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:53.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:53.048 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:53.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:53.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:53.161 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.266976+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:53.161 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.817215+0000 mgr.y (mgr.25421) 759 : cluster 0 pgmap v780: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 833 op/s; 12098/231837 objects degraded (5.218%); 3190/231837 objects misplaced (1.376%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:02:53.161 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.904764+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e191: 8 total, 8 up, 5 in 2023-10-09T09:02:53.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.266976+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:53.162 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.817215+0000 mgr.y (mgr.25421) 759 : cluster 0 pgmap v780: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 833 op/s; 12098/231837 objects degraded (5.218%); 3190/231837 objects misplaced (1.376%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:02:53.162 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:52 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.904764+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e191: 8 total, 8 up, 5 in 2023-10-09T09:02:53.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:53.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:53.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:52 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.266976+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T09:02:53.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:52 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.817215+0000 mgr.y (mgr.25421) 759 : cluster 0 pgmap v780: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 833 op/s; 12098/231837 objects degraded (5.218%); 3190/231837 objects misplaced (1.376%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:02:53.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:52 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.904764+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e191: 8 total, 8 up, 5 in 2023-10-09T09:02:53.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:53.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:53.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:53.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:53.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:53.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:53.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:53.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:53.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:53.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:53.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:53.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:53.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:53.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:53.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:53.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:53.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:53.895 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 79294 79278 65.184 29.75 0.0212633 0.015333 2023-10-09T09:02:53.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:54.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:54.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:54.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:54.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.905005+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.906241+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[75179]: cluster 2023-10-09T09:02:51.907904+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[75179]: cluster 2023-10-09T09:02:52.907701+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[75179]: cluster 2023-10-09T09:02:52.910461+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e192: 8 total, 8 up, 5 in 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.905005+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.906241+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[77892]: cluster 2023-10-09T09:02:51.907904+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[77892]: cluster 2023-10-09T09:02:52.907701+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:54.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:53 smithi016 bash[77892]: cluster 2023-10-09T09:02:52.910461+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e192: 8 total, 8 up, 5 in 2023-10-09T09:02:54.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:54.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:53 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.905005+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:53 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.906241+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:53 smithi094 bash[59677]: cluster 2023-10-09T09:02:51.907904+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T09:02:54.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:53 smithi094 bash[59677]: cluster 2023-10-09T09:02:52.907701+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:54.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:53 smithi094 bash[59677]: cluster 2023-10-09T09:02:52.910461+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e192: 8 total, 8 up, 5 in 2023-10-09T09:02:54.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:54.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:54.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:54.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:54.445 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:54.549 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:54.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:54.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:02:54.677 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:54.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:54.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:54.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:54.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:54.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 15 79981 79966 64.8959 43 0.0445015 0.0154021 2023-10-09T09:02:54.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:54.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:54.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:54.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:54.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[75179]: cluster 2023-10-09T09:02:52.924766+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:55.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[75179]: cluster 2023-10-09T09:02:53.720432+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:55.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[75179]: cluster 2023-10-09T09:02:53.817759+0000 mgr.y (mgr.25421) 760 : cluster 0 pgmap v783: 89 pgs: 4 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 40 MiB/s wr, 645 op/s; 7105/235248 objects degraded (3.020%); 8015/235248 objects misplaced (3.407%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:02:55.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[75179]: cluster 2023-10-09T09:02:54.528766+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 7105/235248 objects degraded (3.020%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:55.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[77892]: cluster 2023-10-09T09:02:52.924766+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:55.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[77892]: cluster 2023-10-09T09:02:53.720432+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:55.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[77892]: cluster 2023-10-09T09:02:53.817759+0000 mgr.y (mgr.25421) 760 : cluster 0 pgmap v783: 89 pgs: 4 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 40 MiB/s wr, 645 op/s; 7105/235248 objects degraded (3.020%); 8015/235248 objects misplaced (3.407%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:02:55.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:54 smithi016 bash[77892]: cluster 2023-10-09T09:02:54.528766+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 7105/235248 objects degraded (3.020%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:55.028 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:02:55.029 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:02:55.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:55.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:55.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:55.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:55.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:02:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:54 smithi094 bash[59677]: cluster 2023-10-09T09:02:52.924766+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:54 smithi094 bash[59677]: cluster 2023-10-09T09:02:53.720432+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T09:02:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:54 smithi094 bash[59677]: cluster 2023-10-09T09:02:53.817759+0000 mgr.y (mgr.25421) 760 : cluster 0 pgmap v783: 89 pgs: 4 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 40 MiB/s wr, 645 op/s; 7105/235248 objects degraded (3.020%); 8015/235248 objects misplaced (3.407%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:02:55.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:54 smithi094 bash[59677]: cluster 2023-10-09T09:02:54.528766+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 7105/235248 objects degraded (3.020%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:02:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:55.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:02:55.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T09:02:55.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:55.371 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:55.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:55.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:02:55.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:55.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:55.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:55.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:55.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:55.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:55.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:55.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:55.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:55.760 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:55.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:55.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:55.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:55.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:55.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:55.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:55.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 80591 80575 64.5519 38.0625 0.0183114 0.0154848 2023-10-09T09:02:55.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:56.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:02:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:02:55] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.33.4" 2023-10-09T09:02:56.019 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:56.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:56.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:56.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:56.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:02:56.250 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:56.310 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:56.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:56.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:56.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:56.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:56.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:56.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:56.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:56.825 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:56.853 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:56.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 81256 81240 64.2608 41.5625 0.00779471 0.0155527 2023-10-09T09:02:56.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:56.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:56.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:57.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:57.101 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:56 smithi094 bash[59677]: cluster 2023-10-09T09:02:55.818577+0000 mgr.y (mgr.25421) 761 : cluster 0 pgmap v784: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 50 MiB/s wr, 804 op/s; 21739/238095 objects degraded (9.130%); 4825/238095 objects misplaced (2.027%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:02:57.115 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:57.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:57.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:57.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:57.232 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:56 smithi016 bash[75179]: cluster 2023-10-09T09:02:55.818577+0000 mgr.y (mgr.25421) 761 : cluster 0 pgmap v784: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 50 MiB/s wr, 804 op/s; 21739/238095 objects degraded (9.130%); 4825/238095 objects misplaced (2.027%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:02:57.233 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:56 smithi016 bash[77892]: cluster 2023-10-09T09:02:55.818577+0000 mgr.y (mgr.25421) 761 : cluster 0 pgmap v784: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 50 MiB/s wr, 804 op/s; 21739/238095 objects degraded (9.130%); 4825/238095 objects misplaced (2.027%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:02:57.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:57.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:57.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:57.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:02:57.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:57.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:02:57.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:57.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:02:57.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:57.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:57.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:57.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:57.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:57.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:57.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:57.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:57.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:57.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:57.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:02:57.892918+0000 min lat: 0.00137305 max lat: 5.82908 avg lat: 0.015627 2023-10-09T09:02:57.894 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:02:57.894 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 81898 81882 63.9591 40.125 0.00604214 0.015627 2023-10-09T09:02:57.970 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:57.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:02:58.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:02:58.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:58.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.260 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:58.366 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.394 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:58.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:58.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:58.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:58.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:58.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:58.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.830 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:58.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 82562 82546 63.6818 41.5 0.0168804 0.0156953 2023-10-09T09:02:58.916 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:58 smithi016 bash[75179]: cluster 2023-10-09T09:02:57.819335+0000 mgr.y (mgr.25421) 762 : cluster 0 pgmap v785: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 39 MiB/s wr, 624 op/s; 21739/242352 objects degraded (8.970%); 4323/242352 objects misplaced (1.784%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T09:02:58.916 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:58 smithi016 bash[77892]: cluster 2023-10-09T09:02:57.819335+0000 mgr.y (mgr.25421) 762 : cluster 0 pgmap v785: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 39 MiB/s wr, 624 op/s; 21739/242352 objects degraded (8.970%); 4323/242352 objects misplaced (1.784%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T09:02:58.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:58.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:58.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:58.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:59.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:59.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:59.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:59.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:59.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:59.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:02:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:59.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:02:59.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:58 smithi094 bash[59677]: cluster 2023-10-09T09:02:57.819335+0000 mgr.y (mgr.25421) 762 : cluster 0 pgmap v785: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 39 MiB/s wr, 624 op/s; 21739/242352 objects degraded (8.970%); 4323/242352 objects misplaced (1.784%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T09:02:59.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:59.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:59.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:59.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:59.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:59.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:59.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:59.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:59.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:59.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:02:59.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:59.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:02:59.720 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:59.725 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:59.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:59.758 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:02:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:59.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:02:59.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:59.884 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:02:59.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:02:59.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:02:59.894 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 83175 83159 63.3724 38.3125 0.0653131 0.0157728 2023-10-09T09:02:59.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:02:59.997 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:02:59 smithi094 bash[59677]: cluster 2023-10-09T09:02:59.530095+0000 mon.a (mon.0) 1040 : cluster 3 Health check update: Degraded data redundancy: 21739/242352 objects degraded (8.970%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:03:00.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:00.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:00.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:00.133 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:02:59 smithi016 bash[75179]: cluster 2023-10-09T09:02:59.530095+0000 mon.a (mon.0) 1040 : cluster 3 Health check update: Degraded data redundancy: 21739/242352 objects degraded (8.970%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:03:00.133 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:02:59 smithi016 bash[77892]: cluster 2023-10-09T09:02:59.530095+0000 mon.a (mon.0) 1040 : cluster 3 Health check update: Degraded data redundancy: 21739/242352 objects degraded (8.970%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:03:00.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:00.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:00.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:00.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:00.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:00.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:00.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:00.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:00.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:00.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:00.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:00.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:00.894 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 83767 83751 63.0546 37 0.00787369 0.0158518 2023-10-09T09:03:00.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:00.937 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:00 smithi016 bash[75179]: cluster 2023-10-09T09:02:59.819894+0000 mgr.y (mgr.25421) 763 : cluster 0 pgmap v786: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 27 MiB/s wr, 437 op/s; 21739/242352 objects degraded (8.970%); 4323/242352 objects misplaced (1.784%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T09:03:00.938 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:00 smithi016 bash[77892]: cluster 2023-10-09T09:02:59.819894+0000 mgr.y (mgr.25421) 763 : cluster 0 pgmap v786: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 27 MiB/s wr, 437 op/s; 21739/242352 objects degraded (8.970%); 4323/242352 objects misplaced (1.784%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T09:03:00.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:00.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:00.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:01.033 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:03:01.070 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:01.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:01.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:01.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:01.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:01.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:00 smithi094 bash[59677]: cluster 2023-10-09T09:02:59.819894+0000 mgr.y (mgr.25421) 763 : cluster 0 pgmap v786: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 27 MiB/s wr, 437 op/s; 21739/242352 objects degraded (8.970%); 4323/242352 objects misplaced (1.784%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T09:03:01.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:01.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:01.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:01.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:01.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:01.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:01.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:01.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:01.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:01.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:01.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:01.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:01.758 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:01.759 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:01.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:01.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:01.896 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 84435 84419 62.799 41.75 0.0353412 0.0159166 2023-10-09T09:03:01.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:01.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:01.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:02.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:02.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:02.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:02.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:02.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:02.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:02.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:02.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:02.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:02.446 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:02.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:02.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:02.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:02.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:02.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:02.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:02.847 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:02.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:02.896 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 85114 85098 62.5594 42.4375 0.00502302 0.0159765 2023-10-09T09:03:02.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:02.965 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:02 smithi094 bash[59677]: cluster 2023-10-09T09:03:01.820677+0000 mgr.y (mgr.25421) 764 : cluster 0 pgmap v787: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 35 MiB/s wr, 556 op/s; 21739/248385 objects degraded (8.752%); 4323/248385 objects misplaced (1.740%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:03:02.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:02.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:03.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:03.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:03.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:03.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:03.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:02 smithi016 bash[75179]: cluster 2023-10-09T09:03:01.820677+0000 mgr.y (mgr.25421) 764 : cluster 0 pgmap v787: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 35 MiB/s wr, 556 op/s; 21739/248385 objects degraded (8.752%); 4323/248385 objects misplaced (1.740%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:03:03.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:02 smithi016 bash[77892]: cluster 2023-10-09T09:03:01.820677+0000 mgr.y (mgr.25421) 764 : cluster 0 pgmap v787: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 35 MiB/s wr, 556 op/s; 21739/248385 objects degraded (8.752%); 4323/248385 objects misplaced (1.740%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:03:03.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:03.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:03.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:03.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:03.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:03.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:03.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:03.498 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:03.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:03.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:03.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:03.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:03.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:03.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:03.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:03.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:03.897 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 85757 85741 62.2993 40.1875 0.00910641 0.0160438 2023-10-09T09:03:03.902 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:03.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:03.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:04.025 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:04.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:04.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:04.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:04.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:04.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:04.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:04.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:04.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:04.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:04.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:04.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:04.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:04.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:04.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:04.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:04.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:04.897 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 86397 86381 62.043 40 0.0221435 0.0161094 2023-10-09T09:03:04.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:04.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:04.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:05.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:05.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:05.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:05.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:05.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:05.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:05.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:05.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:05.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:05.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:05.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:05 smithi094 bash[59677]: cluster 2023-10-09T09:03:03.821396+0000 mgr.y (mgr.25421) 765 : cluster 0 pgmap v788: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 32 MiB/s wr, 512 op/s; 21739/252012 objects degraded (8.626%); 3908/252012 objects misplaced (1.551%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:03:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:05 smithi094 bash[59677]: audit 2023-10-09T09:03:04.235897+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:05 smithi094 bash[59677]: audit 2023-10-09T09:03:04.237161+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:05 smithi094 bash[59677]: cluster 2023-10-09T09:03:04.531338+0000 mon.a (mon.0) 1043 : cluster 3 Health check update: Degraded data redundancy: 21739/252012 objects degraded (8.626%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:05.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:05.540 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:05.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:05.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[75179]: cluster 2023-10-09T09:03:03.821396+0000 mgr.y (mgr.25421) 765 : cluster 0 pgmap v788: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 32 MiB/s wr, 512 op/s; 21739/252012 objects degraded (8.626%); 3908/252012 objects misplaced (1.551%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:03:05.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[75179]: audit 2023-10-09T09:03:04.235897+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:05.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[75179]: audit 2023-10-09T09:03:04.237161+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:05.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[75179]: cluster 2023-10-09T09:03:04.531338+0000 mon.a (mon.0) 1043 : cluster 3 Health check update: Degraded data redundancy: 21739/252012 objects degraded (8.626%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:05.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[77892]: cluster 2023-10-09T09:03:03.821396+0000 mgr.y (mgr.25421) 765 : cluster 0 pgmap v788: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 32 MiB/s wr, 512 op/s; 21739/252012 objects degraded (8.626%); 3908/252012 objects misplaced (1.551%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:03:05.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[77892]: audit 2023-10-09T09:03:04.235897+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:05.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[77892]: audit 2023-10-09T09:03:04.237161+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:05.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[77892]: cluster 2023-10-09T09:03:04.531338+0000 mon.a (mon.0) 1043 : cluster 3 Health check update: Degraded data redundancy: 21739/252012 objects degraded (8.626%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:05.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:05.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:05.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:05.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:05.783 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:03:05.784 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:03:05.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:05.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:05.897 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 86959 86943 61.7371 35.125 0.00698779 0.0161912 2023-10-09T09:03:05.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:05.928 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:03:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:03:05] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T09:03:05.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:05.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:05.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:06.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:06.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:06.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:06.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:06.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:06.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:06.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:06.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:06.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:06.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:06.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:06.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:06.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:06.596 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:06.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:06.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:06.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:06.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:06.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:06.802 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:03:06.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:06.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 87609 87593 61.4986 40.625 0.020135 0.016253 2023-10-09T09:03:06.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:06.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:07.015 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:07.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:07.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:07.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:07.153 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:07.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:07.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:07.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:07.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:07.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:07.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:07.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:07.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:07.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:07 smithi016 bash[75179]: cluster 2023-10-09T09:03:05.822110+0000 mgr.y (mgr.25421) 766 : cluster 0 pgmap v789: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 37 MiB/s wr, 584 op/s; 21739/256284 objects degraded (8.482%); 3908/256284 objects misplaced (1.525%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T09:03:07.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:07 smithi016 bash[77892]: cluster 2023-10-09T09:03:05.822110+0000 mgr.y (mgr.25421) 766 : cluster 0 pgmap v789: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 37 MiB/s wr, 584 op/s; 21739/256284 objects degraded (8.482%); 3908/256284 objects misplaced (1.525%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T09:03:07.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:07.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:07.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:07 smithi094 bash[59677]: cluster 2023-10-09T09:03:05.822110+0000 mgr.y (mgr.25421) 766 : cluster 0 pgmap v789: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 37 MiB/s wr, 584 op/s; 21739/256284 objects degraded (8.482%); 3908/256284 objects misplaced (1.525%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T09:03:07.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:07.606 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:07.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:07.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:07.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:07.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:07.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:07.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:07.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:07.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 4 88236 88232 61.259 39.9375 0.0999952 0.0163151 2023-10-09T09:03:07.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:08.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:08.123 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:08.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:08.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:08.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:08.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:08.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:08.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:08.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:08.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:08.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:08.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:08.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:08.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:08.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:08.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0479 2023-10-09T09:03:08.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 88236 2023-10-09T09:03:08.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-10-09T09:03:08.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-10-09T09:03:08.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 61.2424 2023-10-09T09:03:08.899 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 32.1496 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 155.312 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 0 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 979 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 514.394 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2485 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 0 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.016318 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0741927 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 5.82908 2023-10-09T09:03:08.900 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00137305 2023-10-09T09:03:08.910 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:03:08.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:08.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:09.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:09.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:09.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:09.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:09.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:09.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:09.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:09.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:09.316 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:09 smithi016 bash[75179]: cluster 2023-10-09T09:03:07.822929+0000 mgr.y (mgr.25421) 767 : cluster 0 pgmap v790: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 41 MiB/s wr, 652 op/s; 21739/261600 objects degraded (8.310%); 3446/261600 objects misplaced (1.317%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:03:09.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:09 smithi016 bash[77892]: cluster 2023-10-09T09:03:07.822929+0000 mgr.y (mgr.25421) 767 : cluster 0 pgmap v790: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 41 MiB/s wr, 652 op/s; 21739/261600 objects degraded (8.310%); 3446/261600 objects misplaced (1.317%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:03:09.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:09.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:09.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:09.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:09.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:09.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:09.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:09.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:09.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:09.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:09.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:09 smithi094 bash[59677]: cluster 2023-10-09T09:03:07.822929+0000 mgr.y (mgr.25421) 767 : cluster 0 pgmap v790: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 41 MiB/s wr, 652 op/s; 21739/261600 objects degraded (8.310%); 3446/261600 objects misplaced (1.317%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:03:09.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:09.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:09.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:09.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:09.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:09.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:09.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:09.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:10.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:10.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:10.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:10.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:10.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:10.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:10 smithi016 bash[75179]: cluster 2023-10-09T09:03:09.532531+0000 mon.a (mon.0) 1044 : cluster 3 Health check update: Degraded data redundancy: 21739/261600 objects degraded (8.310%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:10.245 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:10 smithi016 bash[77892]: cluster 2023-10-09T09:03:09.532531+0000 mon.a (mon.0) 1044 : cluster 3 Health check update: Degraded data redundancy: 21739/261600 objects degraded (8.310%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:10.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:10.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:10.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:10.380 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:10 smithi094 bash[59677]: cluster 2023-10-09T09:03:09.532531+0000 mon.a (mon.0) 1044 : cluster 3 Health check update: Degraded data redundancy: 21739/261600 objects degraded (8.310%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:10.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:10.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:10.525 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:10.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:10.652 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:10.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:10.782 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:10.885 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:10.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:10.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:11.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:11.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:11.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:11.173 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:11.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:11.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:11.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:11.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:11.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:11 smithi016 bash[75179]: cluster 2023-10-09T09:03:09.823510+0000 mgr.y (mgr.25421) 768 : cluster 0 pgmap v791: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 33 MiB/s wr, 534 op/s; 21739/261600 objects degraded (8.310%); 3446/261600 objects misplaced (1.317%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:03:11.418 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:11 smithi016 bash[77892]: cluster 2023-10-09T09:03:09.823510+0000 mgr.y (mgr.25421) 768 : cluster 0 pgmap v791: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 33 MiB/s wr, 534 op/s; 21739/261600 objects degraded (8.310%); 3446/261600 objects misplaced (1.317%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:03:11.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:11.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:11.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:11 smithi094 bash[59677]: cluster 2023-10-09T09:03:09.823510+0000 mgr.y (mgr.25421) 768 : cluster 0 pgmap v791: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 33 MiB/s wr, 534 op/s; 21739/261600 objects degraded (8.310%); 3446/261600 objects misplaced (1.317%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:03:11.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:11.564 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:11.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:11.604 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:03:11.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:11.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:11.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:11.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:11.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:11.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:11.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:11.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:12.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:12.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:12.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:12.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:12.198 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:12.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:12.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:12.326 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:12.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:12.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:12.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:12.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:12.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:12.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:12.589 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:12.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:12.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:12.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:12.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:12.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:12.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:12.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:12.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:12.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:12.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:13.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:13.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:13.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:13.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:13 smithi016 bash[75179]: cluster 2023-10-09T09:03:11.824294+0000 mgr.y (mgr.25421) 769 : cluster 0 pgmap v792: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 38 MiB/s wr, 666 op/s; 21739/262542 objects degraded (8.280%); 3446/262542 objects misplaced (1.313%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:03:13.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:13 smithi016 bash[77892]: cluster 2023-10-09T09:03:11.824294+0000 mgr.y (mgr.25421) 769 : cluster 0 pgmap v792: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 38 MiB/s wr, 666 op/s; 21739/262542 objects degraded (8.280%); 3446/262542 objects misplaced (1.313%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:03:13.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:13.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:13.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:13.415 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:13.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:13.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:13.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:13.640 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:13 smithi094 bash[59677]: cluster 2023-10-09T09:03:11.824294+0000 mgr.y (mgr.25421) 769 : cluster 0 pgmap v792: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 38 MiB/s wr, 666 op/s; 21739/262542 objects degraded (8.280%); 3446/262542 objects misplaced (1.313%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:03:13.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:13.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:13.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:13.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:13.934 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:13.955 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:13.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:14.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:14.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:14.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:14.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:14.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:14.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:14.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:14.334 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:14.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:14.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:14.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:14.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:14.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:14.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:14.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:14.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:14.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:14.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:14.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:14.761 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:14.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:14.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:14.891 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:14.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:14.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:15.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:15.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:15.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:15.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:15.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:15.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:15.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:15.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:15.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:15.421 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:15.443 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:15.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:15.549 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:15.558 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:15 smithi094 bash[59677]: cluster 2023-10-09T09:03:13.824985+0000 mgr.y (mgr.25421) 770 : cluster 0 pgmap v793: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 28 MiB/s wr, 581 op/s; 21739/260256 objects degraded (8.353%); 2755/260256 objects misplaced (1.059%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:03:15.559 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:15 smithi094 bash[59677]: cluster 2023-10-09T09:03:14.533715+0000 mon.a (mon.0) 1045 : cluster 3 Health check update: Degraded data redundancy: 21739/260256 objects degraded (8.353%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:15.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:15.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:15.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:15.711 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:15.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:15 smithi016 bash[75179]: cluster 2023-10-09T09:03:13.824985+0000 mgr.y (mgr.25421) 770 : cluster 0 pgmap v793: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 28 MiB/s wr, 581 op/s; 21739/260256 objects degraded (8.353%); 2755/260256 objects misplaced (1.059%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:03:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:15 smithi016 bash[75179]: cluster 2023-10-09T09:03:14.533715+0000 mon.a (mon.0) 1045 : cluster 3 Health check update: Degraded data redundancy: 21739/260256 objects degraded (8.353%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:15 smithi016 bash[77892]: cluster 2023-10-09T09:03:13.824985+0000 mgr.y (mgr.25421) 770 : cluster 0 pgmap v793: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 28 MiB/s wr, 581 op/s; 21739/260256 objects degraded (8.353%); 2755/260256 objects misplaced (1.059%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:03:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:15 smithi016 bash[77892]: cluster 2023-10-09T09:03:14.533715+0000 mon.a (mon.0) 1045 : cluster 3 Health check update: Degraded data redundancy: 21739/260256 objects degraded (8.353%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:15.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:15.842 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:15.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:15.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:15.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:15.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:16.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:16.022 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:03:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:03:15] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T09:03:16.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:16.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:16.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:16.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:16.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:16.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:16.281 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:03:16.281 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:03:16.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:16.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:16.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:16.500 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:16.525 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:16.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:16.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:16.770 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:16.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:16.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:16.896 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:16.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:16.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:16.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:17.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:17.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:17.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:17.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:17.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:17.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:17.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:17.302 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:17 smithi016 bash[75179]: cluster 2023-10-09T09:03:15.825790+0000 mgr.y (mgr.25421) 771 : cluster 0 pgmap v794: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 22 MiB/s wr, 602 op/s; 21739/255858 objects degraded (8.497%); 2755/255858 objects misplaced (1.077%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:03:17.302 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:17 smithi016 bash[77892]: cluster 2023-10-09T09:03:15.825790+0000 mgr.y (mgr.25421) 771 : cluster 0 pgmap v794: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 22 MiB/s wr, 602 op/s; 21739/255858 objects degraded (8.497%); 2755/255858 objects misplaced (1.077%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:03:17.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:17.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:17.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:03:17.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:17.438 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:17 smithi094 bash[59677]: cluster 2023-10-09T09:03:15.825790+0000 mgr.y (mgr.25421) 771 : cluster 0 pgmap v794: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 22 MiB/s wr, 602 op/s; 21739/255858 objects degraded (8.497%); 2755/255858 objects misplaced (1.077%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:03:17.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:17.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:17.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:17.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:17.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:17.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:17.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:17.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:17.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:17.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:17.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:17.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:17.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:18.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:18.017 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:03:18.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:18.080 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:03:18.123 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:18.150 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:18.157 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:03:18.215 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:03:18.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:18.259 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:03:18.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:18.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:03:18.355 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:03:18.403 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.407 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:03:18.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:18.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:18 smithi094 bash[59677]: audit 2023-10-09T09:03:17.964186+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.16:0/4165328393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:03:18.436 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:18.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:18.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:18.646 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:18 smithi016 bash[75179]: audit 2023-10-09T09:03:17.964186+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.16:0/4165328393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:03:18.646 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:18 smithi016 bash[77892]: audit 2023-10-09T09:03:17.964186+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.16:0/4165328393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:03:18.668 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:18.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:18.797 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:18.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:18.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:18.947 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:18.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:19.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:19.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:19.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:19.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:19.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:19.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:19.292 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:03:19.297 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T09:03:19.334 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[75179]: cluster 2023-10-09T09:03:17.826670+0000 mgr.y (mgr.25421) 772 : cluster 0 pgmap v795: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85 B/s rd, 15 MiB/s wr, 635 op/s; 21739/250407 objects degraded (8.681%); 2138/250407 objects misplaced (0.854%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T09:03:19.334 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[75179]: audit 2023-10-09T09:03:18.373815+0000 mon.a (mon.0) 1046 : audit 1 from='client.? 172.21.15.16:0/2302164151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:03:19.334 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[75179]: audit 2023-10-09T09:03:18.383997+0000 mon.a (mon.0) 1047 : audit 1 from='client.? 172.21.15.16:0/2996995028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:03:19.334 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[75179]: audit 2023-10-09T09:03:19.231603+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:19.335 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[77892]: cluster 2023-10-09T09:03:17.826670+0000 mgr.y (mgr.25421) 772 : cluster 0 pgmap v795: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85 B/s rd, 15 MiB/s wr, 635 op/s; 21739/250407 objects degraded (8.681%); 2138/250407 objects misplaced (0.854%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T09:03:19.335 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[77892]: audit 2023-10-09T09:03:18.373815+0000 mon.a (mon.0) 1046 : audit 1 from='client.? 172.21.15.16:0/2302164151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:03:19.335 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[77892]: audit 2023-10-09T09:03:18.383997+0000 mon.a (mon.0) 1047 : audit 1 from='client.? 172.21.15.16:0/2996995028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:03:19.335 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:19 smithi016 bash[77892]: audit 2023-10-09T09:03:19.231603+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:19.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:19.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:19.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:19.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:19.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:19.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:19.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:19.608 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:19.617 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:19 smithi094 bash[59677]: cluster 2023-10-09T09:03:17.826670+0000 mgr.y (mgr.25421) 772 : cluster 0 pgmap v795: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85 B/s rd, 15 MiB/s wr, 635 op/s; 21739/250407 objects degraded (8.681%); 2138/250407 objects misplaced (0.854%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T09:03:19.617 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:19 smithi094 bash[59677]: audit 2023-10-09T09:03:18.373815+0000 mon.a (mon.0) 1046 : audit 1 from='client.? 172.21.15.16:0/2302164151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:03:19.617 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:19 smithi094 bash[59677]: audit 2023-10-09T09:03:18.383997+0000 mon.a (mon.0) 1047 : audit 1 from='client.? 172.21.15.16:0/2996995028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:03:19.617 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:19 smithi094 bash[59677]: audit 2023-10-09T09:03:19.231603+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:19.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:19.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:19.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:19.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:19.765 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:19.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:19.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:19.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:19.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:19.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:20.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:20.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:20.156 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:20.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:20.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:20.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:20.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:20.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:20.522 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:20.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.269390+0000 mon.a (mon.0) 1049 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:03:20.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.282192+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T09:03:20.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.282428+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: audit 2023-10-09T09:03:19.284510+0000 mon.a (mon.0) 1050 : audit 1 from='client.? 172.21.15.16:0/2302164151' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: audit 2023-10-09T09:03:19.284583+0000 mon.a (mon.0) 1051 : audit 1 from='client.? 172.21.15.16:0/2996995028' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.284626+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e193: 8 total, 8 up, 5 in 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.534719+0000 mon.a (mon.0) 1053 : cluster 3 Health check update: Degraded data redundancy: 21739/250407 objects degraded (8.681%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.537994+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.541171+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.541179+0000 mon.a (mon.0) 1054 : cluster 0 osdmap e194: 8 total, 8 up, 5 in 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.543076+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.545291+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:20 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.792200+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:20.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:20.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:20.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:20.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.269390+0000 mon.a (mon.0) 1049 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:03:20.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.282192+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T09:03:20.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.282428+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T09:03:20.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: audit 2023-10-09T09:03:19.284510+0000 mon.a (mon.0) 1050 : audit 1 from='client.? 172.21.15.16:0/2302164151' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:03:20.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: audit 2023-10-09T09:03:19.284583+0000 mon.a (mon.0) 1051 : audit 1 from='client.? 172.21.15.16:0/2996995028' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.284626+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e193: 8 total, 8 up, 5 in 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.534719+0000 mon.a (mon.0) 1053 : cluster 3 Health check update: Degraded data redundancy: 21739/250407 objects degraded (8.681%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.537994+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.541171+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.541179+0000 mon.a (mon.0) 1054 : cluster 0 osdmap e194: 8 total, 8 up, 5 in 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.543076+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.545291+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.792200+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.269390+0000 mon.a (mon.0) 1049 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.282192+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.282428+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: audit 2023-10-09T09:03:19.284510+0000 mon.a (mon.0) 1050 : audit 1 from='client.? 172.21.15.16:0/2302164151' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: audit 2023-10-09T09:03:19.284583+0000 mon.a (mon.0) 1051 : audit 1 from='client.? 172.21.15.16:0/2996995028' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.284626+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e193: 8 total, 8 up, 5 in 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.534719+0000 mon.a (mon.0) 1053 : cluster 3 Health check update: Degraded data redundancy: 21739/250407 objects degraded (8.681%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.537994+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.541171+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.541179+0000 mon.a (mon.0) 1054 : cluster 0 osdmap e194: 8 total, 8 up, 5 in 2023-10-09T09:03:20.727 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.543076+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.727 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.545291+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.727 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:20 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.792200+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:20.790 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:20.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:20.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:20.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:20.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:20.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:21.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:21.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:21.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:21.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:21.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:21.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:21.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:21.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:21.311 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.792119+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:21.311 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:19.827392+0000 mgr.y (mgr.25421) 773 : cluster 0 pgmap v798: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 127 B/s rd, 872 KiB/s wr, 533 op/s; 21739/250407 objects degraded (8.681%); 2138/250407 objects misplaced (0.854%); 10 MiB/s, 163 objects/s recovering 2023-10-09T09:03:21.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:21.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:21.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:21.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:21.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:21.487 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:21.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:21.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:21.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.792119+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:19.827392+0000 mgr.y (mgr.25421) 773 : cluster 0 pgmap v798: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 127 B/s rd, 872 KiB/s wr, 533 op/s; 21739/250407 objects degraded (8.681%); 2138/250407 objects misplaced (0.854%); 10 MiB/s, 163 objects/s recovering 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:20.547524+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:20.548968+0000 osd.0 (osd.0) 9 : cluster 0 8.1 starting backfill to osd.3 from (0'0,0'0] MAX to 192'5960 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:20.549779+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:20.551034+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e195: 8 total, 8 up, 5 in 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:20.553008+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:20.555346+0000 osd.4 (osd.4) 144 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 192'6050 2023-10-09T09:03:21.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:21 smithi094 bash[59677]: cluster 2023-10-09T09:03:20.555600+0000 osd.4 (osd.4) 145 : cluster 0 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 192'5961 2023-10-09T09:03:21.613 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:21.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:21.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:21.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:21.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:20.547524+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:20.548968+0000 osd.0 (osd.0) 9 : cluster 0 8.1 starting backfill to osd.3 from (0'0,0'0] MAX to 192'5960 2023-10-09T09:03:21.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:20.549779+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:20.551034+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e195: 8 total, 8 up, 5 in 2023-10-09T09:03:21.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:20.553008+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:20.555346+0000 osd.4 (osd.4) 144 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 192'6050 2023-10-09T09:03:21.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[75179]: cluster 2023-10-09T09:03:20.555600+0000 osd.4 (osd.4) 145 : cluster 0 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 192'5961 2023-10-09T09:03:21.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.792119+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T09:03:21.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:19.827392+0000 mgr.y (mgr.25421) 773 : cluster 0 pgmap v798: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 127 B/s rd, 872 KiB/s wr, 533 op/s; 21739/250407 objects degraded (8.681%); 2138/250407 objects misplaced (0.854%); 10 MiB/s, 163 objects/s recovering 2023-10-09T09:03:21.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:20.547524+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:20.548968+0000 osd.0 (osd.0) 9 : cluster 0 8.1 starting backfill to osd.3 from (0'0,0'0] MAX to 192'5960 2023-10-09T09:03:21.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:20.549779+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:20.551034+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e195: 8 total, 8 up, 5 in 2023-10-09T09:03:21.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:20.553008+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T09:03:21.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:20.555346+0000 osd.4 (osd.4) 144 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 192'6050 2023-10-09T09:03:21.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:21 smithi016 bash[77892]: cluster 2023-10-09T09:03:20.555600+0000 osd.4 (osd.4) 145 : cluster 0 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 192'5961 2023-10-09T09:03:21.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:21.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:21.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:21.859 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:21.890 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:21.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:21.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:21.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:22.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:22.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:22.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:22.138 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:03:22.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:22.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:22.254 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:22.286 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:22.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:22.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:22.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:22.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:22.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:22.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:22.672 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:22.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:22.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:03:22.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:22.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:22.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:22.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:22.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:22.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:23.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:23.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:23.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:23.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:23.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:23.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:23.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:03:23.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:23.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:23.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:23.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:23.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:23.480 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:23 smithi016 bash[75179]: cluster 2023-10-09T09:03:21.829087+0000 mgr.y (mgr.25421) 774 : cluster 0 pgmap v800: 89 pgs: 2 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 17419/245502 objects degraded (7.095%); 4107/245502 objects misplaced (1.673%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T09:03:23.481 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:23 smithi016 bash[75179]: audit 2023-10-09T09:03:22.879286+0000 mon.a (mon.0) 1056 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:23.481 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:23 smithi016 bash[77892]: cluster 2023-10-09T09:03:21.829087+0000 mgr.y (mgr.25421) 774 : cluster 0 pgmap v800: 89 pgs: 2 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 17419/245502 objects degraded (7.095%); 4107/245502 objects misplaced (1.673%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T09:03:23.481 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:23 smithi016 bash[77892]: audit 2023-10-09T09:03:22.879286+0000 mon.a (mon.0) 1056 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:23.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:23.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:23.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:23.644 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:23.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:23 smithi094 bash[59677]: cluster 2023-10-09T09:03:21.829087+0000 mgr.y (mgr.25421) 774 : cluster 0 pgmap v800: 89 pgs: 2 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 17419/245502 objects degraded (7.095%); 4107/245502 objects misplaced (1.673%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T09:03:23.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:23 smithi094 bash[59677]: audit 2023-10-09T09:03:22.879286+0000 mon.a (mon.0) 1056 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:23.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:23.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:23.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:23.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:23.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:23.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:23.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:24.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:24.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:24.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:24.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:24.195 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:24.300 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:24.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:24.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:24.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:24.454 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:24.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:24.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:24.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:24.692 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:24.719 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:24.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:03:24.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:24.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:24.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:24.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:24.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:24.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:25.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:25.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:25.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:25.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:25.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:25.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:25.355 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:25.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:25.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:25.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:25.496 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:25 smithi016 bash[75179]: cluster 2023-10-09T09:03:23.829828+0000 mgr.y (mgr.25421) 775 : cluster 0 pgmap v801: 89 pgs: 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 421 op/s; 19329/242814 objects degraded (7.960%); 5917/242814 objects misplaced (2.437%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:03:25.497 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:25 smithi016 bash[75179]: cluster 2023-10-09T09:03:24.536538+0000 mon.a (mon.0) 1057 : cluster 3 Health check update: Degraded data redundancy: 19329/242814 objects degraded (7.960%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:25.497 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:25 smithi016 bash[77892]: cluster 2023-10-09T09:03:23.829828+0000 mgr.y (mgr.25421) 775 : cluster 0 pgmap v801: 89 pgs: 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 421 op/s; 19329/242814 objects degraded (7.960%); 5917/242814 objects misplaced (2.437%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:03:25.497 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:25 smithi016 bash[77892]: cluster 2023-10-09T09:03:24.536538+0000 mon.a (mon.0) 1057 : cluster 3 Health check update: Degraded data redundancy: 19329/242814 objects degraded (7.960%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:25.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:25.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:25.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:25.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:25 smithi094 bash[59677]: cluster 2023-10-09T09:03:23.829828+0000 mgr.y (mgr.25421) 775 : cluster 0 pgmap v801: 89 pgs: 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 421 op/s; 19329/242814 objects degraded (7.960%); 5917/242814 objects misplaced (2.437%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:03:25.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:25 smithi094 bash[59677]: cluster 2023-10-09T09:03:24.536538+0000 mon.a (mon.0) 1057 : cluster 3 Health check update: Degraded data redundancy: 19329/242814 objects degraded (7.960%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:25.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:25.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:25.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:25.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:03:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:03:25] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.33.4" 2023-10-09T09:03:25.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:25.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:25.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:25.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:25.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:25.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:25.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:26.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:26.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:26.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:26.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:26.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:26.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:26.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:26.298 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:26.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:26.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:26.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:26.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:26.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:26.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:26.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:26.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:26.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:26.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:26.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:26.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:26.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:26.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:26.852 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:03:26.852 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:03:26.852 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:03:26.852 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:03:26.852 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:03:26.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:26.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:26.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:26.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:27.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:27.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:27.113 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:27.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:27.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:27.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:27.241 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:27.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:27.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:27.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:27.383 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:27 smithi094 bash[59677]: cluster 2023-10-09T09:03:25.830622+0000 mgr.y (mgr.25421) 776 : cluster 0 pgmap v802: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 21406/238803 objects degraded (8.964%); 13506/238803 objects misplaced (5.656%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T09:03:27.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:27.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:27.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:27.506 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:27.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:27.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:27.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:27.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:27.644 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:27 smithi016 bash[75179]: cluster 2023-10-09T09:03:25.830622+0000 mgr.y (mgr.25421) 776 : cluster 0 pgmap v802: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 21406/238803 objects degraded (8.964%); 13506/238803 objects misplaced (5.656%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T09:03:27.644 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:27 smithi016 bash[77892]: cluster 2023-10-09T09:03:25.830622+0000 mgr.y (mgr.25421) 776 : cluster 0 pgmap v802: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 21406/238803 objects degraded (8.964%); 13506/238803 objects misplaced (5.656%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T09:03:27.666 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:27.666 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:27.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:27.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:27.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:27.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:27.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:27.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:03:27.931 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:27.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:28.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:28.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:28.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:28.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:28.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:28.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:28.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:28.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:28.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:28.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:28.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:28.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:28.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:28.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:28.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:28.684 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:28.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:28.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:28.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:28.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:28.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:28.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:28.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:28.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:29.025 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:29.059 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:29.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:29.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:29.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:29.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:29.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:29.307 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:29 smithi094 bash[59677]: cluster 2023-10-09T09:03:27.831342+0000 mgr.y (mgr.25421) 777 : cluster 0 pgmap v803: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 682 op/s; 21406/233433 objects degraded (9.170%); 13308/233433 objects misplaced (5.701%); 5.2 MiB/s, 84 objects/s recovering 2023-10-09T09:03:29.324 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:29.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:29.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:29.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:29 smithi016 bash[75179]: cluster 2023-10-09T09:03:27.831342+0000 mgr.y (mgr.25421) 777 : cluster 0 pgmap v803: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 682 op/s; 21406/233433 objects degraded (9.170%); 13308/233433 objects misplaced (5.701%); 5.2 MiB/s, 84 objects/s recovering 2023-10-09T09:03:29.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:29 smithi016 bash[77892]: cluster 2023-10-09T09:03:27.831342+0000 mgr.y (mgr.25421) 777 : cluster 0 pgmap v803: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 682 op/s; 21406/233433 objects degraded (9.170%); 13308/233433 objects misplaced (5.701%); 5.2 MiB/s, 84 objects/s recovering 2023-10-09T09:03:29.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:29.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:29.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:29.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:29.589 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:29.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:29.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:29.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:29.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:29.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:29.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:29.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:29.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:29.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:30.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:30.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:30.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:30.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:30.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:30.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:30.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:30.398 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:30.431 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:30.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:30.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:30.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:30.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:30.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:30 smithi094 bash[59677]: cluster 2023-10-09T09:03:29.539392+0000 mon.a (mon.0) 1058 : cluster 3 Health check update: Degraded data redundancy: 21406/233433 objects degraded (9.170%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:30.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:30 smithi094 bash[59677]: audit 2023-10-09T09:03:29.727798+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:30.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:30 smithi094 bash[59677]: audit 2023-10-09T09:03:29.742899+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:30.800 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:30.829 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:30.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:30.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:30 smithi016 bash[75179]: cluster 2023-10-09T09:03:29.539392+0000 mon.a (mon.0) 1058 : cluster 3 Health check update: Degraded data redundancy: 21406/233433 objects degraded (9.170%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:30.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:30 smithi016 bash[75179]: audit 2023-10-09T09:03:29.727798+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:30.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:30 smithi016 bash[75179]: audit 2023-10-09T09:03:29.742899+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:30.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:30 smithi016 bash[77892]: cluster 2023-10-09T09:03:29.539392+0000 mon.a (mon.0) 1058 : cluster 3 Health check update: Degraded data redundancy: 21406/233433 objects degraded (9.170%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:30.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:30 smithi016 bash[77892]: audit 2023-10-09T09:03:29.727798+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:30.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:30 smithi016 bash[77892]: audit 2023-10-09T09:03:29.742899+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:30.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:30.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:31.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:31.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:31.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:31.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:31.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:31.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:31.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:31.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:31.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:31.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:31.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:31 smithi094 bash[59677]: cluster 2023-10-09T09:03:29.831875+0000 mgr.y (mgr.25421) 778 : cluster 0 pgmap v804: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 21406/233433 objects degraded (9.170%); 13308/233433 objects misplaced (5.701%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:03:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:31 smithi094 bash[59677]: audit 2023-10-09T09:03:30.768832+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:31.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:31 smithi094 bash[59677]: audit 2023-10-09T09:03:30.776037+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:31.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:31.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:31 smithi016 bash[75179]: cluster 2023-10-09T09:03:29.831875+0000 mgr.y (mgr.25421) 778 : cluster 0 pgmap v804: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 21406/233433 objects degraded (9.170%); 13308/233433 objects misplaced (5.701%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:03:31.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:31 smithi016 bash[75179]: audit 2023-10-09T09:03:30.768832+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:31.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:31 smithi016 bash[75179]: audit 2023-10-09T09:03:30.776037+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:31.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:31 smithi016 bash[77892]: cluster 2023-10-09T09:03:29.831875+0000 mgr.y (mgr.25421) 778 : cluster 0 pgmap v804: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 21406/233433 objects degraded (9.170%); 13308/233433 objects misplaced (5.701%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:03:31.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:31 smithi016 bash[77892]: audit 2023-10-09T09:03:30.768832+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:31.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:31 smithi016 bash[77892]: audit 2023-10-09T09:03:30.776037+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:31.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:31.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:31.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:31.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:31.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:31.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:31.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:31.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:31.883 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:31.910 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:31.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:32.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:32.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:32.155 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:32.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:32.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:32.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:32.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:32.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:32.413 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:32.420 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:03:32.424 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:32 smithi094 bash[59677]: audit 2023-10-09T09:03:31.420446+0000 mon.a (mon.0) 1063 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:32.424 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:32 smithi094 bash[59677]: audit 2023-10-09T09:03:31.421916+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:32.424 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:32 smithi094 bash[59677]: audit 2023-10-09T09:03:31.429224+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:32.443 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:32.553 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:32.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:32 smithi016 bash[75179]: audit 2023-10-09T09:03:31.420446+0000 mon.a (mon.0) 1063 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:32.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:32 smithi016 bash[75179]: audit 2023-10-09T09:03:31.421916+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:32.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:32 smithi016 bash[75179]: audit 2023-10-09T09:03:31.429224+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:32.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:32 smithi016 bash[77892]: audit 2023-10-09T09:03:31.420446+0000 mon.a (mon.0) 1063 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:32.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:32 smithi016 bash[77892]: audit 2023-10-09T09:03:31.421916+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:32.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:32 smithi016 bash[77892]: audit 2023-10-09T09:03:31.429224+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:32.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:32.682 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:32.713 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:32.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:32.841 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:32.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:32.972 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:32.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:32.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:33.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:33.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:33.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:33.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:33.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:33.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:33.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:33.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:33.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:33.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:33.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:33.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:33.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:33.613 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:33 smithi094 bash[59677]: cluster 2023-10-09T09:03:31.832760+0000 mgr.y (mgr.25421) 779 : cluster 0 pgmap v805: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 568 op/s; 21217/227514 objects degraded (9.326%); 12929/227514 objects misplaced (5.683%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T09:03:33.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:33.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:33.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:33.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:33.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:33 smithi016 bash[75179]: cluster 2023-10-09T09:03:31.832760+0000 mgr.y (mgr.25421) 779 : cluster 0 pgmap v805: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 568 op/s; 21217/227514 objects degraded (9.326%); 12929/227514 objects misplaced (5.683%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T09:03:33.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:33 smithi016 bash[77892]: cluster 2023-10-09T09:03:31.832760+0000 mgr.y (mgr.25421) 779 : cluster 0 pgmap v805: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 568 op/s; 21217/227514 objects degraded (9.326%); 12929/227514 objects misplaced (5.683%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T09:03:33.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:33.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:33.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:33.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:33.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:34.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:34.038 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:34.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:34.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:34.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:34.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:34.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:34.417 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:34.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:34.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:34.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:34.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:34.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:34.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:34.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:34.810 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:34.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:34.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:34.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:34.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:34.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:35.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:35.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:35.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:35.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:35.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:35.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[75179]: cluster 2023-10-09T09:03:33.833471+0000 mgr.y (mgr.25421) 780 : cluster 0 pgmap v806: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 599 op/s; 21217/223899 objects degraded (9.476%); 12714/223899 objects misplaced (5.678%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:03:35.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[75179]: audit 2023-10-09T09:03:34.235590+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:35.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[75179]: audit 2023-10-09T09:03:34.237051+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:35.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[75179]: cluster 2023-10-09T09:03:34.540537+0000 mon.a (mon.0) 1068 : cluster 3 Health check update: Degraded data redundancy: 21217/223899 objects degraded (9.476%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:35.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:35.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[77892]: cluster 2023-10-09T09:03:33.833471+0000 mgr.y (mgr.25421) 780 : cluster 0 pgmap v806: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 599 op/s; 21217/223899 objects degraded (9.476%); 12714/223899 objects misplaced (5.678%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:03:35.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[77892]: audit 2023-10-09T09:03:34.235590+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:35.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[77892]: audit 2023-10-09T09:03:34.237051+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:35.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[77892]: cluster 2023-10-09T09:03:34.540537+0000 mon.a (mon.0) 1068 : cluster 3 Health check update: Degraded data redundancy: 21217/223899 objects degraded (9.476%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:35.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:35.384 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:35 smithi094 bash[59677]: cluster 2023-10-09T09:03:33.833471+0000 mgr.y (mgr.25421) 780 : cluster 0 pgmap v806: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 599 op/s; 21217/223899 objects degraded (9.476%); 12714/223899 objects misplaced (5.678%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:03:35.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:35 smithi094 bash[59677]: audit 2023-10-09T09:03:34.235590+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:03:35.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:35 smithi094 bash[59677]: audit 2023-10-09T09:03:34.237051+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:35.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:35 smithi094 bash[59677]: cluster 2023-10-09T09:03:34.540537+0000 mon.a (mon.0) 1068 : cluster 3 Health check update: Degraded data redundancy: 21217/223899 objects degraded (9.476%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:35.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:35.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:35.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:35.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:35.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:35.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:35.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:35.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:35.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:35.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:35.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:35.772 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:03:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:03:35] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-09T09:03:35.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:35.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:35.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:35.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:35.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:36.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:36.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:36.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:36.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:36.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:36.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:36.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:36.318 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:36.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:36.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:36.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:36.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:36.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:36.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:36.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:36.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:36.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:36.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:36.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:36.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:37.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:37.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:37.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:37.184 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:03:37.185 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T09:03:37.185 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T09:03:37.185 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [] 2023-10-09T09:03:37.185 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:03:37.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:37.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:37.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:37.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:37 smithi016 bash[75179]: cluster 2023-10-09T09:03:35.834210+0000 mgr.y (mgr.25421) 781 : cluster 0 pgmap v807: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 642 op/s; 21020/219669 objects degraded (9.569%); 12289/219669 objects misplaced (5.594%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T09:03:37.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:37 smithi016 bash[77892]: cluster 2023-10-09T09:03:35.834210+0000 mgr.y (mgr.25421) 781 : cluster 0 pgmap v807: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 642 op/s; 21020/219669 objects degraded (9.569%); 12289/219669 objects misplaced (5.594%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T09:03:37.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:37.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:37.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:37.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:37 smithi094 bash[59677]: cluster 2023-10-09T09:03:35.834210+0000 mgr.y (mgr.25421) 781 : cluster 0 pgmap v807: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 642 op/s; 21020/219669 objects degraded (9.569%); 12289/219669 objects misplaced (5.594%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T09:03:37.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:37.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:37.624 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:37.653 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:37.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:37.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:37.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:37.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:37.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:37.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:37.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:37.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:37.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:38.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:38.059 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:38.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:38.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:38.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:03:38.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:38.188 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:38.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:38.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:38.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:38.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:38.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:38.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:38.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:38.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:38.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:38.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:38.596 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:38.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:38.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:38.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:38.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:38.748 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:38.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:38.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:38.878 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:38.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:38.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:38.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:39.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:39.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:39.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:39.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:39.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:39.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:39.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:39.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:39.408 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:39.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:39.517 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:39 smithi016 bash[75179]: cluster 2023-10-09T09:03:37.834927+0000 mgr.y (mgr.25421) 782 : cluster 0 pgmap v808: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 693 op/s; 21020/213828 objects degraded (9.830%); 12087/213828 objects misplaced (5.653%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:03:39.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:39 smithi016 bash[77892]: cluster 2023-10-09T09:03:37.834927+0000 mgr.y (mgr.25421) 782 : cluster 0 pgmap v808: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 693 op/s; 21020/213828 objects degraded (9.830%); 12087/213828 objects misplaced (5.653%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:03:39.520 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:39 smithi094 bash[59677]: cluster 2023-10-09T09:03:37.834927+0000 mgr.y (mgr.25421) 782 : cluster 0 pgmap v808: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 693 op/s; 21020/213828 objects degraded (9.830%); 12087/213828 objects misplaced (5.653%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:03:39.554 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:39.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:39.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:39.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:39.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:39.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:39.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:39.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:40.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:40.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:40.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:40.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:40.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:40.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:40.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:40.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:40.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:40.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:40.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:40 smithi016 bash[75179]: cluster 2023-10-09T09:03:39.541693+0000 mon.a (mon.0) 1069 : cluster 3 Health check update: Degraded data redundancy: 21020/213828 objects degraded (9.830%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:40 smithi016 bash[77892]: cluster 2023-10-09T09:03:39.541693+0000 mon.a (mon.0) 1069 : cluster 3 Health check update: Degraded data redundancy: 21020/213828 objects degraded (9.830%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:40.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:40.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:40.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:40.664 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:40.674 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:40 smithi094 bash[59677]: cluster 2023-10-09T09:03:39.541693+0000 mon.a (mon.0) 1069 : cluster 3 Health check update: Degraded data redundancy: 21020/213828 objects degraded (9.830%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:40.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:40.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:40.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:40.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:40.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:40.920 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:40.946 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:40.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:40.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:41.049 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:41.075 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:41.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:41.181 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:41.216 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:41.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:41.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:41.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:41.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:41.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:41.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:41.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:41 smithi094 bash[59677]: cluster 2023-10-09T09:03:39.835546+0000 mgr.y (mgr.25421) 783 : cluster 0 pgmap v809: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 544 op/s; 21020/213828 objects degraded (9.830%); 12087/213828 objects misplaced (5.653%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:03:41.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:41.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:41.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:41.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:41.597 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:41.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:41.705 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:41.732 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:41.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:41.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:41.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:41 smithi016 bash[75179]: cluster 2023-10-09T09:03:39.835546+0000 mgr.y (mgr.25421) 783 : cluster 0 pgmap v809: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 544 op/s; 21020/213828 objects degraded (9.830%); 12087/213828 objects misplaced (5.653%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:03:41.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:41 smithi016 bash[77892]: cluster 2023-10-09T09:03:39.835546+0000 mgr.y (mgr.25421) 783 : cluster 0 pgmap v809: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 544 op/s; 21020/213828 objects degraded (9.830%); 12087/213828 objects misplaced (5.653%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:03:41.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:41.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:41.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:41.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:41.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:42.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:42.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:42.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:42.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:42.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:42.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:42.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:42.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:42.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:42.507 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:42.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:42.627 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:03:42.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:42.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:42.709 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:03:42.771 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:03:42.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:42.802 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:42.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:42.820 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:03:42.866 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:03:42.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:42.916 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:03:42.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:42.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:42.960 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:03:43.000 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:03:43.005 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.005 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.006 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.006 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.006 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.006 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.006 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.006 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:03:43.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:43.068 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:43.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:43.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:43.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:43.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:43.286 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:03:43.326 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:43.335 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:43 smithi094 bash[59677]: cluster 2023-10-09T09:03:41.836398+0000 mgr.y (mgr.25421) 784 : cluster 0 pgmap v810: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 715 op/s; 20782/207654 objects degraded (10.008%); 11662/207654 objects misplaced (5.616%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T09:03:43.335 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:43 smithi094 bash[59677]: audit 2023-10-09T09:03:42.555106+0000 mon.a (mon.0) 1070 : audit 0 from='client.? 172.21.15.16:0/71296384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:03:43.336 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:43 smithi094 bash[59677]: audit 2023-10-09T09:03:43.247710+0000 mon.a (mon.0) 1071 : audit 1 from='client.? 172.21.15.16:0/3605961288' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:03:43.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:43.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:43.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:43.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:43 smithi016 bash[75179]: cluster 2023-10-09T09:03:41.836398+0000 mgr.y (mgr.25421) 784 : cluster 0 pgmap v810: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 715 op/s; 20782/207654 objects degraded (10.008%); 11662/207654 objects misplaced (5.616%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T09:03:43.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:43 smithi016 bash[75179]: audit 2023-10-09T09:03:42.555106+0000 mon.a (mon.0) 1070 : audit 0 from='client.? 172.21.15.16:0/71296384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:03:43.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:43 smithi016 bash[75179]: audit 2023-10-09T09:03:43.247710+0000 mon.a (mon.0) 1071 : audit 1 from='client.? 172.21.15.16:0/3605961288' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:03:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:43 smithi016 bash[77892]: cluster 2023-10-09T09:03:41.836398+0000 mgr.y (mgr.25421) 784 : cluster 0 pgmap v810: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 715 op/s; 20782/207654 objects degraded (10.008%); 11662/207654 objects misplaced (5.616%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T09:03:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:43 smithi016 bash[77892]: audit 2023-10-09T09:03:42.555106+0000 mon.a (mon.0) 1070 : audit 0 from='client.? 172.21.15.16:0/71296384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:03:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:43 smithi016 bash[77892]: audit 2023-10-09T09:03:43.247710+0000 mon.a (mon.0) 1071 : audit 1 from='client.? 172.21.15.16:0/3605961288' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:03:43.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:43.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:43.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:43.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:43.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:43.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:43.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:43.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:43.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:43.907 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:43.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:43.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:44.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:44.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:44.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:44.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:44.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:44.285 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:03:44.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:44.313 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: cluster 2023-10-09T09:03:43.254857+0000 mon.a (mon.0) 1072 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:03:44.313 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: audit 2023-10-09T09:03:43.272035+0000 mon.a (mon.0) 1073 : audit 1 from='client.? 172.21.15.16:0/3605961288' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:03:44.313 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: cluster 2023-10-09T09:03:43.272274+0000 mon.a (mon.0) 1074 : cluster 0 osdmap e196: 8 total, 8 up, 5 in 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: cluster 2023-10-09T09:03:43.273249+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: cluster 2023-10-09T09:03:43.275792+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: audit 2023-10-09T09:03:43.404364+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.16:0/1025108650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: audit 2023-10-09T09:03:43.404843+0000 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: cluster 2023-10-09T09:03:43.559780+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[75179]: cluster 2023-10-09T09:03:43.561200+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: cluster 2023-10-09T09:03:43.254857+0000 mon.a (mon.0) 1072 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: audit 2023-10-09T09:03:43.272035+0000 mon.a (mon.0) 1073 : audit 1 from='client.? 172.21.15.16:0/3605961288' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:03:44.314 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: cluster 2023-10-09T09:03:43.272274+0000 mon.a (mon.0) 1074 : cluster 0 osdmap e196: 8 total, 8 up, 5 in 2023-10-09T09:03:44.315 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: cluster 2023-10-09T09:03:43.273249+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.315 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: cluster 2023-10-09T09:03:43.275792+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.315 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: audit 2023-10-09T09:03:43.404364+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.16:0/1025108650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:03:44.315 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: audit 2023-10-09T09:03:43.404843+0000 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:03:44.315 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: cluster 2023-10-09T09:03:43.559780+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.315 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:44 smithi016 bash[77892]: cluster 2023-10-09T09:03:43.561200+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:44.427 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:44.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:44.533 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:44.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: cluster 2023-10-09T09:03:43.254857+0000 mon.a (mon.0) 1072 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:03:44.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: audit 2023-10-09T09:03:43.272035+0000 mon.a (mon.0) 1073 : audit 1 from='client.? 172.21.15.16:0/3605961288' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:03:44.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: cluster 2023-10-09T09:03:43.272274+0000 mon.a (mon.0) 1074 : cluster 0 osdmap e196: 8 total, 8 up, 5 in 2023-10-09T09:03:44.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: cluster 2023-10-09T09:03:43.273249+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: cluster 2023-10-09T09:03:43.275792+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: audit 2023-10-09T09:03:43.404364+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.16:0/1025108650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:03:44.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: audit 2023-10-09T09:03:43.404843+0000 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:03:44.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: cluster 2023-10-09T09:03:43.559780+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:44 smithi094 bash[59677]: cluster 2023-10-09T09:03:43.561200+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T09:03:44.560 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:44.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:44.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:44.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:44.789 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:44.819 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:44.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:44.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:44.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:45.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:45.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:45.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:45.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:45.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:45.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:45.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:45.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:45.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:45.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:45.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:45.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:43.836963+0000 mgr.y (mgr.25421) 785 : cluster 0 pgmap v812: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 669 op/s; 20782/203793 objects degraded (10.198%); 11457/203793 objects misplaced (5.622%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:44.274796+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: audit 2023-10-09T09:03:44.277657+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:44.279277+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e197: 8 total, 8 up, 5 in 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:44.315095+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:44.324623+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:44.326079+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:44.542839+0000 mon.a (mon.0) 1078 : cluster 3 Health check update: Degraded data redundancy: 20782/203793 objects degraded (10.198%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:45 smithi094 bash[59677]: cluster 2023-10-09T09:03:44.697759+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:45.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:45.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:45.607 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:45.631 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:45.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:45.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:45.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:43.836963+0000 mgr.y (mgr.25421) 785 : cluster 0 pgmap v812: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 669 op/s; 20782/203793 objects degraded (10.198%); 11457/203793 objects misplaced (5.622%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:44.274796+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: audit 2023-10-09T09:03:44.277657+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:44.279277+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e197: 8 total, 8 up, 5 in 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:44.315095+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:44.324623+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:44.326079+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:44.542839+0000 mon.a (mon.0) 1078 : cluster 3 Health check update: Degraded data redundancy: 20782/203793 objects degraded (10.198%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[75179]: cluster 2023-10-09T09:03:44.697759+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:03:45] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:43.836963+0000 mgr.y (mgr.25421) 785 : cluster 0 pgmap v812: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 669 op/s; 20782/203793 objects degraded (10.198%); 11457/203793 objects misplaced (5.622%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:44.274796+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: audit 2023-10-09T09:03:44.277657+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:44.279277+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e197: 8 total, 8 up, 5 in 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:44.315095+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:44.324623+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:44.326079+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:44.542839+0000 mon.a (mon.0) 1078 : cluster 3 Health check update: Degraded data redundancy: 20782/203793 objects degraded (10.198%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:45.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:45 smithi016 bash[77892]: cluster 2023-10-09T09:03:44.697759+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T09:03:45.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:45.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:45.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:45.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:45.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:46.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:46.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:46.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:46.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:46.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:46.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:46.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:46.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:46.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:46.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:46.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:46.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:46.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:46.695 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:46.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:46.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:46.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:46.930 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:46.956 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:46.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:47.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:47.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:47.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:47.194 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:47.224 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:47.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:47.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:47.319 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:47 smithi016 bash[77892]: cluster 2023-10-09T09:03:45.837727+0000 mgr.y (mgr.25421) 786 : cluster 0 pgmap v814: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 600 op/s; 20589/199419 objects degraded (10.324%); 11026/199419 objects misplaced (5.529%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T09:03:47.319 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:47 smithi016 bash[75179]: cluster 2023-10-09T09:03:45.837727+0000 mgr.y (mgr.25421) 786 : cluster 0 pgmap v814: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 600 op/s; 20589/199419 objects degraded (10.324%); 11026/199419 objects misplaced (5.529%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T09:03:47.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:47.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:47.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:47.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:47.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:47.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:47.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:47 smithi094 bash[59677]: cluster 2023-10-09T09:03:45.837727+0000 mgr.y (mgr.25421) 786 : cluster 0 pgmap v814: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 600 op/s; 20589/199419 objects degraded (10.324%); 11026/199419 objects misplaced (5.529%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T09:03:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:47.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:47.593 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:47.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:47.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:47.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:47.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:47.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:47.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:47.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:47.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:47.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:47.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:03:47.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:48.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:48.122 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:48.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:48.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:48.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:48.308 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:48.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:48.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:48.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:48.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:48.565 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:48.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:48.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:48.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:48.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:48.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:48.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:48.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:48.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:48.954 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:48.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:49.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:49.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:49.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:49.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:49.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:49.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:49.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:49.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:49.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:49.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:49.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:49.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:49 smithi094 bash[59677]: cluster 2023-10-09T09:03:47.838384+0000 mgr.y (mgr.25421) 787 : cluster 0 pgmap v815: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 818 op/s; 20499/194178 objects degraded (10.557%); 10905/194178 objects misplaced (5.616%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:03:49.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:49 smithi094 bash[59677]: audit 2023-10-09T09:03:49.232427+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:49.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:49 smithi016 bash[75179]: cluster 2023-10-09T09:03:47.838384+0000 mgr.y (mgr.25421) 787 : cluster 0 pgmap v815: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 818 op/s; 20499/194178 objects degraded (10.557%); 10905/194178 objects misplaced (5.616%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:03:49.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:49 smithi016 bash[75179]: audit 2023-10-09T09:03:49.232427+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:49.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:49 smithi016 bash[77892]: cluster 2023-10-09T09:03:47.838384+0000 mgr.y (mgr.25421) 787 : cluster 0 pgmap v815: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 818 op/s; 20499/194178 objects degraded (10.557%); 10905/194178 objects misplaced (5.616%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:03:49.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:49 smithi016 bash[77892]: audit 2023-10-09T09:03:49.232427+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:03:49.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:49.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:49.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:49.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:03:49.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:49.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:49.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:49.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:49.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:03:49.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:49.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:49.922 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:49.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:49.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:50.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:50.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:50.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:50.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:50.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:50.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:50.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:50.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:50.344 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:50.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:50.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:50.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:50.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:50 smithi016 bash[75179]: cluster 2023-10-09T09:03:49.544110+0000 mon.a (mon.0) 1080 : cluster 3 Health check update: Degraded data redundancy: 20499/194178 objects degraded (10.557%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:50.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:50 smithi016 bash[77892]: cluster 2023-10-09T09:03:49.544110+0000 mon.a (mon.0) 1080 : cluster 3 Health check update: Degraded data redundancy: 20499/194178 objects degraded (10.557%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:50.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:50.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:50.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:50 smithi094 bash[59677]: cluster 2023-10-09T09:03:49.544110+0000 mon.a (mon.0) 1080 : cluster 3 Health check update: Degraded data redundancy: 20499/194178 objects degraded (10.557%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:50.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:50.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:50.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:50.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:50.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:50.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:50.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:50.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:50.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:50.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:51.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:51.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:51.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:51.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:51.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:51.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:51.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:51.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:51.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:51.537 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:51.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:51.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:51.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:51 smithi016 bash[75179]: cluster 2023-10-09T09:03:49.838952+0000 mgr.y (mgr.25421) 788 : cluster 0 pgmap v816: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 20499/194178 objects degraded (10.557%); 10905/194178 objects misplaced (5.616%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:03:51.706 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:51 smithi016 bash[77892]: cluster 2023-10-09T09:03:49.838952+0000 mgr.y (mgr.25421) 788 : cluster 0 pgmap v816: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 20499/194178 objects degraded (10.557%); 10905/194178 objects misplaced (5.616%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:03:51.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:51.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:51.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:51.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:51.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:51 smithi094 bash[59677]: cluster 2023-10-09T09:03:49.838952+0000 mgr.y (mgr.25421) 788 : cluster 0 pgmap v816: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 20499/194178 objects degraded (10.557%); 10905/194178 objects misplaced (5.616%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:03:51.967 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:51.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:51.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:52.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:52.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:52.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:52.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:52.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:52.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:52.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:52.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:52.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:52.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:52.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:52.448 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:52.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:52.503 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:52.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:52.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:52.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:52.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:52.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:52.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:52.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:52 smithi094 bash[59677]: cluster 2023-10-09T09:03:51.839739+0000 mgr.y (mgr.25421) 789 : cluster 0 pgmap v817: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 601 op/s; 20302/188331 objects degraded (10.780%); 10441/188331 objects misplaced (5.544%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:03:52.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:52.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:52 smithi016 bash[75179]: cluster 2023-10-09T09:03:51.839739+0000 mgr.y (mgr.25421) 789 : cluster 0 pgmap v817: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 601 op/s; 20302/188331 objects degraded (10.780%); 10441/188331 objects misplaced (5.544%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:03:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:52 smithi016 bash[77892]: cluster 2023-10-09T09:03:51.839739+0000 mgr.y (mgr.25421) 789 : cluster 0 pgmap v817: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 601 op/s; 20302/188331 objects degraded (10.780%); 10441/188331 objects misplaced (5.544%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:03:52.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:52.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:52.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:52.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:52.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:53.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:53.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:53.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:53.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:53.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:53.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:53.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:53.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:53.433 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:53.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:53.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:53.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:53.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:53.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:53.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:53.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:53.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:53.855 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:53.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:53.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:53.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:53.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:53.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:54.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:54.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:54.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:54.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:54.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:54.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:54.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:54.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:54.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:54.485 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:54.486 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:54.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:54.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:54.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:54.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:54.658 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:54.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:54.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:54.762 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:54.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:54.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:54.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:54.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:54.898 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:54 smithi016 bash[75179]: cluster 2023-10-09T09:03:53.840501+0000 mgr.y (mgr.25421) 790 : cluster 0 pgmap v818: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 631 op/s; 20087/184830 objects degraded (10.868%); 10440/184830 objects misplaced (5.648%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:03:54.898 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:54 smithi016 bash[75179]: cluster 2023-10-09T09:03:54.545276+0000 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 20087/184830 objects degraded (10.868%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:54.899 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:54 smithi094 bash[59677]: cluster 2023-10-09T09:03:53.840501+0000 mgr.y (mgr.25421) 790 : cluster 0 pgmap v818: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 631 op/s; 20087/184830 objects degraded (10.868%); 10440/184830 objects misplaced (5.648%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:03:54.899 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:54 smithi094 bash[59677]: cluster 2023-10-09T09:03:54.545276+0000 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 20087/184830 objects degraded (10.868%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:54.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:54.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:55.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:55.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:55.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:55.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:55.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:55.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:54 smithi016 bash[77892]: cluster 2023-10-09T09:03:53.840501+0000 mgr.y (mgr.25421) 790 : cluster 0 pgmap v818: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 631 op/s; 20087/184830 objects degraded (10.868%); 10440/184830 objects misplaced (5.648%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:03:55.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:54 smithi016 bash[77892]: cluster 2023-10-09T09:03:54.545276+0000 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 20087/184830 objects degraded (10.868%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:55.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:55.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:55.425 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:55.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:55.454 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:03:55.454 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:03:55.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:55.557 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:55.579 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:55.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:55.683 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:55.710 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:55.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:55.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:55.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:55.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:55.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:55.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:56.068 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:03:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:03:55] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.33.4" 2023-10-09T09:03:56.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:56.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:56.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:56.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:56.260 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:56.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:56.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:56.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:56.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:56.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:56.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:56.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:56.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:56.662 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:56.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:56.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:56.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:56.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:56.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:56.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:56.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:56.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:56.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:56.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:56 smithi094 bash[59677]: cluster 2023-10-09T09:03:55.841734+0000 mgr.y (mgr.25421) 791 : cluster 0 pgmap v819: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 19881/180546 objects degraded (11.012%); 9890/180546 objects misplaced (5.478%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T09:03:57.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:57.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:57.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:57.243 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:03:57.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:57.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:03:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:56 smithi016 bash[75179]: cluster 2023-10-09T09:03:55.841734+0000 mgr.y (mgr.25421) 791 : cluster 0 pgmap v819: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 19881/180546 objects degraded (11.012%); 9890/180546 objects misplaced (5.478%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T09:03:57.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:56 smithi016 bash[77892]: cluster 2023-10-09T09:03:55.841734+0000 mgr.y (mgr.25421) 791 : cluster 0 pgmap v819: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 19881/180546 objects degraded (11.012%); 9890/180546 objects misplaced (5.478%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T09:03:57.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:57.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:57.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:57.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:57.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:57.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:57.653 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:57.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:57.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:57.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:57.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:57.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:57.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:58.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:58.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:58.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:58.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:58.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:58.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:58.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:58.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:58.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:58.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:58.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:58.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:03:58.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:58.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:03:58.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:58.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:58.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:58.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:58.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:58.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:58.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:58.868 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:58.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:03:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:58.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:03:58.905 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:58 smithi016 bash[77892]: cluster 2023-10-09T09:03:57.842529+0000 mgr.y (mgr.25421) 792 : cluster 0 pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 679 op/s; 19707/174957 objects degraded (11.264%); 9891/174957 objects misplaced (5.653%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T09:03:59.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:59.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:59.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:59.143 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:58 smithi094 bash[59677]: cluster 2023-10-09T09:03:57.842529+0000 mgr.y (mgr.25421) 792 : cluster 0 pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 679 op/s; 19707/174957 objects degraded (11.264%); 9891/174957 objects misplaced (5.653%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T09:03:59.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:03:59.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:58 smithi016 bash[75179]: cluster 2023-10-09T09:03:57.842529+0000 mgr.y (mgr.25421) 792 : cluster 0 pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 679 op/s; 19707/174957 objects degraded (11.264%); 9891/174957 objects misplaced (5.653%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T09:03:59.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:59.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:59.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:59.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:03:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:03:59.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:59.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:03:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:03:59.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:59.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:03:59.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:03:59.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:59.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:03:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:03:59.909 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:03:59 smithi016 bash[75179]: cluster 2023-10-09T09:03:59.546330+0000 mon.a (mon.0) 1082 : cluster 3 Health check update: Degraded data redundancy: 19707/174957 objects degraded (11.264%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:03:59.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:59.940 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:03:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:03:59.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:00.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:00.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:00.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:00.176 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:03:59 smithi016 bash[77892]: cluster 2023-10-09T09:03:59.546330+0000 mon.a (mon.0) 1082 : cluster 3 Health check update: Degraded data redundancy: 19707/174957 objects degraded (11.264%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:00.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:00.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:00.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:03:59 smithi094 bash[59677]: cluster 2023-10-09T09:03:59.546330+0000 mon.a (mon.0) 1082 : cluster 3 Health check update: Degraded data redundancy: 19707/174957 objects degraded (11.264%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:00.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:00.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:00.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:04:00.426 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:00.452 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:00.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:00.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:00.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:00.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:00.686 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:00.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:00.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:00.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:00.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:00.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:00.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:01.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:01.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:01.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:01.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:01.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:01.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:01.329 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:01.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:01.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:01.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:01.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:01.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:01.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:01.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:01.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:01.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:01.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:01.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:01.731 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:01 smithi094 bash[59677]: cluster 2023-10-09T09:03:59.843120+0000 mgr.y (mgr.25421) 793 : cluster 0 pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 19707/174957 objects degraded (11.264%); 9891/174957 objects misplaced (5.653%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:04:01.731 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:01 smithi094 bash[59677]: cluster 2023-10-09T09:04:00.559693+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.731 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:01 smithi094 bash[59677]: cluster 2023-10-09T09:04:00.561087+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.731 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:01 smithi094 bash[59677]: cluster 2023-10-09T09:04:00.564915+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e198: 8 total, 8 up, 5 in 2023-10-09T09:04:01.731 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:01 smithi094 bash[59677]: cluster 2023-10-09T09:04:00.580290+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.731 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:01 smithi094 bash[59677]: cluster 2023-10-09T09:04:00.584079+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.748 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:01.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[75179]: cluster 2023-10-09T09:03:59.843120+0000 mgr.y (mgr.25421) 793 : cluster 0 pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 19707/174957 objects degraded (11.264%); 9891/174957 objects misplaced (5.653%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:04:01.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[75179]: cluster 2023-10-09T09:04:00.559693+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[75179]: cluster 2023-10-09T09:04:00.561087+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[75179]: cluster 2023-10-09T09:04:00.564915+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e198: 8 total, 8 up, 5 in 2023-10-09T09:04:01.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[75179]: cluster 2023-10-09T09:04:00.580290+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[75179]: cluster 2023-10-09T09:04:00.584079+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[77892]: cluster 2023-10-09T09:03:59.843120+0000 mgr.y (mgr.25421) 793 : cluster 0 pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 19707/174957 objects degraded (11.264%); 9891/174957 objects misplaced (5.653%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:04:01.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[77892]: cluster 2023-10-09T09:04:00.559693+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[77892]: cluster 2023-10-09T09:04:00.561087+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[77892]: cluster 2023-10-09T09:04:00.564915+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e198: 8 total, 8 up, 5 in 2023-10-09T09:04:01.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[77892]: cluster 2023-10-09T09:04:00.580290+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:01 smithi016 bash[77892]: cluster 2023-10-09T09:04:00.584079+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T09:04:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:01.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:01.854 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:01.883 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:01.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:01.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:01.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:02.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:02.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:02.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:02.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:02.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:02.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:02.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:02.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:02.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:02.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:02.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:02.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:02.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:02.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:02 smithi094 bash[59677]: cluster 2023-10-09T09:04:01.563438+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e199: 8 total, 8 up, 5 in 2023-10-09T09:04:02.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:02 smithi094 bash[59677]: cluster 2023-10-09T09:04:01.568070+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:02.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:02 smithi094 bash[59677]: cluster 2023-10-09T09:04:01.568074+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:02.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:02 smithi094 bash[59677]: cluster 2023-10-09T09:04:01.843697+0000 mgr.y (mgr.25421) 794 : cluster 0 pgmap v824: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 658 op/s; 19457/169014 objects degraded (11.512%); 9366/169014 objects misplaced (5.542%); 9.4 MiB/s, 157 objects/s recovering 2023-10-09T09:04:02.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:02.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[75179]: cluster 2023-10-09T09:04:01.563438+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e199: 8 total, 8 up, 5 in 2023-10-09T09:04:02.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[75179]: cluster 2023-10-09T09:04:01.568070+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:02.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[75179]: cluster 2023-10-09T09:04:01.568074+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:02.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[75179]: cluster 2023-10-09T09:04:01.843697+0000 mgr.y (mgr.25421) 794 : cluster 0 pgmap v824: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 658 op/s; 19457/169014 objects degraded (11.512%); 9366/169014 objects misplaced (5.542%); 9.4 MiB/s, 157 objects/s recovering 2023-10-09T09:04:02.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[77892]: cluster 2023-10-09T09:04:01.563438+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e199: 8 total, 8 up, 5 in 2023-10-09T09:04:02.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[77892]: cluster 2023-10-09T09:04:01.568070+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:02.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[77892]: cluster 2023-10-09T09:04:01.568074+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:02.760 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:02 smithi016 bash[77892]: cluster 2023-10-09T09:04:01.843697+0000 mgr.y (mgr.25421) 794 : cluster 0 pgmap v824: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 658 op/s; 19457/169014 objects degraded (11.512%); 9366/169014 objects misplaced (5.542%); 9.4 MiB/s, 157 objects/s recovering 2023-10-09T09:04:02.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:02.888 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:02.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:02.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:03.024 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:03.056 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:03.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:03.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:03.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:03.290 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:03.322 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:03.426 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:03.450 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:03.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:03.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:03 smithi094 bash[59677]: cluster 2023-10-09T09:04:02.298766+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:03.572 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:03 smithi094 bash[59677]: cluster 2023-10-09T09:04:02.298870+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:03.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:03.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:03.713 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:03.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:03.816 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:03.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:03 smithi016 bash[75179]: cluster 2023-10-09T09:04:02.298766+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:03.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:03 smithi016 bash[75179]: cluster 2023-10-09T09:04:02.298870+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:03.825 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:03 smithi016 bash[77892]: cluster 2023-10-09T09:04:02.298766+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:03.825 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:03 smithi016 bash[77892]: cluster 2023-10-09T09:04:02.298870+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T09:04:03.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:03.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:03.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:03.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:04.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:04.107 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:04.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:04.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:04.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:04.346 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:04.370 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:04.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:04.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:04.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:04.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:04.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:04.694 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:04 smithi016 bash[77892]: cluster 2023-10-09T09:04:03.844402+0000 mgr.y (mgr.25421) 795 : cluster 0 pgmap v825: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 613 op/s; 18866/165825 objects degraded (11.377%); 6533/165825 objects misplaced (3.940%); 8.4 MiB/s, 142 objects/s recovering 2023-10-09T09:04:04.694 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:04 smithi016 bash[77892]: audit 2023-10-09T09:04:04.232534+0000 mon.a (mon.0) 1085 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:04.694 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:04 smithi016 bash[77892]: cluster 2023-10-09T09:04:04.547512+0000 mon.a (mon.0) 1086 : cluster 3 Health check update: Degraded data redundancy: 19457/169014 objects degraded (11.512%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:04.738 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:04 smithi094 bash[59677]: cluster 2023-10-09T09:04:03.844402+0000 mgr.y (mgr.25421) 795 : cluster 0 pgmap v825: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 613 op/s; 18866/165825 objects degraded (11.377%); 6533/165825 objects misplaced (3.940%); 8.4 MiB/s, 142 objects/s recovering 2023-10-09T09:04:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:04 smithi094 bash[59677]: audit 2023-10-09T09:04:04.232534+0000 mon.a (mon.0) 1085 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:04.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:04 smithi094 bash[59677]: cluster 2023-10-09T09:04:04.547512+0000 mon.a (mon.0) 1086 : cluster 3 Health check update: Degraded data redundancy: 19457/169014 objects degraded (11.512%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:04.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:04 smithi016 bash[75179]: cluster 2023-10-09T09:04:03.844402+0000 mgr.y (mgr.25421) 795 : cluster 0 pgmap v825: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 613 op/s; 18866/165825 objects degraded (11.377%); 6533/165825 objects misplaced (3.940%); 8.4 MiB/s, 142 objects/s recovering 2023-10-09T09:04:04.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:04 smithi016 bash[75179]: audit 2023-10-09T09:04:04.232534+0000 mon.a (mon.0) 1085 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:04 smithi016 bash[75179]: cluster 2023-10-09T09:04:04.547512+0000 mon.a (mon.0) 1086 : cluster 3 Health check update: Degraded data redundancy: 19457/169014 objects degraded (11.512%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:04.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:04.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:04.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:04.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:05.001 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:05.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:05.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:05.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:05.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:05.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:05.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:05.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:05.641 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.673 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:05.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:05.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:05.913 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:05.940 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:05.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:05.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:04:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:04:05] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-09T09:04:06.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:06.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:06.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:06.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:06.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:06.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:06.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:06.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:06.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:06.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:06.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:06.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:06.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:06.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:06.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:06.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:06.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:06.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:06.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:06.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:06.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:06.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:06.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:06.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:06.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:06 smithi016 bash[75179]: cluster 2023-10-09T09:04:05.848711+0000 mgr.y (mgr.25421) 796 : cluster 0 pgmap v826: 89 pgs: 2 active+recovering+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 527 op/s; 18198/162276 objects degraded (11.214%); 3849/162276 objects misplaced (2.372%); 8.5 MiB/s, 143 objects/s recovering 2023-10-09T09:04:06.908 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:06 smithi016 bash[77892]: cluster 2023-10-09T09:04:05.848711+0000 mgr.y (mgr.25421) 796 : cluster 0 pgmap v826: 89 pgs: 2 active+recovering+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 527 op/s; 18198/162276 objects degraded (11.214%); 3849/162276 objects misplaced (2.372%); 8.5 MiB/s, 143 objects/s recovering 2023-10-09T09:04:06.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:07.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:07.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:07.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:07.123 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:07.131 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:06 smithi094 bash[59677]: cluster 2023-10-09T09:04:05.848711+0000 mgr.y (mgr.25421) 796 : cluster 0 pgmap v826: 89 pgs: 2 active+recovering+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 527 op/s; 18198/162276 objects degraded (11.214%); 3849/162276 objects misplaced (2.372%); 8.5 MiB/s, 143 objects/s recovering 2023-10-09T09:04:07.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:07.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:07.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:07.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:07.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:07.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:07.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:07.392 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:07.415 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:07.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:07.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:07.869 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:07.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:07.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:07.923 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:04:07.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:07.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:08.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:08.112 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:07 smithi016 bash[75179]: audit 2023-10-09T09:04:07.394301+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.16:0/3044759744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:04:08.113 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:07 smithi016 bash[75179]: audit 2023-10-09T09:04:07.394986+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:04:08.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:07 smithi016 bash[77892]: audit 2023-10-09T09:04:07.394301+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.16:0/3044759744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:04:08.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:07 smithi016 bash[77892]: audit 2023-10-09T09:04:07.394986+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:04:08.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:08.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:08.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:07 smithi094 bash[59677]: audit 2023-10-09T09:04:07.394301+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.16:0/3044759744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:04:08.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:07 smithi094 bash[59677]: audit 2023-10-09T09:04:07.394986+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:04:08.260 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:08.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:08.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:08.390 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:08.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:08.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:08.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:08.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:08.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:08.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:08.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:08.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:08.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:08.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:08.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:09.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:09.034 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:08 smithi094 bash[59677]: cluster 2023-10-09T09:04:07.849437+0000 mgr.y (mgr.25421) 797 : cluster 0 pgmap v827: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 17907/157284 objects degraded (11.385%); 7055/157284 objects misplaced (4.486%); 11 MiB/s, 179 objects/s recovering 2023-10-09T09:04:09.034 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:08 smithi094 bash[59677]: cluster 2023-10-09T09:04:07.908350+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T09:04:09.034 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:08 smithi094 bash[59677]: cluster 2023-10-09T09:04:07.909140+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T09:04:09.034 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:08 smithi094 bash[59677]: audit 2023-10-09T09:04:07.911184+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:04:09.034 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:08 smithi094 bash[59677]: cluster 2023-10-09T09:04:07.911450+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e200: 8 total, 8 up, 5 in 2023-10-09T09:04:09.051 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:09.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:09.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:09.284 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:09.308 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:09.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:09.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:09.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[75179]: cluster 2023-10-09T09:04:07.849437+0000 mgr.y (mgr.25421) 797 : cluster 0 pgmap v827: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 17907/157284 objects degraded (11.385%); 7055/157284 objects misplaced (4.486%); 11 MiB/s, 179 objects/s recovering 2023-10-09T09:04:09.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[75179]: cluster 2023-10-09T09:04:07.908350+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T09:04:09.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[75179]: cluster 2023-10-09T09:04:07.909140+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T09:04:09.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[75179]: audit 2023-10-09T09:04:07.911184+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:04:09.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[75179]: cluster 2023-10-09T09:04:07.911450+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e200: 8 total, 8 up, 5 in 2023-10-09T09:04:09.424 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[77892]: cluster 2023-10-09T09:04:07.849437+0000 mgr.y (mgr.25421) 797 : cluster 0 pgmap v827: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 17907/157284 objects degraded (11.385%); 7055/157284 objects misplaced (4.486%); 11 MiB/s, 179 objects/s recovering 2023-10-09T09:04:09.425 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[77892]: cluster 2023-10-09T09:04:07.908350+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T09:04:09.425 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[77892]: cluster 2023-10-09T09:04:07.909140+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T09:04:09.425 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[77892]: audit 2023-10-09T09:04:07.911184+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:04:09.425 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[77892]: cluster 2023-10-09T09:04:07.911450+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e200: 8 total, 8 up, 5 in 2023-10-09T09:04:09.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:09.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:09.552 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:09.577 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:09.684 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:09.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:09.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:09.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:09.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:09.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:09.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:10.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.093 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[75179]: cluster 2023-10-09T09:04:09.548413+0000 mon.a (mon.0) 1090 : cluster 3 Health check update: Degraded data redundancy: 17907/157284 objects degraded (11.385%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:10.093 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:09 smithi016 bash[77892]: cluster 2023-10-09T09:04:09.548413+0000 mon.a (mon.0) 1090 : cluster 3 Health check update: Degraded data redundancy: 17907/157284 objects degraded (11.385%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:10.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:10.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:10.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:09 smithi094 bash[59677]: cluster 2023-10-09T09:04:09.548413+0000 mon.a (mon.0) 1090 : cluster 3 Health check update: Degraded data redundancy: 17907/157284 objects degraded (11.385%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:10.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:10.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:10.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:10.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:10.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:10.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:10.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:10.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:10.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:10.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:10.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:10.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:10.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:10.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:10.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:10.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:10.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:10 smithi016 bash[75179]: cluster 2023-10-09T09:04:09.852368+0000 mgr.y (mgr.25421) 798 : cluster 0 pgmap v829: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 471 op/s; 17907/157284 objects degraded (11.385%); 7055/157284 objects misplaced (4.486%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T09:04:10.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:10 smithi016 bash[77892]: cluster 2023-10-09T09:04:09.852368+0000 mgr.y (mgr.25421) 798 : cluster 0 pgmap v829: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 471 op/s; 17907/157284 objects degraded (11.385%); 7055/157284 objects misplaced (4.486%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T09:04:11.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:11.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:11.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:11.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:11.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:10 smithi094 bash[59677]: cluster 2023-10-09T09:04:09.852368+0000 mgr.y (mgr.25421) 798 : cluster 0 pgmap v829: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 471 op/s; 17907/157284 objects degraded (11.385%); 7055/157284 objects misplaced (4.486%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T09:04:11.311 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:11.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:11.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:11.443 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:11.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:11.471 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T09:04:11.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:04:11.570 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:11.592 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:11.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:11.695 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:11.723 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:11.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:11.826 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:11.851 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:11.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:11.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:11.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:11.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:11.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:12.009 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:04:12.057 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:04:12.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.105 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:04:12.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:12.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:12.148 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:04:12.186 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:04:12.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.230 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:04:12.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:12.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:12.276 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:04:12.315 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:04:12.318 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.318 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.318 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.319 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.319 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.319 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.319 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.319 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:04:12.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:12.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:12.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:12.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:12.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:12.620 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:12.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:12.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:12.746 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:12.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:12.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:12.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:12.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:12.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:12.923 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[75179]: cluster 2023-10-09T09:04:11.853220+0000 mgr.y (mgr.25421) 799 : cluster 0 pgmap v830: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 591 op/s; 17907/151248 objects degraded (11.839%); 6618/151248 objects misplaced (4.376%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:04:12.924 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[75179]: audit 2023-10-09T09:04:11.937508+0000 mon.a (mon.0) 1091 : audit 0 from='client.? 172.21.15.16:0/1942227562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:04:12.924 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[75179]: audit 2023-10-09T09:04:12.163692+0000 mon.c (mon.2) 83 : audit 1 from='client.? 172.21.15.16:0/1023193690' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:04:12.924 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[75179]: audit 2023-10-09T09:04:12.164901+0000 mon.a (mon.0) 1092 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:04:12.944 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:04:13.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:13.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:12 smithi094 bash[59677]: cluster 2023-10-09T09:04:11.853220+0000 mgr.y (mgr.25421) 799 : cluster 0 pgmap v830: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 591 op/s; 17907/151248 objects degraded (11.839%); 6618/151248 objects misplaced (4.376%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:04:13.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:12 smithi094 bash[59677]: audit 2023-10-09T09:04:11.937508+0000 mon.a (mon.0) 1091 : audit 0 from='client.? 172.21.15.16:0/1942227562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:04:13.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:12 smithi094 bash[59677]: audit 2023-10-09T09:04:12.163692+0000 mon.c (mon.2) 83 : audit 1 from='client.? 172.21.15.16:0/1023193690' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:04:13.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:12 smithi094 bash[59677]: audit 2023-10-09T09:04:12.164901+0000 mon.a (mon.0) 1092 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:04:13.032 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:13.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:13.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:13.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:13.180 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[77892]: cluster 2023-10-09T09:04:11.853220+0000 mgr.y (mgr.25421) 799 : cluster 0 pgmap v830: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 591 op/s; 17907/151248 objects degraded (11.839%); 6618/151248 objects misplaced (4.376%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:04:13.180 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[77892]: audit 2023-10-09T09:04:11.937508+0000 mon.a (mon.0) 1091 : audit 0 from='client.? 172.21.15.16:0/1942227562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:04:13.180 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[77892]: audit 2023-10-09T09:04:12.163692+0000 mon.c (mon.2) 83 : audit 1 from='client.? 172.21.15.16:0/1023193690' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:04:13.181 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:12 smithi016 bash[77892]: audit 2023-10-09T09:04:12.164901+0000 mon.a (mon.0) 1092 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:04:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:13.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:13.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:13.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:13.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:13.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:13.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:13.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:13.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:13.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:13.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:13.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:13.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:13.969 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:13.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:13.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[75179]: cluster 2023-10-09T09:04:12.919669+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:04:14.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[75179]: audit 2023-10-09T09:04:12.935548+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:04:14.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[75179]: cluster 2023-10-09T09:04:12.935990+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e201: 8 total, 8 up, 5 in 2023-10-09T09:04:14.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[75179]: cluster 2023-10-09T09:04:12.943173+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[75179]: cluster 2023-10-09T09:04:12.943651+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[75179]: cluster 2023-10-09T09:04:13.178396+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[77892]: cluster 2023-10-09T09:04:12.919669+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:04:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[77892]: audit 2023-10-09T09:04:12.935548+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:04:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[77892]: cluster 2023-10-09T09:04:12.935990+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e201: 8 total, 8 up, 5 in 2023-10-09T09:04:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[77892]: cluster 2023-10-09T09:04:12.943173+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[77892]: cluster 2023-10-09T09:04:12.943651+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:13 smithi016 bash[77892]: cluster 2023-10-09T09:04:13.178396+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:14.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:14.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:14.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:14.209 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:14.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:13 smithi094 bash[59677]: cluster 2023-10-09T09:04:12.919669+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:04:14.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:13 smithi094 bash[59677]: audit 2023-10-09T09:04:12.935548+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:04:14.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:13 smithi094 bash[59677]: cluster 2023-10-09T09:04:12.935990+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e201: 8 total, 8 up, 5 in 2023-10-09T09:04:14.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:13 smithi094 bash[59677]: cluster 2023-10-09T09:04:12.943173+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:13 smithi094 bash[59677]: cluster 2023-10-09T09:04:12.943651+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:13 smithi094 bash[59677]: cluster 2023-10-09T09:04:13.178396+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:14.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:14.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:14.342 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:14.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:14.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:14.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:14.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:14.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:14.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:14.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:14.614 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:14.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:14.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:14.748 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:14.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:14.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:14.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:14.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:14.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:14.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:14.934 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:14 smithi094 bash[59677]: cluster 2023-10-09T09:04:13.173648+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:14.934 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:14 smithi094 bash[59677]: cluster 2023-10-09T09:04:13.853751+0000 mgr.y (mgr.25421) 800 : cluster 0 pgmap v832: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 595 op/s; 17882/147975 objects degraded (12.084%); 6261/147975 objects misplaced (4.231%); 5.4 MiB/s, 88 objects/s recovering 2023-10-09T09:04:14.934 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:14 smithi094 bash[59677]: cluster 2023-10-09T09:04:14.549688+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 17882/147975 objects degraded (12.084%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:15.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:15.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:14 smithi016 bash[75179]: cluster 2023-10-09T09:04:13.173648+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:15.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:14 smithi016 bash[75179]: cluster 2023-10-09T09:04:13.853751+0000 mgr.y (mgr.25421) 800 : cluster 0 pgmap v832: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 595 op/s; 17882/147975 objects degraded (12.084%); 6261/147975 objects misplaced (4.231%); 5.4 MiB/s, 88 objects/s recovering 2023-10-09T09:04:15.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:14 smithi016 bash[75179]: cluster 2023-10-09T09:04:14.549688+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 17882/147975 objects degraded (12.084%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:15.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:14 smithi016 bash[77892]: cluster 2023-10-09T09:04:13.173648+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T09:04:15.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:14 smithi016 bash[77892]: cluster 2023-10-09T09:04:13.853751+0000 mgr.y (mgr.25421) 800 : cluster 0 pgmap v832: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 595 op/s; 17882/147975 objects degraded (12.084%); 6261/147975 objects misplaced (4.231%); 5.4 MiB/s, 88 objects/s recovering 2023-10-09T09:04:15.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:14 smithi016 bash[77892]: cluster 2023-10-09T09:04:14.549688+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 17882/147975 objects degraded (12.084%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:15.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:15.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:15.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:15.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:15.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:15.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:15.311 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:15.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:15.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:15.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:15.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:15.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:15.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:15.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:15.607 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:15.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:15.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:15.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:15.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:15.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:15.739 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:04:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:04:15] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-09T09:04:15.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:15.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:15.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:15.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:15.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:16.012 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:16.041 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:16.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:16.139 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:16.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:16.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:16.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:16.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:16.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:16.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:16.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:04:16.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:16.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:16.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:16.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:16.669 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:16.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:16.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:16.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:16.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:16.949 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:16 smithi094 bash[59677]: cluster 2023-10-09T09:04:15.854470+0000 mgr.y (mgr.25421) 801 : cluster 0 pgmap v833: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 549 op/s; 17882/144096 objects degraded (12.410%); 5982/144096 objects misplaced (4.151%); 3.8 MiB/s, 70 objects/s recovering 2023-10-09T09:04:16.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:16.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:17.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:17.084 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:16 smithi016 bash[75179]: cluster 2023-10-09T09:04:15.854470+0000 mgr.y (mgr.25421) 801 : cluster 0 pgmap v833: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 549 op/s; 17882/144096 objects degraded (12.410%); 5982/144096 objects misplaced (4.151%); 3.8 MiB/s, 70 objects/s recovering 2023-10-09T09:04:17.084 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:16 smithi016 bash[77892]: cluster 2023-10-09T09:04:15.854470+0000 mgr.y (mgr.25421) 801 : cluster 0 pgmap v833: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 549 op/s; 17882/144096 objects degraded (12.410%); 5982/144096 objects misplaced (4.151%); 3.8 MiB/s, 70 objects/s recovering 2023-10-09T09:04:17.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:17.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:17.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:17.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:17.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:17.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:17.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:17.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:17.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:17.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:17.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:17.515 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:17.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:17.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:04:17.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:17.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:17.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:17.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:17.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:17.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:17.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:17.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:17.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:17.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:18.054 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:18.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:18.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:18.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:18.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:18.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:18.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:18.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:18.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:18.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:18.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:18.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:18.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:18.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:18.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:18.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:18.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:18.730 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:18.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:18.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:18.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:18.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:18.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:18.957 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:18 smithi094 bash[59677]: cluster 2023-10-09T09:04:17.855167+0000 mgr.y (mgr.25421) 802 : cluster 0 pgmap v834: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 630 op/s; 17637/138471 objects degraded (12.737%); 5773/138471 objects misplaced (4.169%); 3.1 MiB/s, 81 objects/s recovering 2023-10-09T09:04:18.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:19.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:19.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:19.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:19.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:18 smithi016 bash[75179]: cluster 2023-10-09T09:04:17.855167+0000 mgr.y (mgr.25421) 802 : cluster 0 pgmap v834: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 630 op/s; 17637/138471 objects degraded (12.737%); 5773/138471 objects misplaced (4.169%); 3.1 MiB/s, 81 objects/s recovering 2023-10-09T09:04:19.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:18 smithi016 bash[77892]: cluster 2023-10-09T09:04:17.855167+0000 mgr.y (mgr.25421) 802 : cluster 0 pgmap v834: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 630 op/s; 17637/138471 objects degraded (12.737%); 5773/138471 objects misplaced (4.169%); 3.1 MiB/s, 81 objects/s recovering 2023-10-09T09:04:19.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:19.242 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:19.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:19.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:19.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:19.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:19.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:19.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:19.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:19.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:19.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:19.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:19.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:19.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:19.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:19.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:19.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:19.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:20.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:20.056 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:19 smithi094 bash[59677]: audit 2023-10-09T09:04:19.234442+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:20.056 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:19 smithi094 bash[59677]: cluster 2023-10-09T09:04:19.550933+0000 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 17637/138471 objects degraded (12.737%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:20.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:20.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:20.206 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:20.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:20.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:19 smithi016 bash[75179]: audit 2023-10-09T09:04:19.234442+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:20.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:19 smithi016 bash[75179]: cluster 2023-10-09T09:04:19.550933+0000 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 17637/138471 objects degraded (12.737%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:20.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:19 smithi016 bash[77892]: audit 2023-10-09T09:04:19.234442+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:20.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:19 smithi016 bash[77892]: cluster 2023-10-09T09:04:19.550933+0000 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 17637/138471 objects degraded (12.737%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:20.304 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:20.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:20.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:20.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:20.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:20.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:20.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:20.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:20.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:20.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:20.732 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:20.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:20.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:20.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:20.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:20.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:20.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:20.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:21.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:21.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:21.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:21.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:21.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:21.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:21.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:21.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:21.385 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:21.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:21.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:21.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:21.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:21.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:21.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:21 smithi016 bash[75179]: cluster 2023-10-09T09:04:19.855706+0000 mgr.y (mgr.25421) 803 : cluster 0 pgmap v835: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 626 op/s; 17637/138471 objects degraded (12.737%); 5773/138471 objects misplaced (4.169%); 3.0 MiB/s, 80 objects/s recovering 2023-10-09T09:04:21.560 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:21 smithi016 bash[77892]: cluster 2023-10-09T09:04:19.855706+0000 mgr.y (mgr.25421) 803 : cluster 0 pgmap v835: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 626 op/s; 17637/138471 objects degraded (12.737%); 5773/138471 objects misplaced (4.169%); 3.0 MiB/s, 80 objects/s recovering 2023-10-09T09:04:21.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:21.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:21.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:21 smithi094 bash[59677]: cluster 2023-10-09T09:04:19.855706+0000 mgr.y (mgr.25421) 803 : cluster 0 pgmap v835: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 626 op/s; 17637/138471 objects degraded (12.737%); 5773/138471 objects misplaced (4.169%); 3.0 MiB/s, 80 objects/s recovering 2023-10-09T09:04:21.762 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:21.787 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:21.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:21.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:21.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:21.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:22.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:22.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:22.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:22.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:22.157 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:22.157 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:22.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:22.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:22.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:22.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:22.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:22.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:22.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:22.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:22.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:22.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:22.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:22.569 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:22.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:22.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:22.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:22.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:22.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:22.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:22.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:22.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:22.982 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:23.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:23.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:23.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:23.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:23.240 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:23.274 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:23.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:23.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:23.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:23.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:23.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:23.541 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:23.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:23.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:23.654 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:23 smithi016 bash[77892]: cluster 2023-10-09T09:04:21.856455+0000 mgr.y (mgr.25421) 804 : cluster 0 pgmap v836: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 630 op/s; 17637/132327 objects degraded (13.328%); 5544/132327 objects misplaced (4.190%); 1.7 MiB/s, 70 objects/s recovering 2023-10-09T09:04:23.654 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:23 smithi016 bash[75179]: cluster 2023-10-09T09:04:21.856455+0000 mgr.y (mgr.25421) 804 : cluster 0 pgmap v836: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 630 op/s; 17637/132327 objects degraded (13.328%); 5544/132327 objects misplaced (4.190%); 1.7 MiB/s, 70 objects/s recovering 2023-10-09T09:04:23.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:23.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:23.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:23.798 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:23 smithi094 bash[59677]: cluster 2023-10-09T09:04:21.856455+0000 mgr.y (mgr.25421) 804 : cluster 0 pgmap v836: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 630 op/s; 17637/132327 objects degraded (13.328%); 5544/132327 objects misplaced (4.190%); 1.7 MiB/s, 70 objects/s recovering 2023-10-09T09:04:23.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:23.922 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:23.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:23.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:23.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:24.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:24.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:24.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:24.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:24.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:24.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:24.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:24.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:24.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:24.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:24.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:24.478 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:24.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:24.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:24.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:24.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:24.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:24.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:24.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:24.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:24.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:24.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:24.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:24.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:24.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:24.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:24.970 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:24.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:25.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:25.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:25.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:25.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:25.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:25.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:25.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:25.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:25.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:25.529 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:25.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:25.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:25.766 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:25.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:25 smithi094 bash[59677]: cluster 2023-10-09T09:04:23.857116+0000 mgr.y (mgr.25421) 805 : cluster 0 pgmap v837: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 585 op/s; 17312/128793 objects degraded (13.442%); 5139/128793 objects misplaced (3.990%); 1.7 MiB/s, 94 objects/s recovering 2023-10-09T09:04:25.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:25 smithi094 bash[59677]: cluster 2023-10-09T09:04:24.552483+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 17637/132327 objects degraded (13.328%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:25.794 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:25.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:25.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:25.929 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:25.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:25.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:25 smithi016 bash[75179]: cluster 2023-10-09T09:04:23.857116+0000 mgr.y (mgr.25421) 805 : cluster 0 pgmap v837: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 585 op/s; 17312/128793 objects degraded (13.442%); 5139/128793 objects misplaced (3.990%); 1.7 MiB/s, 94 objects/s recovering 2023-10-09T09:04:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:25 smithi016 bash[75179]: cluster 2023-10-09T09:04:24.552483+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 17637/132327 objects degraded (13.328%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:26.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:04:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:04:25] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.33.4" 2023-10-09T09:04:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:25 smithi016 bash[77892]: cluster 2023-10-09T09:04:23.857116+0000 mgr.y (mgr.25421) 805 : cluster 0 pgmap v837: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 585 op/s; 17312/128793 objects degraded (13.442%); 5139/128793 objects misplaced (3.990%); 1.7 MiB/s, 94 objects/s recovering 2023-10-09T09:04:26.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:25 smithi016 bash[77892]: cluster 2023-10-09T09:04:24.552483+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 17637/132327 objects degraded (13.328%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:26.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:26.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:26.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:26.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:26.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:26.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:26.250 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:04:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:04:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T09:04:26.250 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T09:04:26.250 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:04:26.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:26.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:26.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:26.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:26.453 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:26.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:26.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:26.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:26.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:26.619 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:26.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:26.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:26.722 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:26.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:26.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:26.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:26.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:26.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:26.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:27.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:27.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:27.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:27.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:27.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:27.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:27.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:27.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:27.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:27 smithi016 bash[75179]: cluster 2023-10-09T09:04:25.857819+0000 mgr.y (mgr.25421) 806 : cluster 0 pgmap v838: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 648 op/s; 17312/124614 objects degraded (13.893%); 7467/124614 objects misplaced (5.992%); 1.6 MiB/s, 86 objects/s recovering 2023-10-09T09:04:27.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:27.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:27.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.931 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:27 smithi016 bash[77892]: cluster 2023-10-09T09:04:25.857819+0000 mgr.y (mgr.25421) 806 : cluster 0 pgmap v838: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 648 op/s; 17312/124614 objects degraded (13.893%); 7467/124614 objects misplaced (5.992%); 1.6 MiB/s, 86 objects/s recovering 2023-10-09T09:04:27.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:27.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:27.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:27 smithi094 bash[59677]: cluster 2023-10-09T09:04:25.857819+0000 mgr.y (mgr.25421) 806 : cluster 0 pgmap v838: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 648 op/s; 17312/124614 objects degraded (13.893%); 7467/124614 objects misplaced (5.992%); 1.6 MiB/s, 86 objects/s recovering 2023-10-09T09:04:28.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:28.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:28.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:28.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:28.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:28.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:28.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:28.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:28.359 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:28.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:28.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:28.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:28.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:28.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:28.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:28.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:28.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:28.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:28.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:28.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:28.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:28.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:28.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:29.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:29.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:29.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:29.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:29.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:29.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:29.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:29.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:29.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:29.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:29.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:29.553 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:29.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:29.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:29.704 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:29 smithi016 bash[75179]: cluster 2023-10-09T09:04:27.858459+0000 mgr.y (mgr.25421) 807 : cluster 0 pgmap v839: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 651 op/s; 17312/120633 objects degraded (14.351%); 6906/120633 objects misplaced (5.725%); 3.2 MiB/s, 108 objects/s recovering 2023-10-09T09:04:29.704 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:29 smithi016 bash[75179]: cluster 2023-10-09T09:04:29.553555+0000 mon.a (mon.0) 1100 : cluster 3 Health check update: Degraded data redundancy: 17312/120633 objects degraded (14.351%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:29 smithi016 bash[77892]: cluster 2023-10-09T09:04:27.858459+0000 mgr.y (mgr.25421) 807 : cluster 0 pgmap v839: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 651 op/s; 17312/120633 objects degraded (14.351%); 6906/120633 objects misplaced (5.725%); 3.2 MiB/s, 108 objects/s recovering 2023-10-09T09:04:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:29 smithi016 bash[77892]: cluster 2023-10-09T09:04:29.553555+0000 mon.a (mon.0) 1100 : cluster 3 Health check update: Degraded data redundancy: 17312/120633 objects degraded (14.351%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:29.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:29.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:29.843 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:29 smithi094 bash[59677]: cluster 2023-10-09T09:04:27.858459+0000 mgr.y (mgr.25421) 807 : cluster 0 pgmap v839: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 651 op/s; 17312/120633 objects degraded (14.351%); 6906/120633 objects misplaced (5.725%); 3.2 MiB/s, 108 objects/s recovering 2023-10-09T09:04:29.844 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:29 smithi094 bash[59677]: cluster 2023-10-09T09:04:29.553555+0000 mon.a (mon.0) 1100 : cluster 3 Health check update: Degraded data redundancy: 17312/120633 objects degraded (14.351%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:29.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:29.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:29.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:29.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:29.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:30.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:30.111 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:30.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:30.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:30.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:30.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:30.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:30.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:30.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:30.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:30.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:30.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:30.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:30.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:30.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:30.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:30.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:30.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:30.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:30.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:30 smithi094 bash[59677]: cluster 2023-10-09T09:04:29.859045+0000 mgr.y (mgr.25421) 808 : cluster 0 pgmap v840: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 528 op/s; 16988/119442 objects degraded (14.223%); 6906/119442 objects misplaced (5.782%); 3.2 MiB/s, 115 objects/s recovering 2023-10-09T09:04:30.753 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:30.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:30.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:30.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:30 smithi016 bash[75179]: cluster 2023-10-09T09:04:29.859045+0000 mgr.y (mgr.25421) 808 : cluster 0 pgmap v840: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 528 op/s; 16988/119442 objects degraded (14.223%); 6906/119442 objects misplaced (5.782%); 3.2 MiB/s, 115 objects/s recovering 2023-10-09T09:04:30.790 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:30 smithi016 bash[77892]: cluster 2023-10-09T09:04:29.859045+0000 mgr.y (mgr.25421) 808 : cluster 0 pgmap v840: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 528 op/s; 16988/119442 objects degraded (14.223%); 6906/119442 objects misplaced (5.782%); 3.2 MiB/s, 115 objects/s recovering 2023-10-09T09:04:30.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:30.880 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:30.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:30.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:31.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:31.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:31.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:31.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:31.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:31.240 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:31.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:31.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:31.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:31.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:31.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:31.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:31.535 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:31.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:31.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:31.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:31.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:04:31.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:31.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:31.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:31.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:31.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:31.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:31.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:31.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:31 smithi016 bash[75179]: audit 2023-10-09T09:04:31.501077+0000 mon.a (mon.0) 1101 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:04:31.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:31 smithi016 bash[77892]: audit 2023-10-09T09:04:31.501077+0000 mon.a (mon.0) 1101 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:04:31.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:31 smithi094 bash[59677]: audit 2023-10-09T09:04:31.501077+0000 mon.a (mon.0) 1101 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:04:32.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:32.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:32.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:32.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:32.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:32.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:32.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:32.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:32.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:32.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:32.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:32.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:32.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:32.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:32.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:32.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:32 smithi016 bash[77892]: cluster 2023-10-09T09:04:31.859782+0000 mgr.y (mgr.25421) 809 : cluster 0 pgmap v841: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 685 op/s; 16988/113802 objects degraded (14.928%); 6545/113802 objects misplaced (5.751%); 3.2 MiB/s, 115 objects/s recovering 2023-10-09T09:04:32.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:32 smithi016 bash[75179]: cluster 2023-10-09T09:04:31.859782+0000 mgr.y (mgr.25421) 809 : cluster 0 pgmap v841: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 685 op/s; 16988/113802 objects degraded (14.928%); 6545/113802 objects misplaced (5.751%); 3.2 MiB/s, 115 objects/s recovering 2023-10-09T09:04:32.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:32.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:32.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:32.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:32.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:32.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:32.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:32.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:32 smithi094 bash[59677]: cluster 2023-10-09T09:04:31.859782+0000 mgr.y (mgr.25421) 809 : cluster 0 pgmap v841: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 685 op/s; 16988/113802 objects degraded (14.928%); 6545/113802 objects misplaced (5.751%); 3.2 MiB/s, 115 objects/s recovering 2023-10-09T09:04:33.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:33.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:33.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:33.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:33.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:33.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:33.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:33.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:33.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:33.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:33.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:33.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:33.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:33.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:33.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:33.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:33.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:33.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:33.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:33.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:33.995 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:33.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:34.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:34.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:34.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:34.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:34.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:34.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:34.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:34.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:34.355 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:34.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:34.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:34.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:34.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:34.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:34.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:34.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:34.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:34.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:34.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:34.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:34.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:34.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:34.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:35.015 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:35.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:34 smithi094 bash[59677]: cluster 2023-10-09T09:04:33.860511+0000 mgr.y (mgr.25421) 810 : cluster 0 pgmap v842: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 16645/111000 objects degraded (14.995%); 6137/111000 objects misplaced (5.529%); 4.8 MiB/s, 160 objects/s recovering 2023-10-09T09:04:35.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:34 smithi094 bash[59677]: audit 2023-10-09T09:04:34.233300+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:35.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:34 smithi094 bash[59677]: cluster 2023-10-09T09:04:34.555401+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 16645/111000 objects degraded (14.995%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:35.040 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:35.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:35.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.152 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:34 smithi016 bash[75179]: cluster 2023-10-09T09:04:33.860511+0000 mgr.y (mgr.25421) 810 : cluster 0 pgmap v842: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 16645/111000 objects degraded (14.995%); 6137/111000 objects misplaced (5.529%); 4.8 MiB/s, 160 objects/s recovering 2023-10-09T09:04:35.152 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:34 smithi016 bash[75179]: audit 2023-10-09T09:04:34.233300+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:35.153 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:34 smithi016 bash[75179]: cluster 2023-10-09T09:04:34.555401+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 16645/111000 objects degraded (14.995%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:35.153 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:34 smithi016 bash[77892]: cluster 2023-10-09T09:04:33.860511+0000 mgr.y (mgr.25421) 810 : cluster 0 pgmap v842: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 16645/111000 objects degraded (14.995%); 6137/111000 objects misplaced (5.529%); 4.8 MiB/s, 160 objects/s recovering 2023-10-09T09:04:35.153 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:34 smithi016 bash[77892]: audit 2023-10-09T09:04:34.233300+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:35.153 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:34 smithi016 bash[77892]: cluster 2023-10-09T09:04:34.555401+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 16645/111000 objects degraded (14.995%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:35.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:35.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:35.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:35.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:35.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:35.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:35.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:35.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:35.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:35.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:35.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:35.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:35.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:35.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:35.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:35.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:04:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:04:35] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-09T09:04:36.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:36.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:36.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:36.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:36.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:36.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:36.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:36.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:36.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:36.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:36.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:36.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:36.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:36.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:36.644 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:36.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:36.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:36.748 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:36.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:36.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:36.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:36.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:36.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:36.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:37.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:37.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:37.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:37.158 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:36 smithi094 bash[59677]: cluster 2023-10-09T09:04:35.864141+0000 mgr.y (mgr.25421) 811 : cluster 0 pgmap v843: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 601 op/s; 16645/107112 objects degraded (15.540%); 5769/107112 objects misplaced (5.386%); 3.7 MiB/s, 116 objects/s recovering 2023-10-09T09:04:37.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:37.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:37.297 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:37.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:37.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:37.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:37.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:37.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:37.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:36 smithi016 bash[75179]: cluster 2023-10-09T09:04:35.864141+0000 mgr.y (mgr.25421) 811 : cluster 0 pgmap v843: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 601 op/s; 16645/107112 objects degraded (15.540%); 5769/107112 objects misplaced (5.386%); 3.7 MiB/s, 116 objects/s recovering 2023-10-09T09:04:37.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:36 smithi016 bash[77892]: cluster 2023-10-09T09:04:35.864141+0000 mgr.y (mgr.25421) 811 : cluster 0 pgmap v843: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 601 op/s; 16645/107112 objects degraded (15.540%); 5769/107112 objects misplaced (5.386%); 3.7 MiB/s, 116 objects/s recovering 2023-10-09T09:04:37.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:37.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:37.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:37.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:37.744 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:37.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:37.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:37.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:37.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:37.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:37.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:38.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:38.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:38.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:38.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:38.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:38.292 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:38.395 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:38.416 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:38.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:38.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:38.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:38.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:38.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:38.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:38.721 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:04:38.730 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:04:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:38 smithi016 bash[75179]: audit 2023-10-09T09:04:37.754591+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:38 smithi016 bash[75179]: audit 2023-10-09T09:04:37.769212+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:38.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:38 smithi016 bash[75179]: cluster 2023-10-09T09:04:37.864868+0000 mgr.y (mgr.25421) 812 : cluster 0 pgmap v844: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 598 op/s; 16645/103041 objects degraded (16.154%); 5250/103041 objects misplaced (5.095%); 5.8 MiB/s, 148 objects/s recovering 2023-10-09T09:04:38.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:38 smithi016 bash[77892]: audit 2023-10-09T09:04:37.754591+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:38.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:38 smithi016 bash[77892]: audit 2023-10-09T09:04:37.769212+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:38.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:38 smithi016 bash[77892]: cluster 2023-10-09T09:04:37.864868+0000 mgr.y (mgr.25421) 812 : cluster 0 pgmap v844: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 598 op/s; 16645/103041 objects degraded (16.154%); 5250/103041 objects misplaced (5.095%); 5.8 MiB/s, 148 objects/s recovering 2023-10-09T09:04:38.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:38.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:38.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:38.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:38.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:38.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:39.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:39.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:39.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:39.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:39.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:38 smithi094 bash[59677]: audit 2023-10-09T09:04:37.754591+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:38 smithi094 bash[59677]: audit 2023-10-09T09:04:37.769212+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:38 smithi094 bash[59677]: cluster 2023-10-09T09:04:37.864868+0000 mgr.y (mgr.25421) 812 : cluster 0 pgmap v844: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 598 op/s; 16645/103041 objects degraded (16.154%); 5250/103041 objects misplaced (5.095%); 5.8 MiB/s, 148 objects/s recovering 2023-10-09T09:04:39.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:39.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:39.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:39.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:39.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:39.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:39.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:39.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:39.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:39.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:39.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:39.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:39.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:39.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:39.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:39.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:39.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:39.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:39.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:39.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:40.009 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:40.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:39 smithi094 bash[59677]: audit 2023-10-09T09:04:38.701753+0000 mgr.y (mgr.25421) 813 : audit 0 from='client.35322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:40.018 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:39 smithi094 bash[59677]: cluster 2023-10-09T09:04:39.556406+0000 mon.a (mon.0) 1106 : cluster 3 Health check update: Degraded data redundancy: 16645/103041 objects degraded (16.154%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:40.032 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:40.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:40.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:39 smithi016 bash[75179]: audit 2023-10-09T09:04:38.701753+0000 mgr.y (mgr.25421) 813 : audit 0 from='client.35322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:40.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:39 smithi016 bash[75179]: cluster 2023-10-09T09:04:39.556406+0000 mon.a (mon.0) 1106 : cluster 3 Health check update: Degraded data redundancy: 16645/103041 objects degraded (16.154%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:40.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:39 smithi016 bash[77892]: audit 2023-10-09T09:04:38.701753+0000 mgr.y (mgr.25421) 813 : audit 0 from='client.35322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:40.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:39 smithi016 bash[77892]: cluster 2023-10-09T09:04:39.556406+0000 mon.a (mon.0) 1106 : cluster 3 Health check update: Degraded data redundancy: 16645/103041 objects degraded (16.154%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:40.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:40.162 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:40.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:40.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:40.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:40.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:40.395 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:40.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:40.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:40.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:40.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:40.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:40.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:40.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:40.796 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:40 smithi016 bash[75179]: cluster 2023-10-09T09:04:39.865381+0000 mgr.y (mgr.25421) 814 : cluster 0 pgmap v845: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 524 op/s; 16266/101742 objects degraded (15.987%); 5250/101742 objects misplaced (5.160%); 3.7 MiB/s, 146 objects/s recovering 2023-10-09T09:04:40.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:40 smithi016 bash[77892]: cluster 2023-10-09T09:04:39.865381+0000 mgr.y (mgr.25421) 814 : cluster 0 pgmap v845: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 524 op/s; 16266/101742 objects degraded (15.987%); 5250/101742 objects misplaced (5.160%); 3.7 MiB/s, 146 objects/s recovering 2023-10-09T09:04:40.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:40.920 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:40.946 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:40.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:40.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:41.049 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:41.061 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:40 smithi094 bash[59677]: cluster 2023-10-09T09:04:39.865381+0000 mgr.y (mgr.25421) 814 : cluster 0 pgmap v845: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 524 op/s; 16266/101742 objects degraded (15.987%); 5250/101742 objects misplaced (5.160%); 3.7 MiB/s, 146 objects/s recovering 2023-10-09T09:04:41.088 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:41.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:41.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:41.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:41.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:41.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:41.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:41.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:41.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:41.455 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:41.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:41.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:41.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:41.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:41.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:41.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:41.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:41.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:41.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:41.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:41.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:41.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:41.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:41.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:42.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:42.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:42.107 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:42.135 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:42.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:42.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:42.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:42.313 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":844,"stamp":"2023-10-09T09:04:37.864356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2249922488,"num_objects":34347,"num_object_clones":0,"num_object_copies":103041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16645,"num_objects_misplaced":5250,"num_objects_unfound":0,"num_objects_dirty":34347,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":142411,"num_write_kb":5649550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16259,"num_bytes_recovered":964640078,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85604,"ondisk_log_size":85604,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":283,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":9378096,"kb_used_data":7208776,"kb_used_omap":117,"kb_used_meta":2168970,"kb_avail":459347664,"statfs":{"total":479975178240,"available":470372007936,"internally_reserved":0,"allocated":7381786624,"data_stored":7378403018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":2221025938},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,6,9,18,5,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-471269376,"num_objects":-7191,"num_object_clones":0,"num_object_copies":-21573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-667,"num_objects_misplaced":-2217,"num_objects_unfound":0,"num_objects_dirty":-7191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7191,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1781,"num_bytes_recovered":73007104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.007057"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":63,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914767+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:04:07.914767+0000","last_peered":"2023-10-09T09:04:07.914767+0000","last_clean":"2023-10-09T09:04:07.914767+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:04:07.914767+0000","last_undegraded":"2023-10-09T09:04:07.914767+0000","last_fullsized":"2023-10-09T09:04:07.914767+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":211,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928148+0000","last_change":"2023-10-09T09:02:02.625626+0000","last_active":"2023-10-09T09:04:07.928148+0000","last_peered":"2023-10-09T09:04:07.928148+0000","last_clean":"2023-10-09T09:04:07.928148+0000","last_became_active":"2023-10-09T09:02:02.625312+0000","last_became_peered":"2023-10-09T09:02:02.625312+0000","last_unstale":"2023-10-09T09:04:07.928148+0000","last_undegraded":"2023-10-09T09:04:07.928148+0000","last_fullsized":"2023-10-09T09:04:07.928148+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:01.640943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":165,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:12.941196+0000","last_change":"2023-10-09T09:02:02.626274+0000","last_active":"2023-10-09T09:04:12.941196+0000","last_peered":"2023-10-09T09:04:12.941196+0000","last_clean":"2023-10-09T09:04:12.941196+0000","last_became_active":"2023-10-09T09:02:02.625973+0000","last_became_peered":"2023-10-09T09:02:02.625973+0000","last_unstale":"2023-10-09T09:04:12.941196+0000","last_undegraded":"2023-10-09T09:04:12.941196+0000","last_fullsized":"2023-10-09T09:04:12.941196+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:44.223794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":212,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.940425+0000","last_change":"2023-10-09T09:03:19.554441+0000","last_active":"2023-10-09T09:04:07.940425+0000","last_peered":"2023-10-09T09:04:07.940425+0000","last_clean":"2023-10-09T09:04:07.940425+0000","last_became_active":"2023-10-09T09:03:19.553967+0000","last_became_peered":"2023-10-09T09:03:19.553967+0000","last_unstale":"2023-10-09T09:04:07.940425+0000","last_undegraded":"2023-10-09T09:04:07.940425+0000","last_fullsized":"2023-10-09T09:04:07.940425+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:26.065993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":201,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:10.432769+0000","last_change":"2023-10-09T09:02:02.656821+0000","last_active":"2023-10-09T09:04:10.432769+0000","last_peered":"2023-10-09T09:04:10.432769+0000","last_clean":"2023-10-09T09:04:10.432769+0000","last_became_active":"2023-10-09T09:02:02.656523+0000","last_became_peered":"2023-10-09T09:02:02.656523+0000","last_unstale":"2023-10-09T09:04:10.432769+0000","last_undegraded":"2023-10-09T09:04:10.432769+0000","last_fullsized":"2023-10-09T09:04:10.432769+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:21.635518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":77,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.918566+0000","last_change":"2023-10-09T09:03:19.582769+0000","last_active":"2023-10-09T09:04:07.918566+0000","last_peered":"2023-10-09T09:04:07.918566+0000","last_clean":"2023-10-09T09:04:07.918566+0000","last_became_active":"2023-10-09T09:03:19.577023+0000","last_became_peered":"2023-10-09T09:03:19.577023+0000","last_unstale":"2023-10-09T09:04:07.918566+0000","last_undegraded":"2023-10-09T09:04:07.918566+0000","last_fullsized":"2023-10-09T09:04:07.918566+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":174,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.924291+0000","last_change":"2023-10-09T09:01:36.280403+0000","last_active":"2023-10-09T09:04:07.924291+0000","last_peered":"2023-10-09T09:04:07.924291+0000","last_clean":"2023-10-09T09:04:07.924291+0000","last_became_active":"2023-10-09T09:01:36.280070+0000","last_became_peered":"2023-10-09T09:01:36.280070+0000","last_unstale":"2023-10-09T09:04:07.924291+0000","last_undegraded":"2023-10-09T09:04:07.924291+0000","last_fullsized":"2023-10-09T09:04:07.924291+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:33.796899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":212,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928027+0000","last_change":"2023-10-09T09:02:02.630162+0000","last_active":"2023-10-09T09:04:07.928027+0000","last_peered":"2023-10-09T09:04:07.928027+0000","last_clean":"2023-10-09T09:04:07.928027+0000","last_became_active":"2023-10-09T09:02:02.629768+0000","last_became_peered":"2023-10-09T09:02:02.629768+0000","last_unstale":"2023-10-09T09:04:07.928027+0000","last_undegraded":"2023-10-09T09:04:07.928027+0000","last_fullsized":"2023-10-09T09:04:07.928027+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:36.859890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":18,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.929039+0000","last_change":"2023-10-09T09:03:19.562130+0000","last_active":"2023-10-09T09:04:07.929039+0000","last_peered":"2023-10-09T09:04:07.929039+0000","last_clean":"2023-10-09T09:04:07.929039+0000","last_became_active":"2023-10-09T09:03:19.557757+0000","last_became_peered":"2023-10-09T09:03:19.557757+0000","last_unstale":"2023-10-09T09:04:07.929039+0000","last_undegraded":"2023-10-09T09:04:07.929039+0000","last_fullsized":"2023-10-09T09:04:07.929039+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:46.546761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.932269+0000","last_change":"2023-10-09T09:01:36.279247+0000","last_active":"2023-10-09T09:04:07.932269+0000","last_peered":"2023-10-09T09:04:07.932269+0000","last_clean":"2023-10-09T09:04:07.932269+0000","last_became_active":"2023-10-09T09:01:36.278376+0000","last_became_peered":"2023-10-09T09:01:36.278376+0000","last_unstale":"2023-10-09T09:04:07.932269+0000","last_undegraded":"2023-10-09T09:04:07.932269+0000","last_fullsized":"2023-10-09T09:04:07.932269+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:19.166567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":158,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928251+0000","last_change":"2023-10-09T08:59:53.544302+0000","last_active":"2023-10-09T09:04:07.928251+0000","last_peered":"2023-10-09T09:04:07.928251+0000","last_clean":"2023-10-09T09:04:07.928251+0000","last_became_active":"2023-10-09T08:59:53.531404+0000","last_became_peered":"2023-10-09T08:59:53.531404+0000","last_unstale":"2023-10-09T09:04:07.928251+0000","last_undegraded":"2023-10-09T09:04:07.928251+0000","last_fullsized":"2023-10-09T09:04:07.928251+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:48.670745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.917789+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:04:07.917789+0000","last_peered":"2023-10-09T09:04:07.917789+0000","last_clean":"2023-10-09T09:04:07.917789+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:04:07.917789+0000","last_undegraded":"2023-10-09T09:04:07.917789+0000","last_fullsized":"2023-10-09T09:04:07.917789+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":83,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.916184+0000","last_change":"2023-10-09T09:00:31.867387+0000","last_active":"2023-10-09T09:04:07.916184+0000","last_peered":"2023-10-09T09:04:07.916184+0000","last_clean":"2023-10-09T09:04:07.916184+0000","last_became_active":"2023-10-09T09:00:31.859752+0000","last_became_peered":"2023-10-09T09:00:31.859752+0000","last_unstale":"2023-10-09T09:04:07.916184+0000","last_undegraded":"2023-10-09T09:04:07.916184+0000","last_fullsized":"2023-10-09T09:04:07.916184+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914914+0000","last_change":"2023-10-09T09:02:02.630025+0000","last_active":"2023-10-09T09:04:07.914914+0000","last_peered":"2023-10-09T09:04:07.914914+0000","last_clean":"2023-10-09T09:04:07.914914+0000","last_became_active":"2023-10-09T09:02:02.629474+0000","last_became_peered":"2023-10-09T09:02:02.629474+0000","last_unstale":"2023-10-09T09:04:07.914914+0000","last_undegraded":"2023-10-09T09:04:07.914914+0000","last_fullsized":"2023-10-09T09:04:07.914914+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":75,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.915017+0000","last_change":"2023-10-09T09:00:31.959283+0000","last_active":"2023-10-09T09:04:07.915017+0000","last_peered":"2023-10-09T09:04:07.915017+0000","last_clean":"2023-10-09T09:04:07.915017+0000","last_became_active":"2023-10-09T09:00:31.861638+0000","last_became_peered":"2023-10-09T09:00:31.861638+0000","last_unstale":"2023-10-09T09:04:07.915017+0000","last_undegraded":"2023-10-09T09:04:07.915017+0000","last_fullsized":"2023-10-09T09:04:07.915017+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":194,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928587+0000","last_change":"2023-10-09T09:02:02.636035+0000","last_active":"2023-10-09T09:04:07.928587+0000","last_peered":"2023-10-09T09:04:07.928587+0000","last_clean":"2023-10-09T09:04:07.928587+0000","last_became_active":"2023-10-09T09:02:02.635621+0000","last_became_peered":"2023-10-09T09:02:02.635621+0000","last_unstale":"2023-10-09T09:04:07.928587+0000","last_undegraded":"2023-10-09T09:04:07.928587+0000","last_fullsized":"2023-10-09T09:04:07.928587+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:46.294650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":202,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:12.955816+0000","last_change":"2023-10-09T09:03:19.564077+0000","last_active":"2023-10-09T09:04:12.955816+0000","last_peered":"2023-10-09T09:04:12.955816+0000","last_clean":"2023-10-09T09:04:12.955816+0000","last_became_active":"2023-10-09T09:03:19.563764+0000","last_became_peered":"2023-10-09T09:03:19.563764+0000","last_unstale":"2023-10-09T09:04:12.955816+0000","last_undegraded":"2023-10-09T09:04:12.955816+0000","last_fullsized":"2023-10-09T09:04:12.955816+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:38.043450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928063+0000","last_change":"2023-10-09T09:01:36.279054+0000","last_active":"2023-10-09T09:04:07.928063+0000","last_peered":"2023-10-09T09:04:07.928063+0000","last_clean":"2023-10-09T09:04:07.928063+0000","last_became_active":"2023-10-09T09:01:36.278755+0000","last_became_peered":"2023-10-09T09:01:36.278755+0000","last_unstale":"2023-10-09T09:04:07.928063+0000","last_undegraded":"2023-10-09T09:04:07.928063+0000","last_fullsized":"2023-10-09T09:04:07.928063+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:09.089065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":63,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914335+0000","last_change":"2023-10-09T09:00:31.851267+0000","last_active":"2023-10-09T09:04:07.914335+0000","last_peered":"2023-10-09T09:04:07.914335+0000","last_clean":"2023-10-09T09:04:07.914335+0000","last_became_active":"2023-10-09T09:00:31.850918+0000","last_became_peered":"2023-10-09T09:00:31.850918+0000","last_unstale":"2023-10-09T09:04:07.914335+0000","last_undegraded":"2023-10-09T09:04:07.914335+0000","last_fullsized":"2023-10-09T09:04:07.914335+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.940501+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T09:04:07.940501+0000","last_peered":"2023-10-09T09:04:07.940501+0000","last_clean":"2023-10-09T09:04:07.940501+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T09:04:07.940501+0000","last_undegraded":"2023-10-09T09:04:07.940501+0000","last_fullsized":"2023-10-09T09:04:07.940501+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":79,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.918077+0000","last_change":"2023-10-09T09:01:36.301852+0000","last_active":"2023-10-09T09:04:07.918077+0000","last_peered":"2023-10-09T09:04:07.918077+0000","last_clean":"2023-10-09T09:04:07.918077+0000","last_became_active":"2023-10-09T09:01:36.284416+0000","last_became_peered":"2023-10-09T09:01:36.284416+0000","last_unstale":"2023-10-09T09:04:07.918077+0000","last_undegraded":"2023-10-09T09:04:07.918077+0000","last_fullsized":"2023-10-09T09:04:07.918077+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.933633+0000","last_change":"2023-10-09T09:03:19.575211+0000","last_active":"2023-10-09T09:04:07.933633+0000","last_peered":"2023-10-09T09:04:07.933633+0000","last_clean":"2023-10-09T09:04:07.933633+0000","last_became_active":"2023-10-09T09:03:19.574829+0000","last_became_peered":"2023-10-09T09:03:19.574829+0000","last_unstale":"2023-10-09T09:04:07.933633+0000","last_undegraded":"2023-10-09T09:04:07.933633+0000","last_fullsized":"2023-10-09T09:04:07.933633+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:14.763790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"201'8831","reported_seq":8867,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:35.259684+0000","last_change":"2023-10-09T09:01:36.277698+0000","last_active":"2023-10-09T09:04:35.259684+0000","last_peered":"2023-10-09T09:04:35.259684+0000","last_clean":"2023-10-09T09:04:35.259684+0000","last_became_active":"2023-10-09T09:01:36.276985+0000","last_became_peered":"2023-10-09T09:01:36.276985+0000","last_unstale":"2023-10-09T09:04:35.259684+0000","last_undegraded":"2023-10-09T09:04:35.259684+0000","last_fullsized":"2023-10-09T09:04:35.259684+0000","mapping_epoch":176,"log_start":"185'3300","ondisk_log_start":"185'3300","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5531,"log_dups_size":0,"ondisk_log_size":5531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144637952,"num_objects":2207,"num_object_clones":0,"num_object_copies":6621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8831,"num_write_kb":353216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":84,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914978+0000","last_change":"2023-10-09T09:03:19.626156+0000","last_active":"2023-10-09T09:04:07.914978+0000","last_peered":"2023-10-09T09:04:07.914978+0000","last_clean":"2023-10-09T09:04:07.914978+0000","last_became_active":"2023-10-09T09:03:19.575061+0000","last_became_peered":"2023-10-09T09:03:19.575061+0000","last_unstale":"2023-10-09T09:04:07.914978+0000","last_undegraded":"2023-10-09T09:04:07.914978+0000","last_fullsized":"2023-10-09T09:04:07.914978+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":178,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.917735+0000","last_change":"2023-10-09T09:03:19.582540+0000","last_active":"2023-10-09T09:04:07.917735+0000","last_peered":"2023-10-09T09:04:07.917735+0000","last_clean":"2023-10-09T09:04:07.917735+0000","last_became_active":"2023-10-09T09:03:19.577044+0000","last_became_peered":"2023-10-09T09:03:19.577044+0000","last_unstale":"2023-10-09T09:04:07.917735+0000","last_undegraded":"2023-10-09T09:04:07.917735+0000","last_fullsized":"2023-10-09T09:04:07.917735+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"201'8737","reported_seq":8774,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:36.021784+0000","last_change":"2023-10-09T09:01:16.814785+0000","last_active":"2023-10-09T09:04:36.021784+0000","last_peered":"2023-10-09T09:04:36.021784+0000","last_clean":"2023-10-09T09:04:36.021784+0000","last_became_active":"2023-10-09T09:01:16.814404+0000","last_became_peered":"2023-10-09T09:01:16.814404+0000","last_unstale":"2023-10-09T09:04:36.021784+0000","last_undegraded":"2023-10-09T09:04:36.021784+0000","last_fullsized":"2023-10-09T09:04:36.021784+0000","mapping_epoch":172,"log_start":"186'3500","ondisk_log_start":"186'3500","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5237,"log_dups_size":0,"ondisk_log_size":5237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137166848,"num_objects":2093,"num_object_clones":0,"num_object_copies":6279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8737,"num_write_kb":346560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":71,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914655+0000","last_change":"2023-10-09T09:02:02.724809+0000","last_active":"2023-10-09T09:04:07.914655+0000","last_peered":"2023-10-09T09:04:07.914655+0000","last_clean":"2023-10-09T09:04:07.914655+0000","last_became_active":"2023-10-09T09:02:02.666202+0000","last_became_peered":"2023-10-09T09:02:02.666202+0000","last_unstale":"2023-10-09T09:04:07.914655+0000","last_undegraded":"2023-10-09T09:04:07.914655+0000","last_fullsized":"2023-10-09T09:04:07.914655+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":81,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914241+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:04:07.914241+0000","last_peered":"2023-10-09T09:04:07.914241+0000","last_clean":"2023-10-09T09:04:07.914241+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:04:07.914241+0000","last_undegraded":"2023-10-09T09:04:07.914241+0000","last_fullsized":"2023-10-09T09:04:07.914241+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"201'9019","reported_seq":9058,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:37.522593+0000","last_change":"2023-10-09T09:01:16.948996+0000","last_active":"2023-10-09T09:04:37.522593+0000","last_peered":"2023-10-09T09:04:37.522593+0000","last_clean":"2023-10-09T09:04:37.522593+0000","last_became_active":"2023-10-09T09:01:16.948583+0000","last_became_peered":"2023-10-09T09:01:16.948583+0000","last_unstale":"2023-10-09T09:04:37.522593+0000","last_undegraded":"2023-10-09T09:04:37.522593+0000","last_fullsized":"2023-10-09T09:04:37.522593+0000","mapping_epoch":172,"log_start":"185'3300","ondisk_log_start":"185'3300","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:21.355627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136249344,"num_objects":2079,"num_object_clones":0,"num_object_copies":6237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9019,"num_write_kb":355136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":110,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.918752+0000","last_change":"2023-10-09T09:01:36.305615+0000","last_active":"2023-10-09T09:04:07.918752+0000","last_peered":"2023-10-09T09:04:07.918752+0000","last_clean":"2023-10-09T09:04:07.918752+0000","last_became_active":"2023-10-09T09:01:36.292828+0000","last_became_peered":"2023-10-09T09:01:36.292828+0000","last_unstale":"2023-10-09T09:04:07.918752+0000","last_undegraded":"2023-10-09T09:04:07.918752+0000","last_fullsized":"2023-10-09T09:04:07.918752+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":77,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.917887+0000","last_change":"2023-10-09T09:03:19.564369+0000","last_active":"2023-10-09T09:04:07.917887+0000","last_peered":"2023-10-09T09:04:07.917887+0000","last_clean":"2023-10-09T09:04:07.917887+0000","last_became_active":"2023-10-09T09:03:19.563989+0000","last_became_peered":"2023-10-09T09:03:19.563989+0000","last_unstale":"2023-10-09T09:04:07.917887+0000","last_undegraded":"2023-10-09T09:04:07.917887+0000","last_fullsized":"2023-10-09T09:04:07.917887+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"201'9122","reported_seq":12678,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:35.501986+0000","last_change":"2023-10-09T09:02:52.356438+0000","last_active":"2023-10-09T09:04:35.501986+0000","last_peered":"2023-10-09T09:04:35.501986+0000","last_clean":"2023-10-09T09:02:01.626664+0000","last_became_active":"2023-10-09T09:02:52.345464+0000","last_became_peered":"2023-10-09T09:02:52.345464+0000","last_unstale":"2023-10-09T09:04:35.501986+0000","last_undegraded":"2023-10-09T09:02:52.049405+0000","last_fullsized":"2023-10-09T09:02:51.903806+0000","mapping_epoch":190,"log_start":"189'4486","ondisk_log_start":"189'4486","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:38.670622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141033472,"num_objects":2152,"num_object_clones":0,"num_object_copies":6456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5032,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9122,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1751,"num_bytes_recovered":114753536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"3,4,6","objects":1547},{"shards":"3,6","objects":-2880}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928716+0000","last_change":"2023-10-09T09:02:02.664617+0000","last_active":"2023-10-09T09:04:07.928716+0000","last_peered":"2023-10-09T09:04:07.928716+0000","last_clean":"2023-10-09T09:04:07.928716+0000","last_became_active":"2023-10-09T09:02:02.664305+0000","last_became_peered":"2023-10-09T09:02:02.664305+0000","last_unstale":"2023-10-09T09:04:07.928716+0000","last_undegraded":"2023-10-09T09:04:07.928716+0000","last_fullsized":"2023-10-09T09:04:07.928716+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:14.520177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":193,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.923325+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:04:07.923325+0000","last_peered":"2023-10-09T09:04:07.923325+0000","last_clean":"2023-10-09T09:04:07.923325+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:04:07.923325+0000","last_undegraded":"2023-10-09T09:04:07.923325+0000","last_fullsized":"2023-10-09T09:04:07.923325+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"201'8736","reported_seq":13688,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:35.519087+0000","last_change":"2023-10-09T09:04:01.588490+0000","last_active":"2023-10-09T09:04:35.519087+0000","last_peered":"2023-10-09T09:04:35.519087+0000","last_clean":"2023-10-09T09:04:35.519087+0000","last_became_active":"2023-10-09T09:04:01.574284+0000","last_became_peered":"2023-10-09T09:04:01.574284+0000","last_unstale":"2023-10-09T09:04:35.519087+0000","last_undegraded":"2023-10-09T09:04:35.519087+0000","last_fullsized":"2023-10-09T09:04:35.519087+0000","mapping_epoch":198,"log_start":"187'4032","ondisk_log_start":"187'4032","created":172,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4704,"log_dups_size":0,"ondisk_log_size":4704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:13.836976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140509184,"num_objects":2144,"num_object_clones":0,"num_object_copies":6432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8736,"num_write_kb":348160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2441,"num_bytes_recovered":155975680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":201,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.927760+0000","last_change":"2023-10-09T08:59:53.642095+0000","last_active":"2023-10-09T09:04:07.927760+0000","last_peered":"2023-10-09T09:04:07.927760+0000","last_clean":"2023-10-09T09:04:07.927760+0000","last_became_active":"2023-10-09T08:59:53.531588+0000","last_became_peered":"2023-10-09T08:59:53.531588+0000","last_unstale":"2023-10-09T09:04:07.927760+0000","last_undegraded":"2023-10-09T09:04:07.927760+0000","last_fullsized":"2023-10-09T09:04:07.927760+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.878230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":63,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.916187+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:04:07.916187+0000","last_peered":"2023-10-09T09:04:07.916187+0000","last_clean":"2023-10-09T09:04:07.916187+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:04:07.916187+0000","last_undegraded":"2023-10-09T09:04:07.916187+0000","last_fullsized":"2023-10-09T09:04:07.916187+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"201'8966","reported_seq":16716,"reported_epoch":201,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T09:04:37.587882+0000","last_change":"2023-10-09T09:04:20.762537+0000","last_active":"2023-10-09T09:04:37.587882+0000","last_peered":"2023-10-09T09:04:37.587882+0000","last_clean":"2023-10-09T09:02:01.580217+0000","last_became_active":"2023-10-09T09:03:20.662001+0000","last_became_peered":"2023-10-09T09:03:20.662001+0000","last_unstale":"2023-10-09T09:04:37.587882+0000","last_undegraded":"2023-10-09T09:04:37.587882+0000","last_fullsized":"2023-10-09T09:04:37.587882+0000","mapping_epoch":194,"log_start":"185'3300","ondisk_log_start":"185'3300","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:29.131956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136708096,"num_objects":2086,"num_object_clones":0,"num_object_copies":6258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":921,"num_objects_unfound":0,"num_objects_dirty":2086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8966,"num_write_kb":353664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3842,"num_bytes_recovered":251789312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,6],"avail_no_missing":["0","1","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":2086}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.917888+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:04:07.917888+0000","last_peered":"2023-10-09T09:04:07.917888+0000","last_clean":"2023-10-09T09:04:07.917888+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:04:07.917888+0000","last_undegraded":"2023-10-09T09:04:07.917888+0000","last_fullsized":"2023-10-09T09:04:07.917888+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":63,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.915145+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:04:07.915145+0000","last_peered":"2023-10-09T09:04:07.915145+0000","last_clean":"2023-10-09T09:04:07.915145+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:04:07.915145+0000","last_undegraded":"2023-10-09T09:04:07.915145+0000","last_fullsized":"2023-10-09T09:04:07.915145+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"201'9019","reported_seq":9074,"reported_epoch":201,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:04:35.343750+0000","last_change":"2023-10-09T09:03:20.697136+0000","last_active":"2023-10-09T09:04:35.343750+0000","last_peered":"2023-10-09T09:04:35.343750+0000","last_clean":"2023-10-09T09:03:19.287407+0000","last_became_active":"2023-10-09T09:03:20.665776+0000","last_became_peered":"2023-10-09T09:03:20.665776+0000","last_unstale":"2023-10-09T09:04:35.343750+0000","last_undegraded":"2023-10-09T09:04:35.343750+0000","last_fullsized":"2023-10-09T09:04:35.343750+0000","mapping_epoch":194,"log_start":"186'3500","ondisk_log_start":"186'3500","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143196160,"num_objects":2185,"num_object_clones":0,"num_object_copies":6555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2185,"num_objects_unfound":0,"num_objects_dirty":2185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9019,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":2185}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914213+0000","last_change":"2023-10-09T09:03:19.563963+0000","last_active":"2023-10-09T09:04:07.914213+0000","last_peered":"2023-10-09T09:04:07.914213+0000","last_clean":"2023-10-09T09:04:07.914213+0000","last_became_active":"2023-10-09T09:03:19.563629+0000","last_became_peered":"2023-10-09T09:03:19.563629+0000","last_unstale":"2023-10-09T09:04:07.914213+0000","last_undegraded":"2023-10-09T09:04:07.914213+0000","last_fullsized":"2023-10-09T09:04:07.914213+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914573+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:04:07.914573+0000","last_peered":"2023-10-09T09:04:07.914573+0000","last_clean":"2023-10-09T09:04:07.914573+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:04:07.914573+0000","last_undegraded":"2023-10-09T09:04:07.914573+0000","last_fullsized":"2023-10-09T09:04:07.914573+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"201'8772","reported_seq":13688,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:35.997956+0000","last_change":"2023-10-09T09:02:38.910227+0000","last_active":"2023-10-09T09:04:35.997956+0000","last_peered":"2023-10-09T09:04:35.997956+0000","last_clean":"2023-10-09T09:04:35.997956+0000","last_became_active":"2023-10-09T09:02:38.903339+0000","last_became_peered":"2023-10-09T09:02:38.903339+0000","last_unstale":"2023-10-09T09:04:35.997956+0000","last_undegraded":"2023-10-09T09:04:35.997956+0000","last_fullsized":"2023-10-09T09:04:35.997956+0000","mapping_epoch":188,"log_start":"186'3500","ondisk_log_start":"186'3500","created":172,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139591680,"num_objects":2130,"num_object_clones":0,"num_object_copies":6390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8772,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2426,"num_bytes_recovered":158990336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":18,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.927400+0000","last_change":"2023-10-09T09:03:19.572389+0000","last_active":"2023-10-09T09:04:07.927400+0000","last_peered":"2023-10-09T09:04:07.927400+0000","last_clean":"2023-10-09T09:04:07.927400+0000","last_became_active":"2023-10-09T09:03:19.572091+0000","last_became_peered":"2023-10-09T09:03:19.572091+0000","last_unstale":"2023-10-09T09:04:07.927400+0000","last_undegraded":"2023-10-09T09:04:07.927400+0000","last_fullsized":"2023-10-09T09:04:07.927400+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:27.702718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":156,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:13.064369+0000","last_change":"2023-10-09T09:02:02.636996+0000","last_active":"2023-10-09T09:04:13.064369+0000","last_peered":"2023-10-09T09:04:13.064369+0000","last_clean":"2023-10-09T09:04:13.064369+0000","last_became_active":"2023-10-09T09:02:02.636702+0000","last_became_peered":"2023-10-09T09:02:02.636702+0000","last_unstale":"2023-10-09T09:04:13.064369+0000","last_undegraded":"2023-10-09T09:04:13.064369+0000","last_fullsized":"2023-10-09T09:04:13.064369+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:45.392645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"201'8784","reported_seq":8823,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:37.598982+0000","last_change":"2023-10-09T09:01:16.949929+0000","last_active":"2023-10-09T09:04:37.598982+0000","last_peered":"2023-10-09T09:04:37.598982+0000","last_clean":"2023-10-09T09:04:37.598982+0000","last_became_active":"2023-10-09T09:01:16.949317+0000","last_became_peered":"2023-10-09T09:01:16.949317+0000","last_unstale":"2023-10-09T09:04:37.598982+0000","last_undegraded":"2023-10-09T09:04:37.598982+0000","last_fullsized":"2023-10-09T09:04:37.598982+0000","mapping_epoch":172,"log_start":"185'3100","ondisk_log_start":"185'3100","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:27.017754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135135232,"num_objects":2062,"num_object_clones":0,"num_object_copies":6186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8784,"num_write_kb":347072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":143,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.922918+0000","last_change":"2023-10-09T09:03:19.556450+0000","last_active":"2023-10-09T09:04:07.922918+0000","last_peered":"2023-10-09T09:04:07.922918+0000","last_clean":"2023-10-09T09:04:07.922918+0000","last_became_active":"2023-10-09T09:03:19.556190+0000","last_became_peered":"2023-10-09T09:03:19.556190+0000","last_unstale":"2023-10-09T09:04:07.922918+0000","last_undegraded":"2023-10-09T09:04:07.922918+0000","last_fullsized":"2023-10-09T09:04:07.922918+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:23.704455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928346+0000","last_change":"2023-10-09T09:01:36.285844+0000","last_active":"2023-10-09T09:04:07.928346+0000","last_peered":"2023-10-09T09:04:07.928346+0000","last_clean":"2023-10-09T09:04:07.928346+0000","last_became_active":"2023-10-09T09:01:36.285525+0000","last_became_peered":"2023-10-09T09:01:36.285525+0000","last_unstale":"2023-10-09T09:04:07.928346+0000","last_undegraded":"2023-10-09T09:04:07.928346+0000","last_fullsized":"2023-10-09T09:04:07.928346+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:05.624756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":180,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914589+0000","last_change":"2023-10-09T09:02:52.021749+0000","last_active":"2023-10-09T09:04:07.914589+0000","last_peered":"2023-10-09T09:04:07.914589+0000","last_clean":"2023-10-09T09:04:07.914589+0000","last_became_active":"2023-10-09T09:02:52.021416+0000","last_became_peered":"2023-10-09T09:02:52.021416+0000","last_unstale":"2023-10-09T09:04:07.914589+0000","last_undegraded":"2023-10-09T09:04:07.914589+0000","last_fullsized":"2023-10-09T09:04:07.914589+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"201'8897","reported_seq":8942,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:35.320493+0000","last_change":"2023-10-09T09:02:53.102143+0000","last_active":"2023-10-09T09:04:35.320493+0000","last_peered":"2023-10-09T09:04:35.320493+0000","last_clean":"2023-10-09T09:02:50.898920+0000","last_became_active":"2023-10-09T09:02:53.090937+0000","last_became_peered":"2023-10-09T09:02:53.090937+0000","last_unstale":"2023-10-09T09:04:35.320493+0000","last_undegraded":"2023-10-09T09:02:52.940746+0000","last_fullsized":"2023-10-09T09:02:52.912781+0000","mapping_epoch":191,"log_start":"187'4201","ondisk_log_start":"187'4201","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4696,"log_dups_size":0,"ondisk_log_size":4696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:08.317143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138608640,"num_objects":2115,"num_object_clones":0,"num_object_copies":6345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8897,"num_write_kb":352384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3","objects":-2789},{"shards":"0,3,4","objects":1513}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":168,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.917778+0000","last_change":"2023-10-09T09:03:19.564803+0000","last_active":"2023-10-09T09:04:07.917778+0000","last_peered":"2023-10-09T09:04:07.917778+0000","last_clean":"2023-10-09T09:04:07.917778+0000","last_became_active":"2023-10-09T09:03:19.564520+0000","last_became_peered":"2023-10-09T09:03:19.564520+0000","last_unstale":"2023-10-09T09:04:07.917778+0000","last_undegraded":"2023-10-09T09:04:07.917778+0000","last_fullsized":"2023-10-09T09:04:07.917778+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":63,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.915039+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:04:07.915039+0000","last_peered":"2023-10-09T09:04:07.915039+0000","last_clean":"2023-10-09T09:04:07.915039+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:04:07.915039+0000","last_undegraded":"2023-10-09T09:04:07.915039+0000","last_fullsized":"2023-10-09T09:04:07.915039+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":167,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.926033+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:04:07.926033+0000","last_peered":"2023-10-09T09:04:07.926033+0000","last_clean":"2023-10-09T09:04:07.926033+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:04:07.926033+0000","last_undegraded":"2023-10-09T09:04:07.926033+0000","last_fullsized":"2023-10-09T09:04:07.926033+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"201'8938","reported_seq":14060,"reported_epoch":201,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:04:35.307165+0000","last_change":"2023-10-09T09:04:20.760516+0000","last_active":"2023-10-09T09:04:35.307165+0000","last_peered":"2023-10-09T09:04:35.307165+0000","last_clean":"2023-10-09T09:02:01.574862+0000","last_became_active":"2023-10-09T09:03:20.671903+0000","last_became_peered":"2023-10-09T09:03:20.671903+0000","last_unstale":"2023-10-09T09:04:35.307165+0000","last_undegraded":"2023-10-09T09:04:35.307165+0000","last_fullsized":"2023-10-09T09:04:35.307165+0000","mapping_epoch":194,"log_start":"186'3400","ondisk_log_start":"186'3400","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140509184,"num_objects":2144,"num_object_clones":0,"num_object_copies":6432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2144,"num_objects_unfound":0,"num_objects_dirty":2144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8938,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2531,"num_bytes_recovered":154402816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[4,2,0],"avail_no_missing":["4","0","1","2"],"object_location_counts":[{"shards":"0,2,4","objects":2144}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":225,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.925883+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:04:07.925883+0000","last_peered":"2023-10-09T09:04:07.925883+0000","last_clean":"2023-10-09T09:04:07.925883+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:04:07.925883+0000","last_undegraded":"2023-10-09T09:04:07.925883+0000","last_fullsized":"2023-10-09T09:04:07.925883+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":161,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.917845+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:04:07.917845+0000","last_peered":"2023-10-09T09:04:07.917845+0000","last_clean":"2023-10-09T09:04:07.917845+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:04:07.917845+0000","last_undegraded":"2023-10-09T09:04:07.917845+0000","last_fullsized":"2023-10-09T09:04:07.917845+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":207,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:12.587817+0000","last_change":"2023-10-09T09:03:19.575546+0000","last_active":"2023-10-09T09:04:12.587817+0000","last_peered":"2023-10-09T09:04:12.587817+0000","last_clean":"2023-10-09T09:04:12.587817+0000","last_became_active":"2023-10-09T09:03:19.575075+0000","last_became_peered":"2023-10-09T09:03:19.575075+0000","last_unstale":"2023-10-09T09:04:12.587817+0000","last_undegraded":"2023-10-09T09:04:12.587817+0000","last_fullsized":"2023-10-09T09:04:12.587817+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:14.350886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"201'8939","reported_seq":8985,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:36.010809+0000","last_change":"2023-10-09T09:01:36.284533+0000","last_active":"2023-10-09T09:04:36.010809+0000","last_peered":"2023-10-09T09:04:36.010809+0000","last_clean":"2023-10-09T09:04:36.010809+0000","last_became_active":"2023-10-09T09:01:36.284124+0000","last_became_peered":"2023-10-09T09:01:36.284124+0000","last_unstale":"2023-10-09T09:04:36.010809+0000","last_undegraded":"2023-10-09T09:04:36.010809+0000","last_fullsized":"2023-10-09T09:04:36.010809+0000","mapping_epoch":176,"log_start":"186'3700","ondisk_log_start":"186'3700","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5239,"log_dups_size":0,"ondisk_log_size":5239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139132928,"num_objects":2123,"num_object_clones":0,"num_object_copies":6369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8939,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":131,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.925769+0000","last_change":"2023-10-09T09:01:36.274911+0000","last_active":"2023-10-09T09:04:07.925769+0000","last_peered":"2023-10-09T09:04:07.925769+0000","last_clean":"2023-10-09T09:04:07.925769+0000","last_became_active":"2023-10-09T09:01:36.274560+0000","last_became_peered":"2023-10-09T09:01:36.274560+0000","last_unstale":"2023-10-09T09:04:07.925769+0000","last_undegraded":"2023-10-09T09:04:07.925769+0000","last_fullsized":"2023-10-09T09:04:07.925769+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:55.305969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":440,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:12.955615+0000","last_change":"2023-10-09T09:02:05.534070+0000","last_active":"2023-10-09T09:04:12.955615+0000","last_peered":"2023-10-09T09:04:12.955615+0000","last_clean":"2023-10-09T09:04:12.955615+0000","last_became_active":"2023-10-09T09:02:05.533567+0000","last_became_peered":"2023-10-09T09:02:05.533567+0000","last_unstale":"2023-10-09T09:04:12.955615+0000","last_undegraded":"2023-10-09T09:04:12.955615+0000","last_fullsized":"2023-10-09T09:04:12.955615+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:59.568077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":18,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928571+0000","last_change":"2023-10-09T09:03:19.562659+0000","last_active":"2023-10-09T09:04:07.928571+0000","last_peered":"2023-10-09T09:04:07.928571+0000","last_clean":"2023-10-09T09:04:07.928571+0000","last_became_active":"2023-10-09T09:03:19.562284+0000","last_became_peered":"2023-10-09T09:03:19.562284+0000","last_unstale":"2023-10-09T09:04:07.928571+0000","last_undegraded":"2023-10-09T09:04:07.928571+0000","last_fullsized":"2023-10-09T09:04:07.928571+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:26.537954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":50,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.919011+0000","last_change":"2023-10-09T09:00:31.859121+0000","last_active":"2023-10-09T09:04:07.919011+0000","last_peered":"2023-10-09T09:04:07.919011+0000","last_clean":"2023-10-09T09:04:07.919011+0000","last_became_active":"2023-10-09T09:00:31.858675+0000","last_became_peered":"2023-10-09T09:00:31.858675+0000","last_unstale":"2023-10-09T09:04:07.919011+0000","last_undegraded":"2023-10-09T09:04:07.919011+0000","last_fullsized":"2023-10-09T09:04:07.919011+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"201'8783","reported_seq":12109,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:32.863355+0000","last_change":"2023-10-09T09:03:19.569754+0000","last_active":"2023-10-09T09:04:32.863355+0000","last_peered":"2023-10-09T09:04:32.863355+0000","last_clean":"2023-10-09T09:02:01.567140+0000","last_became_active":"2023-10-09T09:02:06.809579+0000","last_became_peered":"2023-10-09T09:02:06.809579+0000","last_unstale":"2023-10-09T09:04:32.863355+0000","last_undegraded":"2023-10-09T09:03:19.569445+0000","last_fullsized":"2023-10-09T09:02:03.617414+0000","mapping_epoch":182,"log_start":"185'3000","ondisk_log_start":"185'3000","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:47.900903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147783680,"num_objects":2255,"num_object_clones":0,"num_object_copies":6765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8783,"num_write_kb":353216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1634,"num_bytes_recovered":107085824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"2,3","objects":1415}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":94,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.924981+0000","last_change":"2023-10-09T09:02:02.649488+0000","last_active":"2023-10-09T09:04:07.924981+0000","last_peered":"2023-10-09T09:04:07.924981+0000","last_clean":"2023-10-09T09:04:07.924981+0000","last_became_active":"2023-10-09T09:02:02.645550+0000","last_became_peered":"2023-10-09T09:02:02.645550+0000","last_unstale":"2023-10-09T09:04:07.924981+0000","last_undegraded":"2023-10-09T09:04:07.924981+0000","last_fullsized":"2023-10-09T09:04:07.924981+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:40.192731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":230,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:13.065112+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:04:13.065112+0000","last_peered":"2023-10-09T09:04:13.065112+0000","last_clean":"2023-10-09T09:04:13.065112+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:04:13.065112+0000","last_undegraded":"2023-10-09T09:04:13.065112+0000","last_fullsized":"2023-10-09T09:04:13.065112+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":190,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.940341+0000","last_change":"2023-10-09T09:02:02.657733+0000","last_active":"2023-10-09T09:04:07.940341+0000","last_peered":"2023-10-09T09:04:07.940341+0000","last_clean":"2023-10-09T09:04:07.940341+0000","last_became_active":"2023-10-09T09:02:02.656701+0000","last_became_peered":"2023-10-09T09:02:02.656701+0000","last_unstale":"2023-10-09T09:04:07.940341+0000","last_undegraded":"2023-10-09T09:04:07.940341+0000","last_fullsized":"2023-10-09T09:04:07.940341+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:39.552401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"201'8957","reported_seq":9010,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:35.510266+0000","last_change":"2023-10-09T09:02:52.233981+0000","last_active":"2023-10-09T09:04:35.510266+0000","last_peered":"2023-10-09T09:04:35.510266+0000","last_clean":"2023-10-09T09:02:01.581848+0000","last_became_active":"2023-10-09T09:02:52.219467+0000","last_became_peered":"2023-10-09T09:02:52.219467+0000","last_unstale":"2023-10-09T09:04:35.510266+0000","last_undegraded":"2023-10-09T09:02:52.047554+0000","last_fullsized":"2023-10-09T09:02:51.903742+0000","mapping_epoch":190,"log_start":"189'4322","ondisk_log_start":"189'4322","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4635,"log_dups_size":0,"ondisk_log_size":4635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:09:00.904807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142016512,"num_objects":2167,"num_object_clones":0,"num_object_copies":6501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4936,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8957,"num_write_kb":355968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"3,4","objects":-2769},{"shards":"3,4,6","objects":1541}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":173,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.924193+0000","last_change":"2023-10-09T09:02:02.644204+0000","last_active":"2023-10-09T09:04:07.924193+0000","last_peered":"2023-10-09T09:04:07.924193+0000","last_clean":"2023-10-09T09:04:07.924193+0000","last_became_active":"2023-10-09T09:02:02.643713+0000","last_became_peered":"2023-10-09T09:02:02.643713+0000","last_unstale":"2023-10-09T09:04:07.924193+0000","last_undegraded":"2023-10-09T09:04:07.924193+0000","last_fullsized":"2023-10-09T09:04:07.924193+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:20.811353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":169,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.926003+0000","last_change":"2023-10-09T09:02:02.647803+0000","last_active":"2023-10-09T09:04:07.926003+0000","last_peered":"2023-10-09T09:04:07.926003+0000","last_clean":"2023-10-09T09:04:07.926003+0000","last_became_active":"2023-10-09T09:02:02.646911+0000","last_became_peered":"2023-10-09T09:02:02.646911+0000","last_unstale":"2023-10-09T09:04:07.926003+0000","last_undegraded":"2023-10-09T09:04:07.926003+0000","last_fullsized":"2023-10-09T09:04:07.926003+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:35.294072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":188,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:13.065182+0000","last_change":"2023-10-09T09:02:52.016015+0000","last_active":"2023-10-09T09:04:13.065182+0000","last_peered":"2023-10-09T09:04:13.065182+0000","last_clean":"2023-10-09T09:04:13.065182+0000","last_became_active":"2023-10-09T09:02:52.015543+0000","last_became_peered":"2023-10-09T09:02:52.015543+0000","last_unstale":"2023-10-09T09:04:13.065182+0000","last_undegraded":"2023-10-09T09:04:13.065182+0000","last_fullsized":"2023-10-09T09:04:13.065182+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.410980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"201'8913","reported_seq":8961,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:37.556572+0000","last_change":"2023-10-09T09:01:36.279450+0000","last_active":"2023-10-09T09:04:37.556572+0000","last_peered":"2023-10-09T09:04:37.556572+0000","last_clean":"2023-10-09T09:04:37.556572+0000","last_became_active":"2023-10-09T09:01:36.279123+0000","last_became_peered":"2023-10-09T09:01:36.279123+0000","last_unstale":"2023-10-09T09:04:37.556572+0000","last_undegraded":"2023-10-09T09:04:37.556572+0000","last_fullsized":"2023-10-09T09:04:37.556572+0000","mapping_epoch":176,"log_start":"185'3200","ondisk_log_start":"185'3200","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5713,"log_dups_size":0,"ondisk_log_size":5713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:21.843271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138084352,"num_objects":2107,"num_object_clones":0,"num_object_copies":6321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8913,"num_write_kb":352640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":201,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:13.064518+0000","last_change":"2023-10-09T09:01:36.292450+0000","last_active":"2023-10-09T09:04:13.064518+0000","last_peered":"2023-10-09T09:04:13.064518+0000","last_clean":"2023-10-09T09:04:13.064518+0000","last_became_active":"2023-10-09T09:01:36.273029+0000","last_became_peered":"2023-10-09T09:01:36.273029+0000","last_unstale":"2023-10-09T09:04:13.064518+0000","last_undegraded":"2023-10-09T09:04:13.064518+0000","last_fullsized":"2023-10-09T09:04:13.064518+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:26.065684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":163,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.918465+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:04:07.918465+0000","last_peered":"2023-10-09T09:04:07.918465+0000","last_clean":"2023-10-09T09:04:07.918465+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:04:07.918465+0000","last_undegraded":"2023-10-09T09:04:07.918465+0000","last_fullsized":"2023-10-09T09:04:07.918465+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":18,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.927604+0000","last_change":"2023-10-09T09:03:19.572475+0000","last_active":"2023-10-09T09:04:07.927604+0000","last_peered":"2023-10-09T09:04:07.927604+0000","last_clean":"2023-10-09T09:04:07.927604+0000","last_became_active":"2023-10-09T09:03:19.572081+0000","last_became_peered":"2023-10-09T09:03:19.572081+0000","last_unstale":"2023-10-09T09:04:07.927604+0000","last_undegraded":"2023-10-09T09:04:07.927604+0000","last_fullsized":"2023-10-09T09:04:07.927604+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:04.771178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"201'8738","reported_seq":11900,"reported_epoch":201,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:32.866605+0000","last_change":"2023-10-09T09:04:04.889340+0000","last_active":"2023-10-09T09:04:32.866605+0000","last_peered":"2023-10-09T09:04:32.866605+0000","last_clean":"2023-10-09T09:02:01.579768+0000","last_became_active":"2023-10-09T09:02:03.760710+0000","last_became_peered":"2023-10-09T09:02:03.760710+0000","last_unstale":"2023-10-09T09:04:32.866605+0000","last_undegraded":"2023-10-09T09:02:07.347090+0000","last_fullsized":"2023-10-09T09:02:03.619050+0000","mapping_epoch":182,"log_start":"185'2900","ondisk_log_start":"185'2900","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:35.438271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148963352,"num_objects":2274,"num_object_clones":0,"num_object_copies":6822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":933,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2274,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8738,"num_write_kb":352321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1556,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"2,6","objects":1341}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":50,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.918651+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:04:07.918651+0000","last_peered":"2023-10-09T09:04:07.918651+0000","last_clean":"2023-10-09T09:04:07.918651+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:04:07.918651+0000","last_undegraded":"2023-10-09T09:04:07.918651+0000","last_fullsized":"2023-10-09T09:04:07.918651+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":209,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.940371+0000","last_change":"2023-10-09T09:02:02.674657+0000","last_active":"2023-10-09T09:04:07.940371+0000","last_peered":"2023-10-09T09:04:07.940371+0000","last_clean":"2023-10-09T09:04:07.940371+0000","last_became_active":"2023-10-09T09:02:02.657392+0000","last_became_peered":"2023-10-09T09:02:02.657392+0000","last_unstale":"2023-10-09T09:04:07.940371+0000","last_undegraded":"2023-10-09T09:04:07.940371+0000","last_fullsized":"2023-10-09T09:04:07.940371+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:16.286583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914304+0000","last_change":"2023-10-09T09:01:36.280537+0000","last_active":"2023-10-09T09:04:07.914304+0000","last_peered":"2023-10-09T09:04:07.914304+0000","last_clean":"2023-10-09T09:04:07.914304+0000","last_became_active":"2023-10-09T09:01:36.280149+0000","last_became_peered":"2023-10-09T09:01:36.280149+0000","last_unstale":"2023-10-09T09:04:07.914304+0000","last_undegraded":"2023-10-09T09:04:07.914304+0000","last_fullsized":"2023-10-09T09:04:07.914304+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":208,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.923475+0000","last_change":"2023-10-09T09:03:19.554339+0000","last_active":"2023-10-09T09:04:07.923475+0000","last_peered":"2023-10-09T09:04:07.923475+0000","last_clean":"2023-10-09T09:04:07.923475+0000","last_became_active":"2023-10-09T09:03:19.553953+0000","last_became_peered":"2023-10-09T09:03:19.553953+0000","last_unstale":"2023-10-09T09:04:07.923475+0000","last_undegraded":"2023-10-09T09:04:07.923475+0000","last_fullsized":"2023-10-09T09:04:07.923475+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:50.601515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":30,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.925925+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:04:07.925925+0000","last_peered":"2023-10-09T09:04:07.925925+0000","last_clean":"2023-10-09T09:04:07.925925+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:04:07.925925+0000","last_undegraded":"2023-10-09T09:04:07.925925+0000","last_fullsized":"2023-10-09T09:04:07.925925+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":203,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.916116+0000","last_change":"2023-10-09T09:02:52.022788+0000","last_active":"2023-10-09T09:04:07.916116+0000","last_peered":"2023-10-09T09:04:07.916116+0000","last_clean":"2023-10-09T09:04:07.916116+0000","last_became_active":"2023-10-09T09:02:52.016418+0000","last_became_peered":"2023-10-09T09:02:52.016418+0000","last_unstale":"2023-10-09T09:04:07.916116+0000","last_undegraded":"2023-10-09T09:04:07.916116+0000","last_fullsized":"2023-10-09T09:04:07.916116+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":63,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914949+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:04:07.914949+0000","last_peered":"2023-10-09T09:04:07.914949+0000","last_clean":"2023-10-09T09:04:07.914949+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:04:07.914949+0000","last_undegraded":"2023-10-09T09:04:07.914949+0000","last_fullsized":"2023-10-09T09:04:07.914949+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":188,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914958+0000","last_change":"2023-10-09T09:03:19.575218+0000","last_active":"2023-10-09T09:04:07.914958+0000","last_peered":"2023-10-09T09:04:07.914958+0000","last_clean":"2023-10-09T09:04:07.914958+0000","last_became_active":"2023-10-09T09:03:19.557578+0000","last_became_peered":"2023-10-09T09:03:19.557578+0000","last_unstale":"2023-10-09T09:04:07.914958+0000","last_undegraded":"2023-10-09T09:04:07.914958+0000","last_fullsized":"2023-10-09T09:04:07.914958+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":54,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.914429+0000","last_change":"2023-10-09T08:59:53.522794+0000","last_active":"2023-10-09T09:04:07.914429+0000","last_peered":"2023-10-09T09:04:07.914429+0000","last_clean":"2023-10-09T09:04:07.914429+0000","last_became_active":"2023-10-09T08:59:53.522419+0000","last_became_peered":"2023-10-09T08:59:53.522419+0000","last_unstale":"2023-10-09T09:04:07.914429+0000","last_undegraded":"2023-10-09T09:04:07.914429+0000","last_fullsized":"2023-10-09T09:04:07.914429+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":201,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.928191+0000","last_change":"2023-10-09T09:00:31.845771+0000","last_active":"2023-10-09T09:04:07.928191+0000","last_peered":"2023-10-09T09:04:07.928191+0000","last_clean":"2023-10-09T09:04:07.928191+0000","last_became_active":"2023-10-09T09:00:31.844775+0000","last_became_peered":"2023-10-09T09:00:31.844775+0000","last_unstale":"2023-10-09T09:04:07.928191+0000","last_undegraded":"2023-10-09T09:04:07.928191+0000","last_fullsized":"2023-10-09T09:04:07.928191+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:33.452539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":18,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.937189+0000","last_change":"2023-10-09T09:03:19.561924+0000","last_active":"2023-10-09T09:04:07.937189+0000","last_peered":"2023-10-09T09:04:07.937189+0000","last_clean":"2023-10-09T09:04:07.937189+0000","last_became_active":"2023-10-09T09:03:19.557692+0000","last_became_peered":"2023-10-09T09:03:19.557692+0000","last_unstale":"2023-10-09T09:04:07.937189+0000","last_undegraded":"2023-10-09T09:04:07.937189+0000","last_fullsized":"2023-10-09T09:04:07.937189+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:39.940109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":261,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T09:04:12.955856+0000","last_change":"2023-10-09T09:02:52.019620+0000","last_active":"2023-10-09T09:04:12.955856+0000","last_peered":"2023-10-09T09:04:12.955856+0000","last_clean":"2023-10-09T09:04:12.955856+0000","last_became_active":"2023-10-09T09:02:52.014489+0000","last_became_peered":"2023-10-09T09:02:52.014489+0000","last_unstale":"2023-10-09T09:04:12.955856+0000","last_undegraded":"2023-10-09T09:04:12.955856+0000","last_fullsized":"2023-10-09T09:04:12.955856+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:39.550600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T09:04:07.913794+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:04:07.913794+0000","last_peered":"2023-10-09T09:04:07.913794+0000","last_clean":"2023-10-09T09:04:07.913794+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:04:07.913794+0000","last_undegraded":"2023-10-09T09:04:07.913794+0000","last_fullsized":"2023-10-09T09:04:07.913794+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":13518,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2249326616,"num_objects":34323,"num_object_clones":0,"num_object_copies":102969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16645,"num_objects_misplaced":5250,"num_objects_unfound":0,"num_objects_dirty":34323,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142151,"num_write_kb":5647105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16181,"num_bytes_recovered":962265088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8375644160,"data_stored":8375631944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85410,"ondisk_log_size":85410,"up":48,"acting":43,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430609,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626842,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2140632,"kb_used_data":1542672,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":91604520,"statfs":{"total":95995035648,"available":93803028480,"internally_reserved":0,"allocated":1579696128,"data_stored":1579203908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.758,"15min":2.3210000000000002},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.24199999999999999},"max":{"1min":4.54,"5min":58.621000000000002,"15min":153.63800000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.3640000000000001,"5min":1.996,"15min":2.3380000000000001},"min":{"1min":0.38,"5min":0.36199999999999999,"15min":0.33500000000000002},"max":{"1min":4.6079999999999997,"5min":58.590000000000003,"15min":153.72200000000001},"last":0.48699999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":2.1379999999999999,"15min":1.911},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":1.9530000000000001,"5min":109.453,"15min":109.453},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.2010000000000001,"5min":2.302,"15min":1.9670000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":6.0529999999999999,"5min":111.47499999999999,"15min":111.47499999999999},"last":0.59699999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:04:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.1579999999999999,"15min":1.399},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.29799999999999999},"max":{"1min":5.2060000000000004,"5min":5.2910000000000004,"15min":8.3979999999999997},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.1699999999999999,"15min":1.4670000000000001},"min":{"1min":0.38500000000000001,"5min":0.30199999999999999,"15min":0.28100000000000003},"max":{"1min":3.6280000000000001,"5min":9.1549999999999994,"15min":12.194000000000001},"last":1.0920000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.3660000000000001,"15min":2.048},"min":{"1min":0.38400000000000001,"5min":0.38200000000000001,"15min":0.34399999999999997},"max":{"1min":6.992,"5min":7.4130000000000003,"15min":57.636000000000003},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.476,"5min":1.3180000000000001,"15min":1.79},"min":{"1min":0.46500000000000002,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":3.786,"5min":9.4429999999999996,"15min":58.393000000000001},"last":0.97699999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:03:34 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.4570000000000001,"15min":1.5589999999999999},"min":{"1min":0.40300000000000002,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":9.2639999999999993,"5min":32.280999999999999,"15min":32.280999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.262,"15min":1.2090000000000001},"min":{"1min":0.40300000000000002,"5min":0.33900000000000002,"15min":0.30599999999999999},"max":{"1min":2.5720000000000001,"5min":34.689,"15min":34.689},"last":0.92600000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:03:34 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":1.1020000000000001,"15min":1.3140000000000001},"min":{"1min":0.443,"5min":0.38200000000000001,"15min":0.35499999999999998},"max":{"1min":9.218,"5min":9.218,"15min":12.404999999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.024,"5min":0.94099999999999995,"15min":1.1639999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.33900000000000002},"max":{"1min":3.1259999999999999,"5min":4.2610000000000001,"15min":8.8569999999999993},"last":1.0680000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:04:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.052,"15min":1.4199999999999999},"min":{"1min":0.48199999999999998,"5min":0.434,"15min":0.40500000000000003},"max":{"1min":1.474,"5min":9.2460000000000004,"15min":11.382},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.91500000000000004,"15min":1.244},"min":{"1min":0.46100000000000002,"5min":0.45400000000000001,"15min":0.377},"max":{"1min":1.04,"5min":4.3259999999999996,"15min":9.4800000000000004},"last":1.04}]}]},{"osd":1,"up_from":0,"seq":356482285790,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449316,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1609696,"kb_used_data":1215448,"kb_used_omap":28,"kb_used_meta":394211,"kb_avail":92135456,"statfs":{"total":95995035648,"available":94346706944,"internally_reserved":0,"allocated":1244618752,"data_stored":1243827915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":403672912},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.732,"15min":1.5609999999999999},"min":{"1min":0.26000000000000001,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":6.7649999999999997,"5min":9.9879999999999995,"15min":37.380000000000003},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.6859999999999999,"15min":1.5780000000000001},"min":{"1min":0.34100000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.6950000000000003,"5min":18.661000000000001,"15min":38.012999999999998},"last":0.51200000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":3.6720000000000002,"15min":2.3580000000000001},"min":{"1min":0.32400000000000001,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":9.4719999999999995,"5min":71.322999999999993,"15min":71.322999999999993},"last":9.4719999999999995},{"interface":"front","average":{"1min":14.773999999999999,"5min":6.1130000000000004,"15min":3.2589999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":219.00700000000001,"5min":219.00700000000001,"15min":219.00700000000001},"last":1.8740000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:03:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":2.1659999999999999,"15min":2.4529999999999998},"min":{"1min":0.39400000000000002,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":9.7289999999999992,"5min":9.7289999999999992,"15min":110.988},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.7170000000000001,"5min":2.2639999999999998,"15min":2.4119999999999999},"min":{"1min":0.34399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":12.446,"5min":19.167000000000002,"15min":107.66500000000001},"last":2.5939999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":1.5309999999999999,"15min":1.466},"min":{"1min":0.66500000000000004,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.178000000000001,"5min":10.178000000000001,"15min":10.178000000000001},"last":1.952},{"interface":"front","average":{"1min":2.246,"5min":1.716,"15min":1.673},"min":{"1min":0.44500000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.526,"5min":10.526,"15min":11.654},"last":0.47499999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.867,"5min":1.631,"15min":1.4059999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.34499999999999997},"max":{"1min":9.3030000000000008,"5min":9.3030000000000008,"15min":9.3689999999999998},"last":9.3030000000000008},{"interface":"front","average":{"1min":2.5350000000000001,"5min":1.581,"15min":1.5760000000000001},"min":{"1min":0.54900000000000004,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.140000000000001,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.54900000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":2.577,"15min":1.855},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":9.5079999999999991,"5min":62.850999999999999,"15min":62.850999999999999},"last":9.5079999999999991},{"interface":"front","average":{"1min":2.452,"5min":2.5979999999999999,"15min":2.0590000000000002},"min":{"1min":0.49399999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.121,"5min":62.816000000000003,"15min":62.816000000000003},"last":1.9319999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.6799999999999999,"15min":1.6519999999999999},"min":{"1min":0.63600000000000001,"5min":0.42499999999999999,"15min":0.39400000000000002},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":1.903},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.845,"15min":1.855},"min":{"1min":0.53200000000000003,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":10.662000000000001,"5min":14.359,"15min":14.359},"last":0.53200000000000003}]}]},{"osd":2,"up_from":88,"seq":377957122263,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1766864,"kb_used_data":1292552,"kb_used_omap":81,"kb_used_meta":474158,"kb_avail":91978288,"statfs":{"total":95995035648,"available":94185766912,"internally_reserved":0,"allocated":1323573248,"data_stored":1322742654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":485538010},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,3,9,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":4.5979999999999999,"5min":5.1059999999999999,"15min":2.8730000000000002},"min":{"1min":0.27800000000000002,"5min":0.214,"15min":0.191},"max":{"1min":46.421999999999997,"5min":134.458,"15min":134.458},"last":2.649},{"interface":"front","average":{"1min":4.5970000000000004,"5min":4.9930000000000003,"15min":2.5609999999999999},"min":{"1min":0.36899999999999999,"5min":0.254,"15min":0.221},"max":{"1min":46.795999999999999,"5min":138.417,"15min":138.417},"last":3.706}]},{"osd":1,"last update":"Mon Oct 9 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":2.7050000000000001,"15min":2.173},"min":{"1min":0.35399999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.9980000000000002,"5min":57.939999999999998,"15min":57.939999999999998},"last":0.31},{"interface":"front","average":{"1min":1.673,"5min":2.895,"15min":2.0529999999999999},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":8.9139999999999997,"5min":56.667999999999999,"15min":57.122999999999998},"last":1.6379999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:03:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":4.8369999999999997,"15min":2.5150000000000001},"min":{"1min":0.35699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.9819999999999993,"5min":112.47499999999999,"15min":112.47499999999999},"last":1.728},{"interface":"front","average":{"1min":3.5950000000000002,"5min":4.9530000000000003,"15min":2.827},"min":{"1min":0.35299999999999998,"5min":0.247,"15min":0.247},"max":{"1min":10.583,"5min":112.929,"15min":112.929},"last":0.26800000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.6839999999999999,"15min":1.7150000000000001},"min":{"1min":0.433,"5min":0.39000000000000001,"15min":0.35099999999999998},"max":{"1min":3.8380000000000001,"5min":9.8109999999999999,"15min":12.191000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.431,"5min":1.651,"15min":1.71},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.314},"max":{"1min":5.0460000000000003,"5min":10.426,"15min":14.071999999999999},"last":1.7969999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.6539999999999999,"15min":1.448},"min":{"1min":0.49299999999999999,"5min":0.36699999999999999,"15min":0.33900000000000002},"max":{"1min":8.8239999999999998,"5min":12.411,"15min":12.411},"last":1.7010000000000001},{"interface":"front","average":{"1min":1.347,"5min":1.728,"15min":1.7250000000000001},"min":{"1min":0.38900000000000001,"5min":0.35799999999999998,"15min":0.34999999999999998},"max":{"1min":3.956,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.50800000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.639,"15min":1.732},"min":{"1min":0.48999999999999999,"5min":0.42699999999999999,"15min":0.33900000000000002},"max":{"1min":5.0810000000000004,"5min":10.456,"15min":10.456},"last":1.8280000000000001},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.7390000000000001,"15min":1.5509999999999999},"min":{"1min":0.46000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":6.4720000000000004,"5min":12.381,"15min":12.381},"last":1.675}]},{"osd":7,"last update":"Mon Oct 9 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":1.651,"15min":1.7210000000000001},"min":{"1min":0.55700000000000005,"5min":0.46000000000000002,"15min":0.38800000000000001},"max":{"1min":5.1200000000000001,"5min":10.380000000000001,"15min":17.77},"last":1.8560000000000001},{"interface":"front","average":{"1min":1.931,"5min":1.77,"15min":1.518},"min":{"1min":0.54100000000000004,"5min":0.433,"15min":0.38500000000000001},"max":{"1min":8.8789999999999996,"5min":12.438000000000001,"15min":12.438000000000001},"last":1.7549999999999999}]}]},{"osd":3,"up_from":93,"seq":399431958738,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1561996,"kb_used_data":1168708,"kb_used_omap":7,"kb_used_meta":393208,"kb_avail":92183156,"statfs":{"total":95995035648,"available":94395551744,"internally_reserved":0,"allocated":1196756992,"data_stored":1195984024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":402645608},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,5,4,2,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.7170000000000001,"15min":1.5209999999999999},"min":{"1min":0.32600000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":9.0199999999999996,"5min":9.0199999999999996,"15min":13.916},"last":0.93600000000000005},{"interface":"front","average":{"1min":2.1299999999999999,"5min":2.0630000000000002,"15min":1.589},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19800000000000001},"max":{"1min":9.7490000000000006,"5min":14.013999999999999,"15min":14.013999999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":4.9870000000000001,"15min":2.702},"min":{"1min":0.31,"5min":0.26700000000000002,"15min":0.22500000000000001},"max":{"1min":7.6239999999999997,"5min":326.75700000000001,"15min":326.75700000000001},"last":0.751},{"interface":"front","average":{"1min":1.7569999999999999,"5min":5.0960000000000001,"15min":2.6960000000000002},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.245},"max":{"1min":8.9139999999999997,"5min":327.024,"15min":327.024},"last":0.96299999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":2.5489999999999999,"15min":2.371},"min":{"1min":0.28599999999999998,"5min":0.255,"15min":0.255},"max":{"1min":9.0370000000000008,"5min":43.683999999999997,"15min":75.641999999999996},"last":1.054},{"interface":"front","average":{"1min":2.5230000000000001,"5min":2.5990000000000002,"15min":2.2949999999999999},"min":{"1min":0.53300000000000003,"5min":0.32800000000000001,"15min":0.27100000000000002},"max":{"1min":9.3759999999999994,"5min":45.027999999999999,"15min":75.207999999999998},"last":1.2809999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.452,"15min":1.4710000000000001},"min":{"1min":0.42399999999999999,"5min":0.35699999999999998,"15min":0.35399999999999998},"max":{"1min":8.9879999999999995,"5min":9.2940000000000005,"15min":12.877000000000001},"last":1.1950000000000001},{"interface":"front","average":{"1min":1.659,"5min":1.8260000000000001,"15min":1.657},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.33500000000000002},"max":{"1min":9.4649999999999999,"5min":10.486000000000001,"15min":10.486000000000001},"last":0.40100000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.712,"15min":1.571},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.33700000000000002},"max":{"1min":9.4990000000000006,"5min":10.577,"15min":10.577},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.581,"15min":1.647},"min":{"1min":0.44900000000000001,"5min":0.39400000000000002,"15min":0.34300000000000003},"max":{"1min":7.5590000000000002,"5min":7.5590000000000002,"15min":12.204000000000001},"last":0.82699999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.837,"15min":1.7290000000000001},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.1370000000000005,"5min":10.537000000000001,"15min":10.537000000000001},"last":0.439},{"interface":"front","average":{"1min":2.3799999999999999,"5min":1.75,"15min":1.764},"min":{"1min":0.57399999999999995,"5min":0.39600000000000002,"15min":0.35099999999999998},"max":{"1min":7.5220000000000002,"5min":7.5220000000000002,"15min":12.275},"last":0.78900000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.6459999999999999,"15min":1.661},"min":{"1min":0.55900000000000005,"5min":0.443,"15min":0.41299999999999998},"max":{"1min":7.5899999999999999,"5min":7.5899999999999999,"15min":12.244},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.972,"5min":1.51,"15min":1.6200000000000001},"min":{"1min":0.52300000000000002,"5min":0.45200000000000001,"15min":0.39000000000000001},"max":{"1min":9.1560000000000006,"5min":9.2509999999999994,"15min":13.339},"last":1.2450000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019308,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2298908,"kb_used_data":1989396,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91446244,"statfs":{"total":95995035648,"available":93640953856,"internally_reserved":0,"allocated":2037141504,"data_stored":2036644517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":2.9889999999999999,"15min":2.29},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26100000000000001},"max":{"1min":5.077,"5min":141.07400000000001,"15min":141.07400000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.454,"5min":2.8700000000000001,"15min":2.3079999999999998},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":8.6379999999999999,"5min":141.15600000000001,"15min":141.15600000000001},"last":0.68999999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.0700000000000001,"15min":1.4019999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":2.601,"5min":6.0529999999999999,"15min":8.9169999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.427,"15min":1.4950000000000001},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":11.712999999999999},"last":1.01}]},{"osd":2,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":3.093,"15min":2.2650000000000001},"min":{"1min":0.47499999999999998,"5min":0.36899999999999999,"15min":0.30499999999999999},"max":{"1min":8.4090000000000007,"5min":142.816,"15min":142.816},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.498,"5min":2.7320000000000002,"15min":1.9179999999999999},"min":{"1min":0.40600000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.8150000000000004,"5min":143.81800000000001,"15min":143.81800000000001},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 09:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":2.7599999999999998,"15min":2.0979999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29499999999999998},"max":{"1min":6.2640000000000002,"5min":143.13399999999999,"15min":143.13399999999999},"last":1.1339999999999999},{"interface":"front","average":{"1min":2.4780000000000002,"5min":3.2839999999999998,"15min":2.1110000000000002},"min":{"1min":0.497,"5min":0.36699999999999999,"15min":0.35399999999999998},"max":{"1min":13.747999999999999,"5min":152.37299999999999,"15min":152.37299999999999},"last":1.018}]},{"osd":5,"last update":"Mon Oct 9 09:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.0169999999999999,"15min":1.214},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.7350000000000003,"5min":7.7439999999999998,"15min":11.762},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.872,"15min":1.2769999999999999},"min":{"1min":0.38900000000000001,"5min":0.37,"15min":0.28699999999999998},"max":{"1min":3.4900000000000002,"5min":3.4900000000000002,"15min":12.85},"last":0.53000000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.91800000000000004,"15min":1.427},"min":{"1min":0.36799999999999999,"5min":0.35699999999999998,"15min":0.32100000000000001},"max":{"1min":5.1020000000000003,"5min":5.1020000000000003,"15min":10.657999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.242,"5min":1.089,"15min":1.464},"min":{"1min":0.438,"5min":0.42699999999999999,"15min":0.34300000000000003},"max":{"1min":7.6470000000000002,"5min":7.6470000000000002,"15min":14.755000000000001},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.93000000000000005,"15min":1.4119999999999999},"min":{"1min":0.42999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.4660000000000002,"5min":3.4660000000000002,"15min":13.933999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.90400000000000003,"15min":1.5700000000000001},"min":{"1min":0.372,"5min":0.36199999999999999,"15min":0.33200000000000002},"max":{"1min":2.1869999999999998,"5min":5.1520000000000001,"15min":13.911},"last":0.59399999999999997}]}]},{"osd":5,"up_from":0,"seq":154618823076,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1216806912,"data_stored":1216806912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1082589184,"data_stored":1082589184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1310986240,"data_stored":1310982168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1193869312,"data_stored":1193869312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2015105024,"data_stored":2015100952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1556287488,"data_stored":1556283416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:04:42.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:04:42.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:42.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:42.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:42.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:42.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:42.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:42.635 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:42.656 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:42.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:42.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:42.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:42.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:42.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:42.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:42.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:43.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:42 smithi016 bash[75179]: cluster 2023-10-09T09:04:41.866123+0000 mgr.y (mgr.25421) 815 : cluster 0 pgmap v846: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 16266/95301 objects degraded (17.068%); 4795/95301 objects misplaced (5.031%); 3.7 MiB/s, 119 objects/s recovering 2023-10-09T09:04:43.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:42 smithi016 bash[77892]: cluster 2023-10-09T09:04:41.866123+0000 mgr.y (mgr.25421) 815 : cluster 0 pgmap v846: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 16266/95301 objects degraded (17.068%); 4795/95301 objects misplaced (5.031%); 3.7 MiB/s, 119 objects/s recovering 2023-10-09T09:04:43.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:43.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:43.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:43.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:43.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:43.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:43.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:43.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:43.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:42 smithi094 bash[59677]: cluster 2023-10-09T09:04:41.866123+0000 mgr.y (mgr.25421) 815 : cluster 0 pgmap v846: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 16266/95301 objects degraded (17.068%); 4795/95301 objects misplaced (5.031%); 3.7 MiB/s, 119 objects/s recovering 2023-10-09T09:04:43.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:43.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:43.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:43.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:43.390 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:04:43.443 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:04:43.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:43.492 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:04:43.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:43.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:43.530 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:04:43.571 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:04:43.606 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:04:43.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:43.655 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:04:43.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:43.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:43.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:43.690 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:04:43.695 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.696 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:04:43.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:43.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:43.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:43.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:43.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:43.946 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:04:43.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:43.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:44.067 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:44.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:44.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:44.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:44.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:43 smithi094 bash[59677]: audit 2023-10-09T09:04:43.320319+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.16:0/1500933033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:04:44.207 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:43 smithi094 bash[59677]: audit 2023-10-09T09:04:43.446069+0000 mon.a (mon.0) 1107 : audit 1 from='client.? 172.21.15.16:0/3786538955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:04:44.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:44.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:44.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:44.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:43 smithi016 bash[75179]: audit 2023-10-09T09:04:43.320319+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.16:0/1500933033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:04:44.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:43 smithi016 bash[75179]: audit 2023-10-09T09:04:43.446069+0000 mon.a (mon.0) 1107 : audit 1 from='client.? 172.21.15.16:0/3786538955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:04:44.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:43 smithi016 bash[77892]: audit 2023-10-09T09:04:43.320319+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.16:0/1500933033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:04:44.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:43 smithi016 bash[77892]: audit 2023-10-09T09:04:43.446069+0000 mon.a (mon.0) 1107 : audit 1 from='client.? 172.21.15.16:0/3786538955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:04:44.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:44.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:44.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:44.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:44.486 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:44.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:44.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:44.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:44.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:44.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:44.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:44.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:44.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:44.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:44.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:44.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:44.971 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: cluster 2023-10-09T09:04:43.866971+0000 mgr.y (mgr.25421) 816 : cluster 0 pgmap v847: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 15801/91275 objects degraded (17.311%); 4359/91275 objects misplaced (4.776%); 5.6 MiB/s, 188 objects/s recovering 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: cluster 2023-10-09T09:04:43.922747+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: cluster 2023-10-09T09:04:43.933892+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: cluster 2023-10-09T09:04:43.933920+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: audit 2023-10-09T09:04:43.940319+0000 mon.a (mon.0) 1109 : audit 1 from='client.? 172.21.15.16:0/3786538955' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: cluster 2023-10-09T09:04:43.940850+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e202: 8 total, 8 up, 5 in 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: cluster 2023-10-09T09:04:43.946880+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: audit 2023-10-09T09:04:44.449331+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: audit 2023-10-09T09:04:44.455122+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: cluster 2023-10-09T09:04:44.557578+0000 mon.a (mon.0) 1113 : cluster 3 Health check update: Degraded data redundancy: 15801/91275 objects degraded (17.311%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: audit 2023-10-09T09:04:44.921319+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: audit 2023-10-09T09:04:44.922605+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[75179]: audit 2023-10-09T09:04:44.928969+0000 mon.a (mon.0) 1116 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: cluster 2023-10-09T09:04:43.866971+0000 mgr.y (mgr.25421) 816 : cluster 0 pgmap v847: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 15801/91275 objects degraded (17.311%); 4359/91275 objects misplaced (4.776%); 5.6 MiB/s, 188 objects/s recovering 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: cluster 2023-10-09T09:04:43.922747+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: cluster 2023-10-09T09:04:43.933892+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: cluster 2023-10-09T09:04:43.933920+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: audit 2023-10-09T09:04:43.940319+0000 mon.a (mon.0) 1109 : audit 1 from='client.? 172.21.15.16:0/3786538955' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:04:45.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: cluster 2023-10-09T09:04:43.940850+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e202: 8 total, 8 up, 5 in 2023-10-09T09:04:45.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: cluster 2023-10-09T09:04:43.946880+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: audit 2023-10-09T09:04:44.449331+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: audit 2023-10-09T09:04:44.455122+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: cluster 2023-10-09T09:04:44.557578+0000 mon.a (mon.0) 1113 : cluster 3 Health check update: Degraded data redundancy: 15801/91275 objects degraded (17.311%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:45.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: audit 2023-10-09T09:04:44.921319+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:04:45.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: audit 2023-10-09T09:04:44.922605+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:04:45.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:44 smithi016 bash[77892]: audit 2023-10-09T09:04:44.928969+0000 mon.a (mon.0) 1116 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:45.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:45.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:45.134 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:45.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:45.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: cluster 2023-10-09T09:04:43.866971+0000 mgr.y (mgr.25421) 816 : cluster 0 pgmap v847: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 15801/91275 objects degraded (17.311%); 4359/91275 objects misplaced (4.776%); 5.6 MiB/s, 188 objects/s recovering 2023-10-09T09:04:45.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: cluster 2023-10-09T09:04:43.922747+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:04:45.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: cluster 2023-10-09T09:04:43.933892+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: cluster 2023-10-09T09:04:43.933920+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: audit 2023-10-09T09:04:43.940319+0000 mon.a (mon.0) 1109 : audit 1 from='client.? 172.21.15.16:0/3786538955' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: cluster 2023-10-09T09:04:43.940850+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e202: 8 total, 8 up, 5 in 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: cluster 2023-10-09T09:04:43.946880+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: audit 2023-10-09T09:04:44.449331+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: audit 2023-10-09T09:04:44.455122+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: cluster 2023-10-09T09:04:44.557578+0000 mon.a (mon.0) 1113 : cluster 3 Health check update: Degraded data redundancy: 15801/91275 objects degraded (17.311%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: audit 2023-10-09T09:04:44.921319+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:04:45.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: audit 2023-10-09T09:04:44.922605+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:04:45.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:44 smithi094 bash[59677]: audit 2023-10-09T09:04:44.928969+0000 mon.a (mon.0) 1116 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:45.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:45.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:45.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:45.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:45.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:45.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:45.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:45.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:45.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:45.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:45.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:45.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:45.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:45.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:45.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:45.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:45.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:04:45] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-09T09:04:46.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[75179]: cluster 2023-10-09T09:04:44.940740+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[75179]: cluster 2023-10-09T09:04:44.943550+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e203: 8 total, 8 up, 5 in 2023-10-09T09:04:46.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[75179]: cluster 2023-10-09T09:04:44.950661+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[75179]: cluster 2023-10-09T09:04:45.019544+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[75179]: cluster 2023-10-09T09:04:45.022297+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[77892]: cluster 2023-10-09T09:04:44.940740+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[77892]: cluster 2023-10-09T09:04:44.943550+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e203: 8 total, 8 up, 5 in 2023-10-09T09:04:46.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[77892]: cluster 2023-10-09T09:04:44.950661+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[77892]: cluster 2023-10-09T09:04:45.019544+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:45 smithi016 bash[77892]: cluster 2023-10-09T09:04:45.022297+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:46.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:46.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:46.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:46.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:46.191 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:46.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:46.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:45 smithi094 bash[59677]: cluster 2023-10-09T09:04:44.940740+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:45 smithi094 bash[59677]: cluster 2023-10-09T09:04:44.943550+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e203: 8 total, 8 up, 5 in 2023-10-09T09:04:46.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:45 smithi094 bash[59677]: cluster 2023-10-09T09:04:44.950661+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:45 smithi094 bash[59677]: cluster 2023-10-09T09:04:45.019544+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:45 smithi094 bash[59677]: cluster 2023-10-09T09:04:45.022297+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:46.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:46.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:46.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:46.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:46.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:46.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:46.581 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:46.611 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:46.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:46.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:46.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:46.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:46.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:46.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:46.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:46.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:47.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:47.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:47.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:46 smithi016 bash[75179]: cluster 2023-10-09T09:04:45.835179+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:46 smithi016 bash[75179]: cluster 2023-10-09T09:04:45.836642+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:46 smithi016 bash[75179]: cluster 2023-10-09T09:04:45.837810+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:46 smithi016 bash[75179]: cluster 2023-10-09T09:04:45.867692+0000 mgr.y (mgr.25421) 817 : cluster 0 pgmap v850: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 14616/87849 objects degraded (16.638%); 5144/87849 objects misplaced (5.856%); 3.4 MiB/s, 159 objects/s recovering 2023-10-09T09:04:47.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:47 smithi016 bash[77892]: cluster 2023-10-09T09:04:45.835179+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:47 smithi016 bash[77892]: cluster 2023-10-09T09:04:45.836642+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:47 smithi016 bash[77892]: cluster 2023-10-09T09:04:45.837810+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:47 smithi016 bash[77892]: cluster 2023-10-09T09:04:45.867692+0000 mgr.y (mgr.25421) 817 : cluster 0 pgmap v850: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 14616/87849 objects degraded (16.638%); 5144/87849 objects misplaced (5.856%); 3.4 MiB/s, 159 objects/s recovering 2023-10-09T09:04:47.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:47.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:47.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:46 smithi094 bash[59677]: cluster 2023-10-09T09:04:45.835179+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:46 smithi094 bash[59677]: cluster 2023-10-09T09:04:45.836642+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:46 smithi094 bash[59677]: cluster 2023-10-09T09:04:45.837810+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T09:04:47.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:46 smithi094 bash[59677]: cluster 2023-10-09T09:04:45.867692+0000 mgr.y (mgr.25421) 817 : cluster 0 pgmap v850: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 14616/87849 objects degraded (16.638%); 5144/87849 objects misplaced (5.856%); 3.4 MiB/s, 159 objects/s recovering 2023-10-09T09:04:47.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:47.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:47.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:47.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:47.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:47.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:47.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:47.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:47.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:47.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:47.788 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:47.813 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:47.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:47.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:47.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:48.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:48.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:48.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:48.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:48.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:48.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:48.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:48.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:48.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:48.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:48.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:48.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:48.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:48.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:48.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:48.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:48.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:48.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:48.750 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:48.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:48.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:48.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:48.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:48.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:48.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:48 smithi094 bash[59677]: cluster 2023-10-09T09:04:47.868417+0000 mgr.y (mgr.25421) 818 : cluster 0 pgmap v851: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 730 op/s; 14616/84198 objects degraded (17.359%); 5021/84198 objects misplaced (5.963%); 4.0 MiB/s, 122 objects/s recovering 2023-10-09T09:04:49.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:49.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:49.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:49.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:49.158 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:49.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:48 smithi016 bash[75179]: cluster 2023-10-09T09:04:47.868417+0000 mgr.y (mgr.25421) 818 : cluster 0 pgmap v851: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 730 op/s; 14616/84198 objects degraded (17.359%); 5021/84198 objects misplaced (5.963%); 4.0 MiB/s, 122 objects/s recovering 2023-10-09T09:04:49.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:48 smithi016 bash[77892]: cluster 2023-10-09T09:04:47.868417+0000 mgr.y (mgr.25421) 818 : cluster 0 pgmap v851: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 730 op/s; 14616/84198 objects degraded (17.359%); 5021/84198 objects misplaced (5.963%); 4.0 MiB/s, 122 objects/s recovering 2023-10-09T09:04:49.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:49.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:49.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:49.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:49.394 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:49.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:49.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:49.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:49.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:49.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:49.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:49.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:49.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:49.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:49.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:49.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:49.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:04:49.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:49.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:49.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:49.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:04:49.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:50.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:50.110 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:50.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:50.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:50 smithi016 bash[75179]: audit 2023-10-09T09:04:49.238378+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:50.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:50 smithi016 bash[75179]: audit 2023-10-09T09:04:49.240063+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:50.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:50 smithi016 bash[75179]: cluster 2023-10-09T09:04:49.559080+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 14616/84198 objects degraded (17.359%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:50.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:50 smithi016 bash[77892]: audit 2023-10-09T09:04:49.238378+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:50.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:50 smithi016 bash[77892]: audit 2023-10-09T09:04:49.240063+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:50.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:50 smithi016 bash[77892]: cluster 2023-10-09T09:04:49.559080+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 14616/84198 objects degraded (17.359%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:50.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:50.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:50.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:50.376 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:50.387 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:50.395 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:50 smithi094 bash[59677]: audit 2023-10-09T09:04:49.238378+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:04:50.395 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:50 smithi094 bash[59677]: audit 2023-10-09T09:04:49.240063+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:04:50.395 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:50 smithi094 bash[59677]: cluster 2023-10-09T09:04:49.559080+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 14616/84198 objects degraded (17.359%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:50.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:50.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:50.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:50.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:50.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:50.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:50.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:50.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:50.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:50.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:50.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:50.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:50.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:51.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:51.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:51.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:51.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:51.193 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:51.219 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:51.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:51.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:51.324 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:51.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:51.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:51.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:51.455 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:51.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:51.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:51.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:51.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:51.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:51.714 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:51.722 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:51 smithi094 bash[59677]: cluster 2023-10-09T09:04:49.868956+0000 mgr.y (mgr.25421) 819 : cluster 0 pgmap v852: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 14527/83148 objects degraded (17.471%); 5021/83148 objects misplaced (6.039%); 4.0 MiB/s, 134 objects/s recovering 2023-10-09T09:04:51.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:51.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:51.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:51.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:51.848 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:51 smithi016 bash[75179]: cluster 2023-10-09T09:04:49.868956+0000 mgr.y (mgr.25421) 819 : cluster 0 pgmap v852: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 14527/83148 objects degraded (17.471%); 5021/83148 objects misplaced (6.039%); 4.0 MiB/s, 134 objects/s recovering 2023-10-09T09:04:51.848 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:51 smithi016 bash[77892]: cluster 2023-10-09T09:04:49.868956+0000 mgr.y (mgr.25421) 819 : cluster 0 pgmap v852: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 14527/83148 objects degraded (17.471%); 5021/83148 objects misplaced (6.039%); 4.0 MiB/s, 134 objects/s recovering 2023-10-09T09:04:51.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:51.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:51.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:52.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:52.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:52.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:52.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:52.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:52.275 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:52.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:52.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:52.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:52.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:52.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:52.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:52.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:52.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:52.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:52.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:52.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:52.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:52.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:52.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:53.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:53.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:53.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:53.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:53.273 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:04:53.273 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:04:53.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:53.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:53.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:53.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:53.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:53.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:53.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:53.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:53.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:53.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:53 smithi094 bash[59677]: cluster 2023-10-09T09:04:51.869688+0000 mgr.y (mgr.25421) 820 : cluster 0 pgmap v853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 581 op/s; 14634/77325 objects degraded (18.925%); 4489/77325 objects misplaced (5.805%); 1.6 MiB/s, 37 objects/s recovering 2023-10-09T09:04:53.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:53.848 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:53 smithi016 bash[75179]: cluster 2023-10-09T09:04:51.869688+0000 mgr.y (mgr.25421) 820 : cluster 0 pgmap v853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 581 op/s; 14634/77325 objects degraded (18.925%); 4489/77325 objects misplaced (5.805%); 1.6 MiB/s, 37 objects/s recovering 2023-10-09T09:04:53.849 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:53 smithi016 bash[77892]: cluster 2023-10-09T09:04:51.869688+0000 mgr.y (mgr.25421) 820 : cluster 0 pgmap v853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 581 op/s; 14634/77325 objects degraded (18.925%); 4489/77325 objects misplaced (5.805%); 1.6 MiB/s, 37 objects/s recovering 2023-10-09T09:04:53.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:53.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:53.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:53.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:54.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:54.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:54.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:54.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:54.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:54.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:54.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:54.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:54.358 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:54.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:54.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:54.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:54.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:54.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:54.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:54.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:54.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:54.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:54.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:54.738 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:54.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:54 smithi016 bash[75179]: cluster 2023-10-09T09:04:54.560400+0000 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 14634/77325 objects degraded (18.925%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:54.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:54 smithi016 bash[77892]: cluster 2023-10-09T09:04:54.560400+0000 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 14634/77325 objects degraded (18.925%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:54.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:54.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:54.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:54.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:54.878 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:54 smithi094 bash[59677]: cluster 2023-10-09T09:04:54.560400+0000 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 14634/77325 objects degraded (18.925%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:04:54.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:54.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:54.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:55.000 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:55.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:55.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:55.171 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:55.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:55.278 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:55.306 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:55.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:55.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:55.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:55.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:55.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:04:55.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:55.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:55.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:55.700 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:55.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:55 smithi016 bash[75179]: cluster 2023-10-09T09:04:53.870280+0000 mgr.y (mgr.25421) 821 : cluster 0 pgmap v854: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 585 op/s; 14634/73809 objects degraded (19.827%); 4448/73809 objects misplaced (6.026%); 1.3 MiB/s, 30 objects/s recovering 2023-10-09T09:04:55.740 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:04:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:04:55] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.33.4" 2023-10-09T09:04:55.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:55 smithi016 bash[77892]: cluster 2023-10-09T09:04:53.870280+0000 mgr.y (mgr.25421) 821 : cluster 0 pgmap v854: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 585 op/s; 14634/73809 objects degraded (19.827%); 4448/73809 objects misplaced (6.026%); 1.3 MiB/s, 30 objects/s recovering 2023-10-09T09:04:55.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:55.811 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:55 smithi094 bash[59677]: cluster 2023-10-09T09:04:53.870280+0000 mgr.y (mgr.25421) 821 : cluster 0 pgmap v854: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 585 op/s; 14634/73809 objects degraded (19.827%); 4448/73809 objects misplaced (6.026%); 1.3 MiB/s, 30 objects/s recovering 2023-10-09T09:04:55.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:55.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:55.918 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:55.919 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:55.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:55.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:55.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:56.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:56.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:56.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:56.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:56.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:56.239 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:56.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:56.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:56.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:56.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:56.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:56.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:56.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:56.619 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:56.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:56.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:56.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:56.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:56.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:56.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:56.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:56.878 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:56 smithi016 bash[75179]: cluster 2023-10-09T09:04:55.870947+0000 mgr.y (mgr.25421) 822 : cluster 0 pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 659 op/s; 14747/69663 objects degraded (21.169%); 3849/69663 objects misplaced (5.525%); 1.7 MiB/s, 35 objects/s recovering 2023-10-09T09:04:56.878 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:56 smithi016 bash[77892]: cluster 2023-10-09T09:04:55.870947+0000 mgr.y (mgr.25421) 822 : cluster 0 pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 659 op/s; 14747/69663 objects degraded (21.169%); 3849/69663 objects misplaced (5.525%); 1.7 MiB/s, 35 objects/s recovering 2023-10-09T09:04:56.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:56.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:56.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:56.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:56 smithi094 bash[59677]: cluster 2023-10-09T09:04:55.870947+0000 mgr.y (mgr.25421) 822 : cluster 0 pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 659 op/s; 14747/69663 objects degraded (21.169%); 3849/69663 objects misplaced (5.525%); 1.7 MiB/s, 35 objects/s recovering 2023-10-09T09:04:57.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:57.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:57.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:57.133 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:57.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:57.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:57.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:57.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:57.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:57.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:57.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:57.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:57.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:57.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:57.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:57.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:57.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:57.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:57.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:57.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:58.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:58.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:58.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:58.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:58.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:58.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:58.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:58.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:58.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:58.359 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:58.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:58.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:58.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:04:58.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:58.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:04:58.592 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:58.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:58.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:58.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:58.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:58.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:58.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:58.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:58.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:58.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:58.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:04:58.933 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:58 smithi094 bash[59677]: cluster 2023-10-09T09:04:57.871602+0000 mgr.y (mgr.25421) 823 : cluster 0 pgmap v856: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 14747/65331 objects degraded (22.573%); 3795/65331 objects misplaced (5.809%); 1.2 MiB/s, 26 objects/s recovering 2023-10-09T09:04:58.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:58.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:58 smithi016 bash[75179]: cluster 2023-10-09T09:04:57.871602+0000 mgr.y (mgr.25421) 823 : cluster 0 pgmap v856: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 14747/65331 objects degraded (22.573%); 3795/65331 objects misplaced (5.809%); 1.2 MiB/s, 26 objects/s recovering 2023-10-09T09:04:59.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:58 smithi016 bash[77892]: cluster 2023-10-09T09:04:57.871602+0000 mgr.y (mgr.25421) 823 : cluster 0 pgmap v856: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 14747/65331 objects degraded (22.573%); 3795/65331 objects misplaced (5.809%); 1.2 MiB/s, 26 objects/s recovering 2023-10-09T09:04:59.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:04:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:04:59.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:59.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:04:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:04:59.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:59.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:59.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:59.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:59.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:04:59.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:04:59.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:59.548 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:04:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:04:59.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:59.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:04:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:04:59.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:59.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:04:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:04:59.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:59.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:04:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:04:59.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:00.011 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:00.052 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:00.083 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:00.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:00.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:00.197 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:04:59 smithi094 bash[59677]: cluster 2023-10-09T09:04:59.561257+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 14747/65331 objects degraded (22.573%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:00.212 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:00.217 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:04:59 smithi016 bash[75179]: cluster 2023-10-09T09:04:59.561257+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 14747/65331 objects degraded (22.573%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:00.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:04:59 smithi016 bash[77892]: cluster 2023-10-09T09:04:59.561257+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 14747/65331 objects degraded (22.573%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:00.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:00.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:00.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:00.441 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:00.467 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:05:00.532 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:05:00.569 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:00.592 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:00.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:00.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:00.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:00.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:00.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:00.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:00.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:01.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:01.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:01.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:01.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:01.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:01.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:01.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:01.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:01.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:01.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:01.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:01.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:01.506 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:01.533 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:01.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:01.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:01.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:01 smithi016 bash[75179]: cluster 2023-10-09T09:04:59.876472+0000 mgr.y (mgr.25421) 824 : cluster 0 pgmap v857: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 559 op/s; 14747/64059 objects degraded (23.021%); 3795/64059 objects misplaced (5.924%); 730 KiB/s, 18 objects/s recovering 2023-10-09T09:05:01.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:01.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:01.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:01.772 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:01.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:01 smithi094 bash[59677]: cluster 2023-10-09T09:04:59.876472+0000 mgr.y (mgr.25421) 824 : cluster 0 pgmap v857: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 559 op/s; 14747/64059 objects degraded (23.021%); 3795/64059 objects misplaced (5.924%); 730 KiB/s, 18 objects/s recovering 2023-10-09T09:05:01.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:01 smithi094 bash[59677]: audit 2023-10-09T09:05:00.506484+0000 mgr.y (mgr.25421) 825 : audit 0 from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:05:01.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:01.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:01.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:01.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:01 smithi016 bash[75179]: audit 2023-10-09T09:05:00.506484+0000 mgr.y (mgr.25421) 825 : audit 0 from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:05:01.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:01 smithi016 bash[77892]: cluster 2023-10-09T09:04:59.876472+0000 mgr.y (mgr.25421) 824 : cluster 0 pgmap v857: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 559 op/s; 14747/64059 objects degraded (23.021%); 3795/64059 objects misplaced (5.924%); 730 KiB/s, 18 objects/s recovering 2023-10-09T09:05:01.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:01 smithi016 bash[77892]: audit 2023-10-09T09:05:00.506484+0000 mgr.y (mgr.25421) 825 : audit 0 from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:05:01.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:01.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:01.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:02.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:02.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:02.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:02.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:02.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:02.326 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:02.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:02.457 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:02.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:02.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:02.701 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:02.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:02.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:02.857 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:02.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:02.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:02.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:03.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:03.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:03.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:03.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:03.226 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:03.255 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:03.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:03.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:03.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:03.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:03.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:03.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:03.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:03.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:03.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:03.614 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:03.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:03.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:03.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:03.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:03 smithi016 bash[75179]: cluster 2023-10-09T09:05:01.877239+0000 mgr.y (mgr.25421) 826 : cluster 0 pgmap v858: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 716 op/s; 14875/57327 objects degraded (25.948%); 3123/57327 objects misplaced (5.448%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:05:03.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:03 smithi016 bash[77892]: cluster 2023-10-09T09:05:01.877239+0000 mgr.y (mgr.25421) 826 : cluster 0 pgmap v858: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 716 op/s; 14875/57327 objects degraded (25.948%); 3123/57327 objects misplaced (5.448%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:05:03.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:03.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:03.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:03.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:03.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:03.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:03.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:03.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:03 smithi094 bash[59677]: cluster 2023-10-09T09:05:01.877239+0000 mgr.y (mgr.25421) 826 : cluster 0 pgmap v858: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 716 op/s; 14875/57327 objects degraded (25.948%); 3123/57327 objects misplaced (5.448%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:05:04.005 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:04.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:04.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:04.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:04.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:04.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:04.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:04.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:04.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:04.394 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:04.419 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:04.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:04.522 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:04.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:04.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:04.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:04.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:04.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:04.797 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:04 smithi094 bash[59677]: audit 2023-10-09T09:05:04.234036+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:04.797 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:04 smithi094 bash[59677]: cluster 2023-10-09T09:05:04.562369+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 14875/57327 objects degraded (25.948%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:04.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:04.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:04.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:04.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:04.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:04.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:04 smithi016 bash[75179]: audit 2023-10-09T09:05:04.234036+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:04 smithi016 bash[75179]: cluster 2023-10-09T09:05:04.562369+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 14875/57327 objects degraded (25.948%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:04.958 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:04 smithi016 bash[77892]: audit 2023-10-09T09:05:04.234036+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:04.958 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:04 smithi016 bash[77892]: cluster 2023-10-09T09:05:04.562369+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 14875/57327 objects degraded (25.948%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:05.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:05.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:05.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:05.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:05.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:05.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:05.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:05.249 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":857,"stamp":"2023-10-09T09:04:59.876026+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1398347704,"num_objects":21353,"num_object_clones":0,"num_object_copies":64059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14747,"num_objects_misplaced":3795,"num_objects_unfound":0,"num_objects_dirty":21353,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":155405,"num_write_kb":5649550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17848,"num_bytes_recovered":1007566158,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85498,"ondisk_log_size":85498,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":283,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":7073104,"kb_used_data":4836904,"kb_used_omap":117,"kb_used_meta":2235914,"kb_avail":461652656,"statfs":{"total":479975178240,"available":472732319744,"internally_reserved":0,"allocated":4952989696,"data_stored":4949588195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":2289576594},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-439943168,"num_objects":-6713,"num_object_clones":0,"num_object_copies":-20139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":131,"num_objects_misplaced":-1226,"num_objects_unfound":0,"num_objects_dirty":-6713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6713,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":227,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.008124"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":65,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945529+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:04:43.945529+0000","last_peered":"2023-10-09T09:04:43.945529+0000","last_clean":"2023-10-09T09:04:43.945529+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:04:43.945529+0000","last_undegraded":"2023-10-09T09:04:43.945529+0000","last_fullsized":"2023-10-09T09:04:43.945529+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":214,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:44.952197+0000","last_change":"2023-10-09T09:02:02.625626+0000","last_active":"2023-10-09T09:04:44.952197+0000","last_peered":"2023-10-09T09:04:44.952197+0000","last_clean":"2023-10-09T09:04:44.952197+0000","last_became_active":"2023-10-09T09:02:02.625312+0000","last_became_peered":"2023-10-09T09:02:02.625312+0000","last_unstale":"2023-10-09T09:04:44.952197+0000","last_undegraded":"2023-10-09T09:04:44.952197+0000","last_fullsized":"2023-10-09T09:04:44.952197+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:01.640943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":167,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956053+0000","last_change":"2023-10-09T09:02:02.626274+0000","last_active":"2023-10-09T09:04:43.956053+0000","last_peered":"2023-10-09T09:04:43.956053+0000","last_clean":"2023-10-09T09:04:43.956053+0000","last_became_active":"2023-10-09T09:02:02.625973+0000","last_became_peered":"2023-10-09T09:02:02.625973+0000","last_unstale":"2023-10-09T09:04:43.956053+0000","last_undegraded":"2023-10-09T09:04:43.956053+0000","last_fullsized":"2023-10-09T09:04:43.956053+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:44.223794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":214,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.965225+0000","last_change":"2023-10-09T09:03:19.554441+0000","last_active":"2023-10-09T09:04:43.965225+0000","last_peered":"2023-10-09T09:04:43.965225+0000","last_clean":"2023-10-09T09:04:43.965225+0000","last_became_active":"2023-10-09T09:03:19.553967+0000","last_became_peered":"2023-10-09T09:03:19.553967+0000","last_unstale":"2023-10-09T09:04:43.965225+0000","last_undegraded":"2023-10-09T09:04:43.965225+0000","last_fullsized":"2023-10-09T09:04:43.965225+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:26.065993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":203,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956280+0000","last_change":"2023-10-09T09:02:02.656821+0000","last_active":"2023-10-09T09:04:43.956280+0000","last_peered":"2023-10-09T09:04:43.956280+0000","last_clean":"2023-10-09T09:04:43.956280+0000","last_became_active":"2023-10-09T09:02:02.656523+0000","last_became_peered":"2023-10-09T09:02:02.656523+0000","last_unstale":"2023-10-09T09:04:43.956280+0000","last_undegraded":"2023-10-09T09:04:43.956280+0000","last_fullsized":"2023-10-09T09:04:43.956280+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:21.635518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952744+0000","last_change":"2023-10-09T09:03:19.582769+0000","last_active":"2023-10-09T09:04:43.952744+0000","last_peered":"2023-10-09T09:04:43.952744+0000","last_clean":"2023-10-09T09:04:43.952744+0000","last_became_active":"2023-10-09T09:03:19.577023+0000","last_became_peered":"2023-10-09T09:03:19.577023+0000","last_unstale":"2023-10-09T09:04:43.952744+0000","last_undegraded":"2023-10-09T09:04:43.952744+0000","last_fullsized":"2023-10-09T09:04:43.952744+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":176,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.957527+0000","last_change":"2023-10-09T09:01:36.280403+0000","last_active":"2023-10-09T09:04:43.957527+0000","last_peered":"2023-10-09T09:04:43.957527+0000","last_clean":"2023-10-09T09:04:43.957527+0000","last_became_active":"2023-10-09T09:01:36.280070+0000","last_became_peered":"2023-10-09T09:01:36.280070+0000","last_unstale":"2023-10-09T09:04:43.957527+0000","last_undegraded":"2023-10-09T09:04:43.957527+0000","last_fullsized":"2023-10-09T09:04:43.957527+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:33.796899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":214,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956354+0000","last_change":"2023-10-09T09:02:02.630162+0000","last_active":"2023-10-09T09:04:43.956354+0000","last_peered":"2023-10-09T09:04:43.956354+0000","last_clean":"2023-10-09T09:04:43.956354+0000","last_became_active":"2023-10-09T09:02:02.629768+0000","last_became_peered":"2023-10-09T09:02:02.629768+0000","last_unstale":"2023-10-09T09:04:43.956354+0000","last_undegraded":"2023-10-09T09:04:43.956354+0000","last_fullsized":"2023-10-09T09:04:43.956354+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:36.859890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.969459+0000","last_change":"2023-10-09T09:03:19.562130+0000","last_active":"2023-10-09T09:04:43.969459+0000","last_peered":"2023-10-09T09:04:43.969459+0000","last_clean":"2023-10-09T09:04:43.969459+0000","last_became_active":"2023-10-09T09:03:19.557757+0000","last_became_peered":"2023-10-09T09:03:19.557757+0000","last_unstale":"2023-10-09T09:04:43.969459+0000","last_undegraded":"2023-10-09T09:04:43.969459+0000","last_fullsized":"2023-10-09T09:04:43.969459+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:46.546761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952215+0000","last_change":"2023-10-09T09:01:36.279247+0000","last_active":"2023-10-09T09:04:43.952215+0000","last_peered":"2023-10-09T09:04:43.952215+0000","last_clean":"2023-10-09T09:04:43.952215+0000","last_became_active":"2023-10-09T09:01:36.278376+0000","last_became_peered":"2023-10-09T09:01:36.278376+0000","last_unstale":"2023-10-09T09:04:43.952215+0000","last_undegraded":"2023-10-09T09:04:43.952215+0000","last_fullsized":"2023-10-09T09:04:43.952215+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:19.166567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":160,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.968842+0000","last_change":"2023-10-09T08:59:53.544302+0000","last_active":"2023-10-09T09:04:43.968842+0000","last_peered":"2023-10-09T09:04:43.968842+0000","last_clean":"2023-10-09T09:04:43.968842+0000","last_became_active":"2023-10-09T08:59:53.531404+0000","last_became_peered":"2023-10-09T08:59:53.531404+0000","last_unstale":"2023-10-09T09:04:43.968842+0000","last_undegraded":"2023-10-09T09:04:43.968842+0000","last_fullsized":"2023-10-09T09:04:43.968842+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:48.670745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":162,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952416+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:04:43.952416+0000","last_peered":"2023-10-09T09:04:43.952416+0000","last_clean":"2023-10-09T09:04:43.952416+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:04:43.952416+0000","last_undegraded":"2023-10-09T09:04:43.952416+0000","last_fullsized":"2023-10-09T09:04:43.952416+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":85,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945285+0000","last_change":"2023-10-09T09:00:31.867387+0000","last_active":"2023-10-09T09:04:43.945285+0000","last_peered":"2023-10-09T09:04:43.945285+0000","last_clean":"2023-10-09T09:04:43.945285+0000","last_became_active":"2023-10-09T09:00:31.859752+0000","last_became_peered":"2023-10-09T09:00:31.859752+0000","last_unstale":"2023-10-09T09:04:43.945285+0000","last_undegraded":"2023-10-09T09:04:43.945285+0000","last_fullsized":"2023-10-09T09:04:43.945285+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":74,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945133+0000","last_change":"2023-10-09T09:02:02.630025+0000","last_active":"2023-10-09T09:04:43.945133+0000","last_peered":"2023-10-09T09:04:43.945133+0000","last_clean":"2023-10-09T09:04:43.945133+0000","last_became_active":"2023-10-09T09:02:02.629474+0000","last_became_peered":"2023-10-09T09:02:02.629474+0000","last_unstale":"2023-10-09T09:04:43.945133+0000","last_undegraded":"2023-10-09T09:04:43.945133+0000","last_fullsized":"2023-10-09T09:04:43.945133+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":77,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945382+0000","last_change":"2023-10-09T09:00:31.959283+0000","last_active":"2023-10-09T09:04:43.945382+0000","last_peered":"2023-10-09T09:04:43.945382+0000","last_clean":"2023-10-09T09:04:43.945382+0000","last_became_active":"2023-10-09T09:00:31.861638+0000","last_became_peered":"2023-10-09T09:00:31.861638+0000","last_unstale":"2023-10-09T09:04:43.945382+0000","last_undegraded":"2023-10-09T09:04:43.945382+0000","last_fullsized":"2023-10-09T09:04:43.945382+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":196,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.968737+0000","last_change":"2023-10-09T09:02:02.636035+0000","last_active":"2023-10-09T09:04:43.968737+0000","last_peered":"2023-10-09T09:04:43.968737+0000","last_clean":"2023-10-09T09:04:43.968737+0000","last_became_active":"2023-10-09T09:02:02.635621+0000","last_became_peered":"2023-10-09T09:02:02.635621+0000","last_unstale":"2023-10-09T09:04:43.968737+0000","last_undegraded":"2023-10-09T09:04:43.968737+0000","last_fullsized":"2023-10-09T09:04:43.968737+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:46.294650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":204,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956830+0000","last_change":"2023-10-09T09:03:19.564077+0000","last_active":"2023-10-09T09:04:43.956830+0000","last_peered":"2023-10-09T09:04:43.956830+0000","last_clean":"2023-10-09T09:04:43.956830+0000","last_became_active":"2023-10-09T09:03:19.563764+0000","last_became_peered":"2023-10-09T09:03:19.563764+0000","last_unstale":"2023-10-09T09:04:43.956830+0000","last_undegraded":"2023-10-09T09:04:43.956830+0000","last_fullsized":"2023-10-09T09:04:43.956830+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:38.043450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":160,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.968670+0000","last_change":"2023-10-09T09:01:36.279054+0000","last_active":"2023-10-09T09:04:43.968670+0000","last_peered":"2023-10-09T09:04:43.968670+0000","last_clean":"2023-10-09T09:04:43.968670+0000","last_became_active":"2023-10-09T09:01:36.278755+0000","last_became_peered":"2023-10-09T09:01:36.278755+0000","last_unstale":"2023-10-09T09:04:43.968670+0000","last_undegraded":"2023-10-09T09:04:43.968670+0000","last_fullsized":"2023-10-09T09:04:43.968670+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:09.089065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":65,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945044+0000","last_change":"2023-10-09T09:00:31.851267+0000","last_active":"2023-10-09T09:04:43.945044+0000","last_peered":"2023-10-09T09:04:43.945044+0000","last_clean":"2023-10-09T09:04:43.945044+0000","last_became_active":"2023-10-09T09:00:31.850918+0000","last_became_peered":"2023-10-09T09:00:31.850918+0000","last_unstale":"2023-10-09T09:04:43.945044+0000","last_undegraded":"2023-10-09T09:04:43.945044+0000","last_fullsized":"2023-10-09T09:04:43.945044+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.965354+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T09:04:43.965354+0000","last_peered":"2023-10-09T09:04:43.965354+0000","last_clean":"2023-10-09T09:04:43.965354+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T09:04:43.965354+0000","last_undegraded":"2023-10-09T09:04:43.965354+0000","last_fullsized":"2023-10-09T09:04:43.965354+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":81,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952517+0000","last_change":"2023-10-09T09:01:36.301852+0000","last_active":"2023-10-09T09:04:43.952517+0000","last_peered":"2023-10-09T09:04:43.952517+0000","last_clean":"2023-10-09T09:04:43.952517+0000","last_became_active":"2023-10-09T09:01:36.284416+0000","last_became_peered":"2023-10-09T09:01:36.284416+0000","last_unstale":"2023-10-09T09:04:43.952517+0000","last_undegraded":"2023-10-09T09:04:43.952517+0000","last_fullsized":"2023-10-09T09:04:43.952517+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":20,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.984995+0000","last_change":"2023-10-09T09:03:19.575211+0000","last_active":"2023-10-09T09:04:43.984995+0000","last_peered":"2023-10-09T09:04:43.984995+0000","last_clean":"2023-10-09T09:04:43.984995+0000","last_became_active":"2023-10-09T09:03:19.574829+0000","last_became_peered":"2023-10-09T09:03:19.574829+0000","last_unstale":"2023-10-09T09:04:43.984995+0000","last_undegraded":"2023-10-09T09:04:43.984995+0000","last_fullsized":"2023-10-09T09:04:43.984995+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:14.763790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"203'9639","reported_seq":9677,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:55.336145+0000","last_change":"2023-10-09T09:01:36.277698+0000","last_active":"2023-10-09T09:04:55.336145+0000","last_peered":"2023-10-09T09:04:55.336145+0000","last_clean":"2023-10-09T09:04:55.336145+0000","last_became_active":"2023-10-09T09:01:36.276985+0000","last_became_peered":"2023-10-09T09:01:36.276985+0000","last_unstale":"2023-10-09T09:04:55.336145+0000","last_undegraded":"2023-10-09T09:04:55.336145+0000","last_fullsized":"2023-10-09T09:04:55.336145+0000","mapping_epoch":176,"log_start":"187'4100","ondisk_log_start":"187'4100","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91684864,"num_objects":1399,"num_object_clones":0,"num_object_copies":4197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9639,"num_write_kb":353216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":86,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945540+0000","last_change":"2023-10-09T09:03:19.626156+0000","last_active":"2023-10-09T09:04:43.945540+0000","last_peered":"2023-10-09T09:04:43.945540+0000","last_clean":"2023-10-09T09:04:43.945540+0000","last_became_active":"2023-10-09T09:03:19.575061+0000","last_became_peered":"2023-10-09T09:03:19.575061+0000","last_unstale":"2023-10-09T09:04:43.945540+0000","last_undegraded":"2023-10-09T09:04:43.945540+0000","last_fullsized":"2023-10-09T09:04:43.945540+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.953434+0000","last_change":"2023-10-09T09:03:19.582540+0000","last_active":"2023-10-09T09:04:43.953434+0000","last_peered":"2023-10-09T09:04:43.953434+0000","last_clean":"2023-10-09T09:04:43.953434+0000","last_became_active":"2023-10-09T09:03:19.577044+0000","last_became_peered":"2023-10-09T09:03:19.577044+0000","last_unstale":"2023-10-09T09:04:43.953434+0000","last_undegraded":"2023-10-09T09:04:43.953434+0000","last_fullsized":"2023-10-09T09:04:43.953434+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"203'9507","reported_seq":9546,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:56.036826+0000","last_change":"2023-10-09T09:01:16.814785+0000","last_active":"2023-10-09T09:04:56.036826+0000","last_peered":"2023-10-09T09:04:56.036826+0000","last_clean":"2023-10-09T09:04:56.036826+0000","last_became_active":"2023-10-09T09:01:16.814404+0000","last_became_peered":"2023-10-09T09:01:16.814404+0000","last_unstale":"2023-10-09T09:04:56.036826+0000","last_undegraded":"2023-10-09T09:04:56.036826+0000","last_fullsized":"2023-10-09T09:04:56.036826+0000","mapping_epoch":172,"log_start":"189'4300","ondisk_log_start":"189'4300","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5207,"log_dups_size":0,"ondisk_log_size":5207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9507,"num_write_kb":346560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":73,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945418+0000","last_change":"2023-10-09T09:02:02.724809+0000","last_active":"2023-10-09T09:04:43.945418+0000","last_peered":"2023-10-09T09:04:43.945418+0000","last_clean":"2023-10-09T09:04:43.945418+0000","last_became_active":"2023-10-09T09:02:02.666202+0000","last_became_peered":"2023-10-09T09:02:02.666202+0000","last_unstale":"2023-10-09T09:04:43.945418+0000","last_undegraded":"2023-10-09T09:04:43.945418+0000","last_fullsized":"2023-10-09T09:04:43.945418+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.944890+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:04:43.944890+0000","last_peered":"2023-10-09T09:04:43.944890+0000","last_clean":"2023-10-09T09:04:43.944890+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:04:43.944890+0000","last_undegraded":"2023-10-09T09:04:43.944890+0000","last_fullsized":"2023-10-09T09:04:43.944890+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"203'9813","reported_seq":9854,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:57.613463+0000","last_change":"2023-10-09T09:01:16.948996+0000","last_active":"2023-10-09T09:04:57.613463+0000","last_peered":"2023-10-09T09:04:57.613463+0000","last_clean":"2023-10-09T09:04:57.613463+0000","last_became_active":"2023-10-09T09:01:16.948583+0000","last_became_peered":"2023-10-09T09:01:16.948583+0000","last_unstale":"2023-10-09T09:04:57.613463+0000","last_undegraded":"2023-10-09T09:04:57.613463+0000","last_fullsized":"2023-10-09T09:04:57.613463+0000","mapping_epoch":172,"log_start":"187'4100","ondisk_log_start":"187'4100","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5713,"log_dups_size":0,"ondisk_log_size":5713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:21.355627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84213760,"num_objects":1285,"num_object_clones":0,"num_object_copies":3855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9813,"num_write_kb":355136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":112,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952990+0000","last_change":"2023-10-09T09:01:36.305615+0000","last_active":"2023-10-09T09:04:43.952990+0000","last_peered":"2023-10-09T09:04:43.952990+0000","last_clean":"2023-10-09T09:04:43.952990+0000","last_became_active":"2023-10-09T09:01:36.292828+0000","last_became_peered":"2023-10-09T09:01:36.292828+0000","last_unstale":"2023-10-09T09:04:43.952990+0000","last_undegraded":"2023-10-09T09:04:43.952990+0000","last_fullsized":"2023-10-09T09:04:43.952990+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952579+0000","last_change":"2023-10-09T09:03:19.564369+0000","last_active":"2023-10-09T09:04:43.952579+0000","last_peered":"2023-10-09T09:04:43.952579+0000","last_clean":"2023-10-09T09:04:43.952579+0000","last_became_active":"2023-10-09T09:03:19.563989+0000","last_became_peered":"2023-10-09T09:03:19.563989+0000","last_unstale":"2023-10-09T09:04:43.952579+0000","last_undegraded":"2023-10-09T09:04:43.952579+0000","last_fullsized":"2023-10-09T09:04:43.952579+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"203'9890","reported_seq":13448,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:55.594916+0000","last_change":"2023-10-09T09:02:52.356438+0000","last_active":"2023-10-09T09:04:55.594916+0000","last_peered":"2023-10-09T09:04:55.594916+0000","last_clean":"2023-10-09T09:02:01.626664+0000","last_became_active":"2023-10-09T09:02:52.345464+0000","last_became_peered":"2023-10-09T09:02:52.345464+0000","last_unstale":"2023-10-09T09:04:55.594916+0000","last_undegraded":"2023-10-09T09:02:52.049405+0000","last_fullsized":"2023-10-09T09:02:51.903806+0000","mapping_epoch":190,"log_start":"192'5186","ondisk_log_start":"192'5186","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4704,"log_dups_size":0,"ondisk_log_size":4704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:38.670622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90701824,"num_objects":1384,"num_object_clones":0,"num_object_copies":4152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5032,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9890,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1751,"num_bytes_recovered":114753536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"3,4,6","objects":779},{"shards":"3,6","objects":-3648}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.968910+0000","last_change":"2023-10-09T09:02:02.664617+0000","last_active":"2023-10-09T09:04:43.968910+0000","last_peered":"2023-10-09T09:04:43.968910+0000","last_clean":"2023-10-09T09:04:43.968910+0000","last_became_active":"2023-10-09T09:02:02.664305+0000","last_became_peered":"2023-10-09T09:02:02.664305+0000","last_unstale":"2023-10-09T09:04:43.968910+0000","last_undegraded":"2023-10-09T09:04:43.968910+0000","last_fullsized":"2023-10-09T09:04:43.968910+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:14.520177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":195,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956190+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:04:43.956190+0000","last_peered":"2023-10-09T09:04:43.956190+0000","last_clean":"2023-10-09T09:04:43.956190+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:04:43.956190+0000","last_undegraded":"2023-10-09T09:04:43.956190+0000","last_fullsized":"2023-10-09T09:04:43.956190+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"203'9520","reported_seq":14474,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:55.602783+0000","last_change":"2023-10-09T09:04:01.588490+0000","last_active":"2023-10-09T09:04:55.602783+0000","last_peered":"2023-10-09T09:04:55.602783+0000","last_clean":"2023-10-09T09:04:55.602783+0000","last_became_active":"2023-10-09T09:04:01.574284+0000","last_became_peered":"2023-10-09T09:04:01.574284+0000","last_unstale":"2023-10-09T09:04:55.602783+0000","last_undegraded":"2023-10-09T09:04:55.602783+0000","last_fullsized":"2023-10-09T09:04:55.602783+0000","mapping_epoch":198,"log_start":"191'4832","ondisk_log_start":"191'4832","created":172,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:13.836976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89128960,"num_objects":1360,"num_object_clones":0,"num_object_copies":4080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9520,"num_write_kb":348160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2441,"num_bytes_recovered":155975680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":203,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.970418+0000","last_change":"2023-10-09T08:59:53.642095+0000","last_active":"2023-10-09T09:04:43.970418+0000","last_peered":"2023-10-09T09:04:43.970418+0000","last_clean":"2023-10-09T09:04:43.970418+0000","last_became_active":"2023-10-09T08:59:53.531588+0000","last_became_peered":"2023-10-09T08:59:53.531588+0000","last_unstale":"2023-10-09T09:04:43.970418+0000","last_undegraded":"2023-10-09T09:04:43.970418+0000","last_fullsized":"2023-10-09T09:04:43.970418+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.878230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":65,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945119+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:04:43.945119+0000","last_peered":"2023-10-09T09:04:43.945119+0000","last_clean":"2023-10-09T09:04:43.945119+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:04:43.945119+0000","last_undegraded":"2023-10-09T09:04:43.945119+0000","last_fullsized":"2023-10-09T09:04:43.945119+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"203'9778","reported_seq":18431,"reported_epoch":203,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:04:57.535669+0000","last_change":"2023-10-09T09:04:43.869830+0000","last_active":"2023-10-09T09:04:57.535669+0000","last_peered":"2023-10-09T09:04:57.535669+0000","last_clean":"2023-10-09T09:02:01.580217+0000","last_became_active":"2023-10-09T09:03:20.662001+0000","last_became_peered":"2023-10-09T09:03:20.662001+0000","last_unstale":"2023-10-09T09:04:57.535669+0000","last_undegraded":"2023-10-09T09:04:57.535669+0000","last_fullsized":"2023-10-09T09:04:57.535669+0000","mapping_epoch":194,"log_start":"187'4100","ondisk_log_start":"187'4100","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:29.131956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83492864,"num_objects":1274,"num_object_clones":0,"num_object_copies":3822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":267,"num_objects_unfound":0,"num_objects_dirty":1274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9778,"num_write_kb":353664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4291,"num_bytes_recovered":281214976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,6],"avail_no_missing":["0","1","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1274}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952601+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:04:43.952601+0000","last_peered":"2023-10-09T09:04:43.952601+0000","last_clean":"2023-10-09T09:04:43.952601+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:04:43.952601+0000","last_undegraded":"2023-10-09T09:04:43.952601+0000","last_fullsized":"2023-10-09T09:04:43.952601+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":65,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945786+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:04:43.945786+0000","last_peered":"2023-10-09T09:04:43.945786+0000","last_clean":"2023-10-09T09:04:43.945786+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:04:43.945786+0000","last_undegraded":"2023-10-09T09:04:43.945786+0000","last_fullsized":"2023-10-09T09:04:43.945786+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"203'9810","reported_seq":9868,"reported_epoch":203,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:04:55.355240+0000","last_change":"2023-10-09T09:03:20.697136+0000","last_active":"2023-10-09T09:04:55.355240+0000","last_peered":"2023-10-09T09:04:55.355240+0000","last_clean":"2023-10-09T09:03:19.287407+0000","last_became_active":"2023-10-09T09:03:20.665776+0000","last_became_peered":"2023-10-09T09:03:20.665776+0000","last_unstale":"2023-10-09T09:04:55.355240+0000","last_undegraded":"2023-10-09T09:04:55.355240+0000","last_fullsized":"2023-10-09T09:04:55.355240+0000","mapping_epoch":194,"log_start":"187'4300","ondisk_log_start":"187'4300","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91357184,"num_objects":1394,"num_object_clones":0,"num_object_copies":4182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1394,"num_objects_unfound":0,"num_objects_dirty":1394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9810,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":1394}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":74,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.944906+0000","last_change":"2023-10-09T09:03:19.563963+0000","last_active":"2023-10-09T09:04:43.944906+0000","last_peered":"2023-10-09T09:04:43.944906+0000","last_clean":"2023-10-09T09:04:43.944906+0000","last_became_active":"2023-10-09T09:03:19.563629+0000","last_became_peered":"2023-10-09T09:03:19.563629+0000","last_unstale":"2023-10-09T09:04:43.944906+0000","last_undegraded":"2023-10-09T09:04:43.944906+0000","last_fullsized":"2023-10-09T09:04:43.944906+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":74,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945884+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:04:43.945884+0000","last_peered":"2023-10-09T09:04:43.945884+0000","last_clean":"2023-10-09T09:04:43.945884+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:04:43.945884+0000","last_undegraded":"2023-10-09T09:04:43.945884+0000","last_fullsized":"2023-10-09T09:04:43.945884+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"203'9568","reported_seq":14486,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:56.037198+0000","last_change":"2023-10-09T09:02:38.910227+0000","last_active":"2023-10-09T09:04:56.037198+0000","last_peered":"2023-10-09T09:04:56.037198+0000","last_clean":"2023-10-09T09:04:56.037198+0000","last_became_active":"2023-10-09T09:02:38.903339+0000","last_became_peered":"2023-10-09T09:02:38.903339+0000","last_unstale":"2023-10-09T09:04:56.037198+0000","last_undegraded":"2023-10-09T09:04:56.037198+0000","last_fullsized":"2023-10-09T09:04:56.037198+0000","mapping_epoch":188,"log_start":"189'4300","ondisk_log_start":"189'4300","created":172,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87425024,"num_objects":1334,"num_object_clones":0,"num_object_copies":4002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9568,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2426,"num_bytes_recovered":158990336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.954900+0000","last_change":"2023-10-09T09:03:19.572389+0000","last_active":"2023-10-09T09:04:43.954900+0000","last_peered":"2023-10-09T09:04:43.954900+0000","last_clean":"2023-10-09T09:04:43.954900+0000","last_became_active":"2023-10-09T09:03:19.572091+0000","last_became_peered":"2023-10-09T09:03:19.572091+0000","last_unstale":"2023-10-09T09:04:43.954900+0000","last_undegraded":"2023-10-09T09:04:43.954900+0000","last_fullsized":"2023-10-09T09:04:43.954900+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:27.702718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":158,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:44.000143+0000","last_change":"2023-10-09T09:02:02.636996+0000","last_active":"2023-10-09T09:04:44.000143+0000","last_peered":"2023-10-09T09:04:44.000143+0000","last_clean":"2023-10-09T09:04:44.000143+0000","last_became_active":"2023-10-09T09:02:02.636702+0000","last_became_peered":"2023-10-09T09:02:02.636702+0000","last_unstale":"2023-10-09T09:04:44.000143+0000","last_undegraded":"2023-10-09T09:04:44.000143+0000","last_fullsized":"2023-10-09T09:04:44.000143+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:45.392645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"203'9577","reported_seq":9618,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:57.586811+0000","last_change":"2023-10-09T09:01:16.949929+0000","last_active":"2023-10-09T09:04:57.586811+0000","last_peered":"2023-10-09T09:04:57.586811+0000","last_clean":"2023-10-09T09:04:57.586811+0000","last_became_active":"2023-10-09T09:01:16.949317+0000","last_became_peered":"2023-10-09T09:01:16.949317+0000","last_unstale":"2023-10-09T09:04:57.586811+0000","last_undegraded":"2023-10-09T09:04:57.586811+0000","last_fullsized":"2023-10-09T09:04:57.586811+0000","mapping_epoch":172,"log_start":"187'3900","ondisk_log_start":"187'3900","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:27.017754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9577,"num_write_kb":347072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":145,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.957138+0000","last_change":"2023-10-09T09:03:19.556450+0000","last_active":"2023-10-09T09:04:43.957138+0000","last_peered":"2023-10-09T09:04:43.957138+0000","last_clean":"2023-10-09T09:04:43.957138+0000","last_became_active":"2023-10-09T09:03:19.556190+0000","last_became_peered":"2023-10-09T09:03:19.556190+0000","last_unstale":"2023-10-09T09:04:43.957138+0000","last_undegraded":"2023-10-09T09:04:43.957138+0000","last_fullsized":"2023-10-09T09:04:43.957138+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:23.704455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.968926+0000","last_change":"2023-10-09T09:01:36.285844+0000","last_active":"2023-10-09T09:04:43.968926+0000","last_peered":"2023-10-09T09:04:43.968926+0000","last_clean":"2023-10-09T09:04:43.968926+0000","last_became_active":"2023-10-09T09:01:36.285525+0000","last_became_peered":"2023-10-09T09:01:36.285525+0000","last_unstale":"2023-10-09T09:04:43.968926+0000","last_undegraded":"2023-10-09T09:04:43.968926+0000","last_fullsized":"2023-10-09T09:04:43.968926+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:05.624756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945680+0000","last_change":"2023-10-09T09:02:52.021749+0000","last_active":"2023-10-09T09:04:43.945680+0000","last_peered":"2023-10-09T09:04:43.945680+0000","last_clean":"2023-10-09T09:04:43.945680+0000","last_became_active":"2023-10-09T09:02:52.021416+0000","last_became_peered":"2023-10-09T09:02:52.021416+0000","last_unstale":"2023-10-09T09:04:43.945680+0000","last_undegraded":"2023-10-09T09:04:43.945680+0000","last_fullsized":"2023-10-09T09:04:43.945680+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"203'9626","reported_seq":9673,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:55.618737+0000","last_change":"2023-10-09T09:02:53.102143+0000","last_active":"2023-10-09T09:04:55.618737+0000","last_peered":"2023-10-09T09:04:55.618737+0000","last_clean":"2023-10-09T09:02:50.898920+0000","last_became_active":"2023-10-09T09:02:53.090937+0000","last_became_peered":"2023-10-09T09:02:53.090937+0000","last_unstale":"2023-10-09T09:04:55.618737+0000","last_undegraded":"2023-10-09T09:02:52.940746+0000","last_fullsized":"2023-10-09T09:02:52.912781+0000","mapping_epoch":191,"log_start":"192'5001","ondisk_log_start":"192'5001","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4625,"log_dups_size":0,"ondisk_log_size":4625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:08.317143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90832896,"num_objects":1386,"num_object_clones":0,"num_object_copies":4158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9626,"num_write_kb":352384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3","objects":-3518},{"shards":"0,3,4","objects":784}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":170,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952306+0000","last_change":"2023-10-09T09:03:19.564803+0000","last_active":"2023-10-09T09:04:43.952306+0000","last_peered":"2023-10-09T09:04:43.952306+0000","last_clean":"2023-10-09T09:04:43.952306+0000","last_became_active":"2023-10-09T09:03:19.564520+0000","last_became_peered":"2023-10-09T09:03:19.564520+0000","last_unstale":"2023-10-09T09:04:43.952306+0000","last_undegraded":"2023-10-09T09:04:43.952306+0000","last_fullsized":"2023-10-09T09:04:43.952306+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":65,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945817+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:04:43.945817+0000","last_peered":"2023-10-09T09:04:43.945817+0000","last_clean":"2023-10-09T09:04:43.945817+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:04:43.945817+0000","last_undegraded":"2023-10-09T09:04:43.945817+0000","last_fullsized":"2023-10-09T09:04:43.945817+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":169,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.946987+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:04:43.946987+0000","last_peered":"2023-10-09T09:04:43.946987+0000","last_clean":"2023-10-09T09:04:43.946987+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:04:43.946987+0000","last_undegraded":"2023-10-09T09:04:43.946987+0000","last_fullsized":"2023-10-09T09:04:43.946987+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"203'9707","reported_seq":14831,"reported_epoch":203,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:04:55.324576+0000","last_change":"2023-10-09T09:04:20.760516+0000","last_active":"2023-10-09T09:04:55.324576+0000","last_peered":"2023-10-09T09:04:55.324576+0000","last_clean":"2023-10-09T09:02:01.574862+0000","last_became_active":"2023-10-09T09:03:20.671903+0000","last_became_peered":"2023-10-09T09:03:20.671903+0000","last_unstale":"2023-10-09T09:04:55.324576+0000","last_undegraded":"2023-10-09T09:04:55.324576+0000","last_fullsized":"2023-10-09T09:04:55.324576+0000","mapping_epoch":194,"log_start":"187'4200","ondisk_log_start":"187'4200","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5507,"log_dups_size":0,"ondisk_log_size":5507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90112000,"num_objects":1375,"num_object_clones":0,"num_object_copies":4125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1375,"num_objects_unfound":0,"num_objects_dirty":1375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9707,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2531,"num_bytes_recovered":154402816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[4,2,0],"avail_no_missing":["4","0","1","2"],"object_location_counts":[{"shards":"0,2,4","objects":1375}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":227,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956961+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:04:43.956961+0000","last_peered":"2023-10-09T09:04:43.956961+0000","last_clean":"2023-10-09T09:04:43.956961+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:04:43.956961+0000","last_undegraded":"2023-10-09T09:04:43.956961+0000","last_fullsized":"2023-10-09T09:04:43.956961+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952129+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:04:43.952129+0000","last_peered":"2023-10-09T09:04:43.952129+0000","last_clean":"2023-10-09T09:04:43.952129+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:04:43.952129+0000","last_undegraded":"2023-10-09T09:04:43.952129+0000","last_fullsized":"2023-10-09T09:04:43.952129+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":209,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.957454+0000","last_change":"2023-10-09T09:03:19.575546+0000","last_active":"2023-10-09T09:04:43.957454+0000","last_peered":"2023-10-09T09:04:43.957454+0000","last_clean":"2023-10-09T09:04:43.957454+0000","last_became_active":"2023-10-09T09:03:19.575075+0000","last_became_peered":"2023-10-09T09:03:19.575075+0000","last_unstale":"2023-10-09T09:04:43.957454+0000","last_undegraded":"2023-10-09T09:04:43.957454+0000","last_fullsized":"2023-10-09T09:04:43.957454+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:14.350886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"203'9679","reported_seq":9727,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:56.052197+0000","last_change":"2023-10-09T09:01:36.284533+0000","last_active":"2023-10-09T09:04:56.052197+0000","last_peered":"2023-10-09T09:04:56.052197+0000","last_clean":"2023-10-09T09:04:56.052197+0000","last_became_active":"2023-10-09T09:01:36.284124+0000","last_became_peered":"2023-10-09T09:01:36.284124+0000","last_unstale":"2023-10-09T09:04:56.052197+0000","last_undegraded":"2023-10-09T09:04:56.052197+0000","last_fullsized":"2023-10-09T09:04:56.052197+0000","mapping_epoch":176,"log_start":"189'4500","ondisk_log_start":"189'4500","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90636288,"num_objects":1383,"num_object_clones":0,"num_object_copies":4149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9679,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":133,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.946832+0000","last_change":"2023-10-09T09:01:36.274911+0000","last_active":"2023-10-09T09:04:43.946832+0000","last_peered":"2023-10-09T09:04:43.946832+0000","last_clean":"2023-10-09T09:04:43.946832+0000","last_became_active":"2023-10-09T09:01:36.274560+0000","last_became_peered":"2023-10-09T09:01:36.274560+0000","last_unstale":"2023-10-09T09:04:43.946832+0000","last_undegraded":"2023-10-09T09:04:43.946832+0000","last_fullsized":"2023-10-09T09:04:43.946832+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:55.305969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":442,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956158+0000","last_change":"2023-10-09T09:02:05.534070+0000","last_active":"2023-10-09T09:04:43.956158+0000","last_peered":"2023-10-09T09:04:43.956158+0000","last_clean":"2023-10-09T09:04:43.956158+0000","last_became_active":"2023-10-09T09:02:05.533567+0000","last_became_peered":"2023-10-09T09:02:05.533567+0000","last_unstale":"2023-10-09T09:04:43.956158+0000","last_undegraded":"2023-10-09T09:04:43.956158+0000","last_fullsized":"2023-10-09T09:04:43.956158+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:59.568077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.970158+0000","last_change":"2023-10-09T09:03:19.562659+0000","last_active":"2023-10-09T09:04:43.970158+0000","last_peered":"2023-10-09T09:04:43.970158+0000","last_clean":"2023-10-09T09:04:43.970158+0000","last_became_active":"2023-10-09T09:03:19.562284+0000","last_became_peered":"2023-10-09T09:03:19.562284+0000","last_unstale":"2023-10-09T09:04:43.970158+0000","last_undegraded":"2023-10-09T09:04:43.970158+0000","last_fullsized":"2023-10-09T09:04:43.970158+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:26.537954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.953425+0000","last_change":"2023-10-09T09:00:31.859121+0000","last_active":"2023-10-09T09:04:43.953425+0000","last_peered":"2023-10-09T09:04:43.953425+0000","last_clean":"2023-10-09T09:04:43.953425+0000","last_became_active":"2023-10-09T09:00:31.858675+0000","last_became_peered":"2023-10-09T09:00:31.858675+0000","last_unstale":"2023-10-09T09:04:43.953425+0000","last_undegraded":"2023-10-09T09:04:43.953425+0000","last_fullsized":"2023-10-09T09:04:43.953425+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"203'9779","reported_seq":13106,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:58.074946+0000","last_change":"2023-10-09T09:03:19.569754+0000","last_active":"2023-10-09T09:04:58.074946+0000","last_peered":"2023-10-09T09:04:58.074946+0000","last_clean":"2023-10-09T09:02:01.567140+0000","last_became_active":"2023-10-09T09:02:06.809579+0000","last_became_peered":"2023-10-09T09:02:06.809579+0000","last_unstale":"2023-10-09T09:04:58.074946+0000","last_undegraded":"2023-10-09T09:03:19.569445+0000","last_fullsized":"2023-10-09T09:02:03.617414+0000","mapping_epoch":182,"log_start":"187'4000","ondisk_log_start":"187'4000","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5779,"log_dups_size":0,"ondisk_log_size":5779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:47.900903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82509824,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9779,"num_write_kb":353216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1634,"num_bytes_recovered":107085824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"2,3","objects":419}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":96,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956309+0000","last_change":"2023-10-09T09:02:02.649488+0000","last_active":"2023-10-09T09:04:43.956309+0000","last_peered":"2023-10-09T09:04:43.956309+0000","last_clean":"2023-10-09T09:04:43.956309+0000","last_became_active":"2023-10-09T09:02:02.645550+0000","last_became_peered":"2023-10-09T09:02:02.645550+0000","last_unstale":"2023-10-09T09:04:43.956309+0000","last_undegraded":"2023-10-09T09:04:43.956309+0000","last_fullsized":"2023-10-09T09:04:43.956309+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:40.192731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":232,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.960182+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:04:43.960182+0000","last_peered":"2023-10-09T09:04:43.960182+0000","last_clean":"2023-10-09T09:04:43.960182+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:04:43.960182+0000","last_undegraded":"2023-10-09T09:04:43.960182+0000","last_fullsized":"2023-10-09T09:04:43.960182+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":192,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.965052+0000","last_change":"2023-10-09T09:02:02.657733+0000","last_active":"2023-10-09T09:04:43.965052+0000","last_peered":"2023-10-09T09:04:43.965052+0000","last_clean":"2023-10-09T09:04:43.965052+0000","last_became_active":"2023-10-09T09:02:02.656701+0000","last_became_peered":"2023-10-09T09:02:02.656701+0000","last_unstale":"2023-10-09T09:04:43.965052+0000","last_undegraded":"2023-10-09T09:04:43.965052+0000","last_fullsized":"2023-10-09T09:04:43.965052+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:39.552401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"203'9736","reported_seq":10202,"reported_epoch":203,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T09:04:55.629738+0000","last_change":"2023-10-09T09:04:43.879573+0000","last_active":"2023-10-09T09:04:55.629738+0000","last_peered":"2023-10-09T09:04:55.629738+0000","last_clean":"2023-10-09T09:02:01.581848+0000","last_became_active":"2023-10-09T09:02:52.219467+0000","last_became_peered":"2023-10-09T09:02:52.219467+0000","last_unstale":"2023-10-09T09:04:55.629738+0000","last_undegraded":"2023-10-09T09:02:52.047554+0000","last_fullsized":"2023-10-09T09:02:51.903742+0000","mapping_epoch":190,"log_start":"192'5022","ondisk_log_start":"192'5022","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4714,"log_dups_size":0,"ondisk_log_size":4714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:09:00.904807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3971,"num_objects_misplaced":759,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9736,"num_write_kb":355968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":206,"num_bytes_recovered":13500416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"3,4","objects":-3342},{"shards":"3,4,6","objects":759}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":176,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.080572+0000","last_change":"2023-10-09T09:02:02.644204+0000","last_active":"2023-10-09T09:04:45.080572+0000","last_peered":"2023-10-09T09:04:45.080572+0000","last_clean":"2023-10-09T09:04:45.080572+0000","last_became_active":"2023-10-09T09:02:02.643713+0000","last_became_peered":"2023-10-09T09:02:02.643713+0000","last_unstale":"2023-10-09T09:04:45.080572+0000","last_undegraded":"2023-10-09T09:04:45.080572+0000","last_fullsized":"2023-10-09T09:04:45.080572+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:20.811353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":171,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.946808+0000","last_change":"2023-10-09T09:02:02.647803+0000","last_active":"2023-10-09T09:04:43.946808+0000","last_peered":"2023-10-09T09:04:43.946808+0000","last_clean":"2023-10-09T09:04:43.946808+0000","last_became_active":"2023-10-09T09:02:02.646911+0000","last_became_peered":"2023-10-09T09:02:02.646911+0000","last_unstale":"2023-10-09T09:04:43.946808+0000","last_undegraded":"2023-10-09T09:04:43.946808+0000","last_fullsized":"2023-10-09T09:04:43.946808+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:35.294072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":190,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956435+0000","last_change":"2023-10-09T09:02:52.016015+0000","last_active":"2023-10-09T09:04:43.956435+0000","last_peered":"2023-10-09T09:04:43.956435+0000","last_clean":"2023-10-09T09:04:43.956435+0000","last_became_active":"2023-10-09T09:02:52.015543+0000","last_became_peered":"2023-10-09T09:02:52.015543+0000","last_unstale":"2023-10-09T09:04:43.956435+0000","last_undegraded":"2023-10-09T09:04:43.956435+0000","last_fullsized":"2023-10-09T09:04:43.956435+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.410980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"203'9723","reported_seq":9773,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:57.582824+0000","last_change":"2023-10-09T09:01:36.279450+0000","last_active":"2023-10-09T09:04:57.582824+0000","last_peered":"2023-10-09T09:04:57.582824+0000","last_clean":"2023-10-09T09:04:57.582824+0000","last_became_active":"2023-10-09T09:01:36.279123+0000","last_became_peered":"2023-10-09T09:01:36.279123+0000","last_unstale":"2023-10-09T09:04:57.582824+0000","last_undegraded":"2023-10-09T09:04:57.582824+0000","last_fullsized":"2023-10-09T09:04:57.582824+0000","mapping_epoch":176,"log_start":"187'4000","ondisk_log_start":"187'4000","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:21.843271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85000192,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9723,"num_write_kb":352640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":203,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:44.000222+0000","last_change":"2023-10-09T09:01:36.292450+0000","last_active":"2023-10-09T09:04:44.000222+0000","last_peered":"2023-10-09T09:04:44.000222+0000","last_clean":"2023-10-09T09:04:44.000222+0000","last_became_active":"2023-10-09T09:01:36.273029+0000","last_became_peered":"2023-10-09T09:01:36.273029+0000","last_unstale":"2023-10-09T09:04:44.000222+0000","last_undegraded":"2023-10-09T09:04:44.000222+0000","last_fullsized":"2023-10-09T09:04:44.000222+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:26.065684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":165,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952746+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:04:43.952746+0000","last_peered":"2023-10-09T09:04:43.952746+0000","last_clean":"2023-10-09T09:04:43.952746+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:04:43.952746+0000","last_undegraded":"2023-10-09T09:04:43.952746+0000","last_fullsized":"2023-10-09T09:04:43.952746+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":20,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.954994+0000","last_change":"2023-10-09T09:03:19.572475+0000","last_active":"2023-10-09T09:04:43.954994+0000","last_peered":"2023-10-09T09:04:43.954994+0000","last_clean":"2023-10-09T09:04:43.954994+0000","last_became_active":"2023-10-09T09:03:19.572081+0000","last_became_peered":"2023-10-09T09:03:19.572081+0000","last_unstale":"2023-10-09T09:04:43.954994+0000","last_undegraded":"2023-10-09T09:04:43.954994+0000","last_fullsized":"2023-10-09T09:04:43.954994+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:04.771178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"203'9793","reported_seq":14833,"reported_epoch":203,"state":"active+recovery_wait","last_fresh":"2023-10-09T09:04:58.079023+0000","last_change":"2023-10-09T09:04:45.074782+0000","last_active":"2023-10-09T09:04:58.079023+0000","last_peered":"2023-10-09T09:04:58.079023+0000","last_clean":"2023-10-09T09:02:01.579768+0000","last_became_active":"2023-10-09T09:04:45.039242+0000","last_became_peered":"2023-10-09T09:04:45.039242+0000","last_unstale":"2023-10-09T09:04:58.079023+0000","last_undegraded":"2023-10-09T09:04:58.079023+0000","last_fullsized":"2023-10-09T09:04:58.079023+0000","mapping_epoch":202,"log_start":"187'4000","ondisk_log_start":"187'4000","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5793,"log_dups_size":0,"ondisk_log_size":5793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:39.547884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79822872,"num_objects":1219,"num_object_clones":0,"num_object_copies":3657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1219,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9793,"num_write_kb":352321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2490,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":["2","1","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1219}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":52,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.953052+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:04:43.953052+0000","last_peered":"2023-10-09T09:04:43.953052+0000","last_clean":"2023-10-09T09:04:43.953052+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:04:43.953052+0000","last_undegraded":"2023-10-09T09:04:43.953052+0000","last_fullsized":"2023-10-09T09:04:43.953052+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":211,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.965148+0000","last_change":"2023-10-09T09:02:02.674657+0000","last_active":"2023-10-09T09:04:43.965148+0000","last_peered":"2023-10-09T09:04:43.965148+0000","last_clean":"2023-10-09T09:04:43.965148+0000","last_became_active":"2023-10-09T09:02:02.657392+0000","last_became_peered":"2023-10-09T09:02:02.657392+0000","last_unstale":"2023-10-09T09:04:43.965148+0000","last_undegraded":"2023-10-09T09:04:43.965148+0000","last_fullsized":"2023-10-09T09:04:43.965148+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:16.286583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":74,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945023+0000","last_change":"2023-10-09T09:01:36.280537+0000","last_active":"2023-10-09T09:04:43.945023+0000","last_peered":"2023-10-09T09:04:43.945023+0000","last_clean":"2023-10-09T09:04:43.945023+0000","last_became_active":"2023-10-09T09:01:36.280149+0000","last_became_peered":"2023-10-09T09:01:36.280149+0000","last_unstale":"2023-10-09T09:04:43.945023+0000","last_undegraded":"2023-10-09T09:04:43.945023+0000","last_fullsized":"2023-10-09T09:04:43.945023+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":210,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.956353+0000","last_change":"2023-10-09T09:03:19.554339+0000","last_active":"2023-10-09T09:04:43.956353+0000","last_peered":"2023-10-09T09:04:43.956353+0000","last_clean":"2023-10-09T09:04:43.956353+0000","last_became_active":"2023-10-09T09:03:19.553953+0000","last_became_peered":"2023-10-09T09:03:19.553953+0000","last_unstale":"2023-10-09T09:04:43.956353+0000","last_undegraded":"2023-10-09T09:04:43.956353+0000","last_fullsized":"2023-10-09T09:04:43.956353+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:50.601515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":32,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.946712+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:04:43.946712+0000","last_peered":"2023-10-09T09:04:43.946712+0000","last_clean":"2023-10-09T09:04:43.946712+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:04:43.946712+0000","last_undegraded":"2023-10-09T09:04:43.946712+0000","last_fullsized":"2023-10-09T09:04:43.946712+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":205,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945129+0000","last_change":"2023-10-09T09:02:52.022788+0000","last_active":"2023-10-09T09:04:43.945129+0000","last_peered":"2023-10-09T09:04:43.945129+0000","last_clean":"2023-10-09T09:04:43.945129+0000","last_became_active":"2023-10-09T09:02:52.016418+0000","last_became_peered":"2023-10-09T09:02:52.016418+0000","last_unstale":"2023-10-09T09:04:43.945129+0000","last_undegraded":"2023-10-09T09:04:43.945129+0000","last_fullsized":"2023-10-09T09:04:43.945129+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":65,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945247+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:04:43.945247+0000","last_peered":"2023-10-09T09:04:43.945247+0000","last_clean":"2023-10-09T09:04:43.945247+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:04:43.945247+0000","last_undegraded":"2023-10-09T09:04:43.945247+0000","last_fullsized":"2023-10-09T09:04:43.945247+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":190,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.945487+0000","last_change":"2023-10-09T09:03:19.575218+0000","last_active":"2023-10-09T09:04:43.945487+0000","last_peered":"2023-10-09T09:04:43.945487+0000","last_clean":"2023-10-09T09:04:43.945487+0000","last_became_active":"2023-10-09T09:03:19.557578+0000","last_became_peered":"2023-10-09T09:03:19.557578+0000","last_unstale":"2023-10-09T09:04:43.945487+0000","last_undegraded":"2023-10-09T09:04:43.945487+0000","last_fullsized":"2023-10-09T09:04:43.945487+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.944988+0000","last_change":"2023-10-09T08:59:53.522794+0000","last_active":"2023-10-09T09:04:43.944988+0000","last_peered":"2023-10-09T09:04:43.944988+0000","last_clean":"2023-10-09T09:04:43.944988+0000","last_became_active":"2023-10-09T08:59:53.522419+0000","last_became_peered":"2023-10-09T08:59:53.522419+0000","last_unstale":"2023-10-09T09:04:43.944988+0000","last_undegraded":"2023-10-09T09:04:43.944988+0000","last_fullsized":"2023-10-09T09:04:43.944988+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":203,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.969182+0000","last_change":"2023-10-09T09:00:31.845771+0000","last_active":"2023-10-09T09:04:43.969182+0000","last_peered":"2023-10-09T09:04:43.969182+0000","last_clean":"2023-10-09T09:04:43.969182+0000","last_became_active":"2023-10-09T09:00:31.844775+0000","last_became_peered":"2023-10-09T09:00:31.844775+0000","last_unstale":"2023-10-09T09:04:43.969182+0000","last_undegraded":"2023-10-09T09:04:43.969182+0000","last_fullsized":"2023-10-09T09:04:43.969182+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:33.452539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":20,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.952478+0000","last_change":"2023-10-09T09:03:19.561924+0000","last_active":"2023-10-09T09:04:43.952478+0000","last_peered":"2023-10-09T09:04:43.952478+0000","last_clean":"2023-10-09T09:04:43.952478+0000","last_became_active":"2023-10-09T09:03:19.557692+0000","last_became_peered":"2023-10-09T09:03:19.557692+0000","last_unstale":"2023-10-09T09:04:43.952478+0000","last_undegraded":"2023-10-09T09:04:43.952478+0000","last_fullsized":"2023-10-09T09:04:43.952478+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:39.940109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":263,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.957603+0000","last_change":"2023-10-09T09:02:52.019620+0000","last_active":"2023-10-09T09:04:43.957603+0000","last_peered":"2023-10-09T09:04:43.957603+0000","last_clean":"2023-10-09T09:04:43.957603+0000","last_became_active":"2023-10-09T09:02:52.014489+0000","last_became_peered":"2023-10-09T09:02:52.014489+0000","last_unstale":"2023-10-09T09:04:43.957603+0000","last_undegraded":"2023-10-09T09:04:43.957603+0000","last_fullsized":"2023-10-09T09:04:43.957603+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:39.550600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":83,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T09:04:43.944827+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:04:43.944827+0000","last_peered":"2023-10-09T09:04:43.944827+0000","last_clean":"2023-10-09T09:04:43.944827+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:04:43.944827+0000","last_undegraded":"2023-10-09T09:04:43.944827+0000","last_fullsized":"2023-10-09T09:04:43.944827+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":13518,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1397751832,"num_objects":21329,"num_object_clones":0,"num_object_copies":63987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14747,"num_objects_misplaced":3795,"num_objects_unfound":0,"num_objects_dirty":21329,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155145,"num_write_kb":5647105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17770,"num_bytes_recovered":1005191168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5910769664,"data_stored":5910757448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85304,"ondisk_log_size":85304,"up":48,"acting":44,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430613,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626846,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1634024,"kb_used_data":1036128,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":92111128,"statfs":{"total":95995035648,"available":94321795072,"internally_reserved":0,"allocated":1060995072,"data_stored":1060502344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.758,"15min":2.3210000000000002},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.24199999999999999},"max":{"1min":4.54,"5min":58.621000000000002,"15min":153.63800000000001},"last":3.6259999999999999},{"interface":"front","average":{"1min":1.3640000000000001,"5min":1.996,"15min":2.3380000000000001},"min":{"1min":0.38,"5min":0.36199999999999999,"15min":0.33500000000000002},"max":{"1min":4.6079999999999997,"5min":58.590000000000003,"15min":153.72200000000001},"last":3.6539999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.921,"15min":1.8819999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28299999999999997},"max":{"1min":0.96599999999999997,"5min":109.453,"15min":109.453},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.621,"5min":2.1110000000000002,"15min":1.946},"min":{"1min":0.316,"5min":0.314,"15min":0.314},"max":{"1min":1.512,"5min":111.47499999999999,"15min":111.47499999999999},"last":0.78200000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:04:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.1579999999999999,"15min":1.399},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.29799999999999999},"max":{"1min":5.2060000000000004,"5min":5.2910000000000004,"15min":8.3979999999999997},"last":3.7200000000000002},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.1699999999999999,"15min":1.4670000000000001},"min":{"1min":0.38500000000000001,"5min":0.30199999999999999,"15min":0.28100000000000003},"max":{"1min":3.6280000000000001,"5min":9.1549999999999994,"15min":12.194000000000001},"last":0.83699999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:04:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.321,"15min":2.073},"min":{"1min":0.46100000000000002,"5min":0.38300000000000001,"15min":0.34399999999999997},"max":{"1min":5.0990000000000002,"5min":7.4130000000000003,"15min":57.636000000000003},"last":5.7130000000000001},{"interface":"front","average":{"1min":2.5760000000000001,"5min":1.587,"15min":1.8999999999999999},"min":{"1min":0.64700000000000002,"5min":0.38800000000000001,"15min":0.34699999999999998},"max":{"1min":7.4850000000000003,"5min":9.4429999999999996,"15min":58.393000000000001},"last":0.81100000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:04:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.403,"15min":1.5389999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.32500000000000001},"max":{"1min":1.5249999999999999,"5min":32.280999999999999,"15min":32.280999999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.2609999999999999,"15min":1.1830000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.30599999999999999},"max":{"1min":0.98299999999999998,"5min":34.689,"15min":34.689},"last":0.86199999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:04:36 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.014,"15min":1.2889999999999999},"min":{"1min":0.42799999999999999,"5min":0.38200000000000001,"15min":0.35499999999999998},"max":{"1min":1.494,"5min":9.218,"15min":12.404999999999999},"last":1.141},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.93300000000000005,"15min":1.1379999999999999},"min":{"1min":0.46700000000000003,"5min":0.41599999999999998,"15min":0.33900000000000002},"max":{"1min":1.0680000000000001,"5min":4.2610000000000001,"15min":8.8569999999999993},"last":0.89700000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:04:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.052,"15min":1.4199999999999999},"min":{"1min":0.48199999999999998,"5min":0.434,"15min":0.40500000000000003},"max":{"1min":1.474,"5min":9.2460000000000004,"15min":11.382},"last":1.0980000000000001},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.91500000000000004,"15min":1.244},"min":{"1min":0.46100000000000002,"5min":0.45400000000000001,"15min":0.377},"max":{"1min":1.04,"5min":4.3259999999999996,"15min":9.4800000000000004},"last":0.92500000000000004}]}]},{"osd":1,"up_from":0,"seq":356482285794,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449320,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199796,"kb_used_data":765164,"kb_used_omap":28,"kb_used_meta":434531,"kb_avail":92545356,"statfs":{"total":95995035648,"available":94766444544,"internally_reserved":0,"allocated":783527936,"data_stored":782732747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":444960592},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:04:51 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.375,"15min":1.617},"min":{"1min":0.22,"5min":0.22,"15min":0.21199999999999999},"max":{"1min":4.2400000000000002,"5min":6.7649999999999997,"15min":37.380000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.415,"15min":1.6379999999999999},"min":{"1min":0.308,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.102,"5min":18.661000000000001,"15min":38.012999999999998},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":3.6720000000000002,"15min":2.3580000000000001},"min":{"1min":0.32400000000000001,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":9.4719999999999995,"5min":71.322999999999993,"15min":71.322999999999993},"last":0.59099999999999997},{"interface":"front","average":{"1min":14.773999999999999,"5min":6.1130000000000004,"15min":3.2589999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":219.00700000000001,"5min":219.00700000000001,"15min":219.00700000000001},"last":0.435}]},{"osd":3,"last update":"Mon Oct 9 09:04:37 2023","interfaces":[{"interface":"back","average":{"1min":14.757,"5min":4.4870000000000001,"15min":3.3860000000000001},"min":{"1min":0.32300000000000001,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":219.041,"5min":219.041,"15min":219.041},"last":0.91800000000000004},{"interface":"front","average":{"1min":3.4990000000000001,"5min":2.2309999999999999,"15min":2.5899999999999999},"min":{"1min":0.29499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":20.466000000000001,"5min":20.466000000000001,"15min":107.66500000000001},"last":0.82999999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":1.5309999999999999,"15min":1.466},"min":{"1min":0.66500000000000004,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.178000000000001,"5min":10.178000000000001,"15min":10.178000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.246,"5min":1.716,"15min":1.673},"min":{"1min":0.44500000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.526,"5min":10.526,"15min":11.654},"last":0.73899999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.867,"5min":1.631,"15min":1.4059999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.34499999999999997},"max":{"1min":9.3030000000000008,"5min":9.3030000000000008,"15min":9.3689999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.5350000000000001,"5min":1.581,"15min":1.5760000000000001},"min":{"1min":0.54900000000000004,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.140000000000001,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.497}]},{"osd":6,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":2.577,"15min":1.855},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":9.5079999999999991,"5min":62.850999999999999,"15min":62.850999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.452,"5min":2.5979999999999999,"15min":2.0590000000000002},"min":{"1min":0.49399999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.121,"5min":62.816000000000003,"15min":62.816000000000003},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:04:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.6799999999999999,"15min":1.6519999999999999},"min":{"1min":0.63600000000000001,"5min":0.42499999999999999,"15min":0.39400000000000002},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.845,"15min":1.855},"min":{"1min":0.53200000000000003,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":10.662000000000001,"5min":14.359,"15min":14.359},"last":0.76400000000000001}]}]},{"osd":2,"up_from":88,"seq":377957122268,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1257320,"kb_used_data":773216,"kb_used_omap":81,"kb_used_meta":484014,"kb_avail":92487832,"statfs":{"total":95995035648,"available":94707539968,"internally_reserved":0,"allocated":791773184,"data_stored":790934415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":495630554},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":3.5070000000000001,"15min":3.0099999999999998},"min":{"1min":0.42499999999999999,"5min":0.214,"15min":0.191},"max":{"1min":8.2129999999999992,"5min":54.713999999999999,"15min":134.458},"last":1.649},{"interface":"front","average":{"1min":1.6950000000000001,"5min":3.2570000000000001,"15min":2.6280000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.221},"max":{"1min":5.532,"5min":53.670000000000002,"15min":138.417},"last":2.7869999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:04:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.4319999999999999,"15min":2.2690000000000001},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.21299999999999999},"max":{"1min":8.1560000000000006,"5min":57.939999999999998,"15min":57.939999999999998},"last":1.619},{"interface":"front","average":{"1min":0.72799999999999998,"5min":2.3069999999999999,"15min":2.0510000000000002},"min":{"1min":0.41999999999999998,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":1.6379999999999999,"5min":56.667999999999999,"15min":57.122999999999998},"last":1.9610000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:04:52 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":4.5750000000000002,"15min":2.6230000000000002},"min":{"1min":0.38700000000000001,"5min":0.34999999999999998,"15min":0.29399999999999998},"max":{"1min":8.157,"5min":112.47499999999999,"15min":112.47499999999999},"last":1.7},{"interface":"front","average":{"1min":2.8730000000000002,"5min":4.8879999999999999,"15min":2.9649999999999999},"min":{"1min":0.26800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":8.4410000000000007,"5min":112.929,"15min":112.929},"last":1.778}]},{"osd":4,"last update":"Mon Oct 9 09:04:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":1.6779999999999999,"15min":1.835},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.3079999999999998,"5min":9.8109999999999999,"15min":12.191000000000001},"last":0.316},{"interface":"front","average":{"1min":2.3849999999999998,"5min":1.669,"15min":1.804},"min":{"1min":0.46700000000000003,"5min":0.38800000000000001,"15min":0.314},"max":{"1min":8.1850000000000005,"5min":9.8439999999999994,"15min":14.071999999999999},"last":1.681}]},{"osd":5,"last update":"Mon Oct 9 09:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.353,"15min":1.4430000000000001},"min":{"1min":0.52800000000000002,"5min":0.38600000000000001,"15min":0.33900000000000002},"max":{"1min":1.7010000000000001,"5min":11.138,"15min":12.411},"last":1.732},{"interface":"front","average":{"1min":2.641,"5min":1.6639999999999999,"15min":1.8340000000000001},"min":{"1min":0.41899999999999998,"5min":0.38900000000000001,"15min":0.34999999999999998},"max":{"1min":8.407,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.36899999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:04:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":1.6739999999999999,"15min":1.829},"min":{"1min":0.53300000000000003,"5min":0.47799999999999998,"15min":0.33900000000000002},"max":{"1min":8.1129999999999995,"5min":9.9139999999999997,"15min":10.456},"last":1.716},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.5109999999999999,"15min":1.556},"min":{"1min":0.54200000000000004,"5min":0.41599999999999998,"15min":0.39300000000000002},"max":{"1min":5.319,"5min":11.869999999999999,"15min":12.381},"last":1.8009999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:04:33 2023","interfaces":[{"interface":"back","average":{"1min":2.649,"5min":1.714,"15min":1.823},"min":{"1min":0.61699999999999999,"5min":0.51400000000000001,"15min":0.38800000000000001},"max":{"1min":8.2379999999999995,"5min":9.8889999999999993,"15min":17.77},"last":2.2330000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.53,"15min":1.518},"min":{"1min":0.52200000000000002,"5min":0.442,"15min":0.38500000000000001},"max":{"1min":4.1239999999999997,"5min":11.933,"15min":12.438000000000001},"last":1.819}]}]},{"osd":3,"up_from":93,"seq":399431958742,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1200288,"kb_used_data":777944,"kb_used_omap":7,"kb_used_meta":422264,"kb_avail":92544864,"statfs":{"total":95995035648,"available":94765940736,"internally_reserved":0,"allocated":796614656,"data_stored":795837336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":432398952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.4219999999999999,"15min":1.5669999999999999},"min":{"1min":0.29799999999999999,"5min":0.19500000000000001,"15min":0.19400000000000001},"max":{"1min":3.6179999999999999,"5min":9.0199999999999996,"15min":13.916},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.722,"15min":1.645},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":7.6180000000000003,"5min":10.616,"15min":14.013999999999999},"last":0.63900000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":4.8700000000000001,"15min":2.7749999999999999},"min":{"1min":0.39500000000000002,"5min":0.26700000000000002,"15min":0.22500000000000001},"max":{"1min":5.8159999999999998,"5min":326.75700000000001,"15min":326.75700000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.0600000000000001,"5min":4.7999999999999998,"15min":2.714},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.1230000000000002,"5min":327.024,"15min":327.024},"last":0.60999999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":3.1219999999999999,"5min":2.5179999999999998,"15min":2.5310000000000001},"min":{"1min":0.38800000000000001,"5min":0.28599999999999998,"15min":0.255},"max":{"1min":9.875,"5min":43.683999999999997,"15min":75.641999999999996},"last":3.3969999999999998},{"interface":"front","average":{"1min":1.998,"5min":2.4820000000000002,"15min":2.3769999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4509999999999996,"5min":45.027999999999999,"15min":75.207999999999998},"last":0.67800000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.3140000000000001,"15min":1.498},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.5870000000000002,"5min":8.9879999999999995,"15min":12.877000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.46,"5min":1.696,"15min":1.6919999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33500000000000002},"max":{"1min":3.6240000000000001,"5min":10.486000000000001,"15min":10.486000000000001},"last":2.4180000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.528,"15min":1.589},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":3.6789999999999998,"5min":10.577,"15min":10.577},"last":2.3900000000000001},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.528,"15min":1.7170000000000001},"min":{"1min":0.58599999999999997,"5min":0.44900000000000001,"15min":0.34300000000000003},"max":{"1min":5.8710000000000004,"5min":7.5590000000000002,"15min":12.204000000000001},"last":0.82599999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.7310000000000001,"15min":1.772},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":3.9260000000000002,"5min":10.537000000000001,"15min":10.537000000000001},"last":2.3580000000000001},{"interface":"front","average":{"1min":2.8399999999999999,"5min":1.8300000000000001,"15min":1.8799999999999999},"min":{"1min":0.54000000000000004,"5min":0.40699999999999997,"15min":0.35099999999999998},"max":{"1min":9.5090000000000003,"5min":9.5090000000000003,"15min":12.275},"last":0.75800000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.591,"15min":1.7310000000000001},"min":{"1min":0.56599999999999995,"5min":0.52700000000000002,"15min":0.41299999999999998},"max":{"1min":6.0289999999999999,"5min":7.5899999999999999,"15min":12.244},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.7350000000000001,"5min":1.415,"15min":1.6599999999999999},"min":{"1min":0.56100000000000005,"5min":0.48599999999999999,"15min":0.39000000000000001},"max":{"1min":5.6959999999999997,"5min":9.1560000000000006,"15min":13.339},"last":0.91200000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019312,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1781676,"kb_used_data":1484452,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91963476,"statfs":{"total":95995035648,"available":94170599424,"internally_reserved":0,"allocated":1520078848,"data_stored":1519581353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":2.9889999999999999,"15min":2.29},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26100000000000001},"max":{"1min":5.077,"5min":141.07400000000001,"15min":141.07400000000001},"last":4.2709999999999999},{"interface":"front","average":{"1min":1.454,"5min":2.8700000000000001,"15min":2.3079999999999998},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":8.6379999999999999,"5min":141.15600000000001,"15min":141.15600000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:04:46 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.81100000000000005,"15min":1.3899999999999999},"min":{"1min":0.34799999999999998,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":1.8759999999999999,"5min":6.0529999999999999,"15min":8.9169999999999998},"last":0.373},{"interface":"front","average":{"1min":0.68500000000000005,"5min":1.1799999999999999,"15min":1.474},"min":{"1min":0.35199999999999998,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":2.6880000000000002,"5min":7.6879999999999997,"15min":11.712999999999999},"last":0.33800000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":3.093,"15min":2.2650000000000001},"min":{"1min":0.47499999999999998,"5min":0.36899999999999999,"15min":0.30499999999999999},"max":{"1min":8.4090000000000007,"5min":142.816,"15min":142.816},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.498,"5min":2.7320000000000002,"15min":1.9179999999999999},"min":{"1min":0.40600000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.8150000000000004,"5min":143.81800000000001,"15min":143.81800000000001},"last":0.32100000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:04:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":2.645,"15min":2.1019999999999999},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.29499999999999998},"max":{"1min":2.48,"5min":143.13399999999999,"15min":143.13399999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.597,"5min":3.3380000000000001,"15min":2.149},"min":{"1min":0.46300000000000002,"5min":0.41299999999999998,"15min":0.35399999999999998},"max":{"1min":5.4630000000000001,"5min":152.37299999999999,"15min":152.37299999999999},"last":0.48599999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.0169999999999999,"15min":1.214},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.7350000000000003,"5min":7.7439999999999998,"15min":11.762},"last":0.504},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.872,"15min":1.2769999999999999},"min":{"1min":0.38900000000000001,"5min":0.37,"15min":0.28699999999999998},"max":{"1min":3.4900000000000002,"5min":3.4900000000000002,"15min":12.85},"last":0.57199999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:04:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.88500000000000001,"15min":1.401},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.32100000000000001},"max":{"1min":1.643,"5min":5.1020000000000003,"15min":10.657999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.115,"15min":1.448},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.34300000000000003},"max":{"1min":2.7290000000000001,"5min":7.6470000000000002,"15min":14.755000000000001},"last":0.39500000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.93000000000000005,"15min":1.4119999999999999},"min":{"1min":0.42999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.4660000000000002,"5min":3.4660000000000002,"15min":13.933999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.90400000000000003,"15min":1.5700000000000001},"min":{"1min":0.372,"5min":0.36199999999999999,"15min":0.33200000000000002},"max":{"1min":2.1869999999999998,"5min":5.1520000000000001,"15min":13.911},"last":0.58299999999999996}]}]},{"osd":5,"up_from":0,"seq":154618823080,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":756416512,"data_stored":756416512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1043267584,"data_stored":1043267584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":775950336,"data_stored":775946264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":793772032,"data_stored":793772032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1500778496,"data_stored":1500774424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1040584704,"data_stored":1040580632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:05:05.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T09:05:05.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:05.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:05.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:05.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:05.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:05.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:05.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:05.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:05.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:05.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:05.728 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:05.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:05.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:05.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:05.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:05.869 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:05 smithi016 bash[75179]: cluster 2023-10-09T09:05:03.877888+0000 mgr.y (mgr.25421) 827 : cluster 0 pgmap v859: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 14875/53151 objects degraded (27.986%); 3073/53151 objects misplaced (5.782%); 847 KiB/s, 13 objects/s recovering 2023-10-09T09:05:05.869 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:05:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:05:05] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.33.4" 2023-10-09T09:05:05.870 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:05 smithi016 bash[77892]: cluster 2023-10-09T09:05:03.877888+0000 mgr.y (mgr.25421) 827 : cluster 0 pgmap v859: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 14875/53151 objects degraded (27.986%); 3073/53151 objects misplaced (5.782%); 847 KiB/s, 13 objects/s recovering 2023-10-09T09:05:05.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:05.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:05.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:05.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:05 smithi094 bash[59677]: cluster 2023-10-09T09:05:03.877888+0000 mgr.y (mgr.25421) 827 : cluster 0 pgmap v859: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 14875/53151 objects degraded (27.986%); 3073/53151 objects misplaced (5.782%); 847 KiB/s, 13 objects/s recovering 2023-10-09T09:05:05.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:06.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:06.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:06.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:06.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:06.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:06.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:06.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:06.423 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:06.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:06.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:06.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:06.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:06.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:06 smithi094 bash[59677]: cluster 2023-10-09T09:05:05.878674+0000 mgr.y (mgr.25421) 828 : cluster 0 pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1007 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 705 op/s; 15030/48396 objects degraded (31.056%); 2415/48396 objects misplaced (4.990%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:05:06.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:06.828 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:06.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:06.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:06.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:07.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:06 smithi016 bash[77892]: cluster 2023-10-09T09:05:05.878674+0000 mgr.y (mgr.25421) 828 : cluster 0 pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1007 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 705 op/s; 15030/48396 objects degraded (31.056%); 2415/48396 objects misplaced (4.990%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:05:07.006 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:06 smithi016 bash[75179]: cluster 2023-10-09T09:05:05.878674+0000 mgr.y (mgr.25421) 828 : cluster 0 pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1007 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 705 op/s; 15030/48396 objects degraded (31.056%); 2415/48396 objects misplaced (4.990%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:05:07.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:07.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:07.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:07.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:07.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:07.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:07.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:07.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:07.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:07.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:07.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:07.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:07.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:07.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:07.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:07.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:07.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:07.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:07.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:07.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:07.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:07.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:07.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:07.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:07.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:07.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:08.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:08.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:08.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:08.145 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:08.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:08.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:08.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:08.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:08.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:08.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:08.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:08.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:08.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:08.662 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:08.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:08.765 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:08.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:08.904 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:08.935 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:08.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:08.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:09.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:09.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:09.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:09.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:09.180 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:09.191 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:08 smithi094 bash[59677]: cluster 2023-10-09T09:05:07.879316+0000 mgr.y (mgr.25421) 829 : cluster 0 pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 912 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 717 op/s; 15030/43821 objects degraded (34.299%); 2365/43821 objects misplaced (5.397%); 767 KiB/s, 11 objects/s recovering 2023-10-09T09:05:09.205 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:09.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:09.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:09.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:08 smithi016 bash[75179]: cluster 2023-10-09T09:05:07.879316+0000 mgr.y (mgr.25421) 829 : cluster 0 pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 912 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 717 op/s; 15030/43821 objects degraded (34.299%); 2365/43821 objects misplaced (5.397%); 767 KiB/s, 11 objects/s recovering 2023-10-09T09:05:09.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:08 smithi016 bash[77892]: cluster 2023-10-09T09:05:07.879316+0000 mgr.y (mgr.25421) 829 : cluster 0 pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 912 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 717 op/s; 15030/43821 objects degraded (34.299%); 2365/43821 objects misplaced (5.397%); 767 KiB/s, 11 objects/s recovering 2023-10-09T09:05:09.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:09.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:09.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:09.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:09.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:09.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:09.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:09.578 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:09.609 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:09.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:09.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:09.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:09.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:09.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:09.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:09.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:09.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:09.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:09.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:09.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:09 smithi094 bash[59677]: cluster 2023-10-09T09:05:09.563465+0000 mon.a (mon.0) 1125 : cluster 3 Health check update: Degraded data redundancy: 15030/43821 objects degraded (34.299%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:10.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:10.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.128 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:10 smithi016 bash[75179]: cluster 2023-10-09T09:05:09.563465+0000 mon.a (mon.0) 1125 : cluster 3 Health check update: Degraded data redundancy: 15030/43821 objects degraded (34.299%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:10.128 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:10 smithi016 bash[77892]: cluster 2023-10-09T09:05:09.563465+0000 mon.a (mon.0) 1125 : cluster 3 Health check update: Degraded data redundancy: 15030/43821 objects degraded (34.299%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:10.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:10.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:10.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:10.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:10.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:10.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:10.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:10.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:10.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:10.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:10.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:11.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:11.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:11.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:11.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:11.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:11.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:11.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:11.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:11.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:11.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:11.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:11.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:11.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:11.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:11.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:11.630 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:11 smithi094 bash[59677]: cluster 2023-10-09T09:05:09.879869+0000 mgr.y (mgr.25421) 830 : cluster 0 pgmap v862: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 886 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 15030/42555 objects degraded (35.319%); 2365/42555 objects misplaced (5.558%); 767 KiB/s, 11 objects/s recovering 2023-10-09T09:05:11.644 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:11.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:11.748 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:11 smithi016 bash[75179]: cluster 2023-10-09T09:05:09.879869+0000 mgr.y (mgr.25421) 830 : cluster 0 pgmap v862: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 886 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 15030/42555 objects degraded (35.319%); 2365/42555 objects misplaced (5.558%); 767 KiB/s, 11 objects/s recovering 2023-10-09T09:05:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:11 smithi016 bash[77892]: cluster 2023-10-09T09:05:09.879869+0000 mgr.y (mgr.25421) 830 : cluster 0 pgmap v862: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 886 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 15030/42555 objects degraded (35.319%); 2365/42555 objects misplaced (5.558%); 767 KiB/s, 11 objects/s recovering 2023-10-09T09:05:11.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:11.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:11.878 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:11.899 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:11.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:11.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:12.002 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:12.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:12.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:12.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:12.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:12.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:12.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:12.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:12.408 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:12.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:12.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:12.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:12.575 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:12 smithi094 bash[59677]: audit 2023-10-09T09:05:11.731413+0000 mon.a (mon.0) 1126 : audit 1 from='client.? 172.21.15.16:0/3676372177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:05:12.606 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:05:12.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:12.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:12.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:12.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:12.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:12 smithi016 bash[75179]: audit 2023-10-09T09:05:11.731413+0000 mon.a (mon.0) 1126 : audit 1 from='client.? 172.21.15.16:0/3676372177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:05:12.820 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:12 smithi016 bash[77892]: audit 2023-10-09T09:05:11.731413+0000 mon.a (mon.0) 1126 : audit 1 from='client.? 172.21.15.16:0/3676372177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:05:12.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:12.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:12.886 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:05:12.941 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:05:12.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:12.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:12.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:12.992 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:05:13.049 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:05:13.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:13.086 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:05:13.106 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:13.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:13.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:13.123 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:05:13.156 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:05:13.194 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:05:13.195 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.195 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.195 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.196 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:05:13.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:13.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:13.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:13.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:13.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:13.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:13.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:13.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:13.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:13.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:13.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:13.654 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:13.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:13.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:13.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[75179]: cluster 2023-10-09T09:05:11.880600+0000 mgr.y (mgr.25421) 831 : cluster 0 pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 758 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 766 op/s; 15160/36447 objects degraded (41.595%); 1778/36447 objects misplaced (4.878%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:05:13.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[75179]: cluster 2023-10-09T09:05:12.569548+0000 mon.a (mon.0) 1127 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:05:13.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[75179]: cluster 2023-10-09T09:05:12.589401+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T09:05:13.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[75179]: cluster 2023-10-09T09:05:12.591011+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[75179]: audit 2023-10-09T09:05:12.592646+0000 mon.a (mon.0) 1128 : audit 1 from='client.? 172.21.15.16:0/3676372177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[75179]: cluster 2023-10-09T09:05:12.592743+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e204: 8 total, 8 up, 5 in 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[75179]: audit 2023-10-09T09:05:12.832846+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.16:0/898854043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[77892]: cluster 2023-10-09T09:05:11.880600+0000 mgr.y (mgr.25421) 831 : cluster 0 pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 758 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 766 op/s; 15160/36447 objects degraded (41.595%); 1778/36447 objects misplaced (4.878%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[77892]: cluster 2023-10-09T09:05:12.569548+0000 mon.a (mon.0) 1127 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[77892]: cluster 2023-10-09T09:05:12.589401+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[77892]: cluster 2023-10-09T09:05:12.591011+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[77892]: audit 2023-10-09T09:05:12.592646+0000 mon.a (mon.0) 1128 : audit 1 from='client.? 172.21.15.16:0/3676372177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:05:13.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[77892]: cluster 2023-10-09T09:05:12.592743+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e204: 8 total, 8 up, 5 in 2023-10-09T09:05:13.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:13 smithi016 bash[77892]: audit 2023-10-09T09:05:12.832846+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.16:0/898854043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:05:13.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:13.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:13.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:13.896 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:13.904 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:13 smithi094 bash[59677]: cluster 2023-10-09T09:05:11.880600+0000 mgr.y (mgr.25421) 831 : cluster 0 pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 758 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 766 op/s; 15160/36447 objects degraded (41.595%); 1778/36447 objects misplaced (4.878%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:05:13.904 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:13 smithi094 bash[59677]: cluster 2023-10-09T09:05:12.569548+0000 mon.a (mon.0) 1127 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:05:13.904 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:13 smithi094 bash[59677]: cluster 2023-10-09T09:05:12.589401+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T09:05:13.904 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:13 smithi094 bash[59677]: cluster 2023-10-09T09:05:12.591011+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T09:05:13.904 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:13 smithi094 bash[59677]: audit 2023-10-09T09:05:12.592646+0000 mon.a (mon.0) 1128 : audit 1 from='client.? 172.21.15.16:0/3676372177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:05:13.905 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:13 smithi094 bash[59677]: cluster 2023-10-09T09:05:12.592743+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e204: 8 total, 8 up, 5 in 2023-10-09T09:05:13.905 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:13 smithi094 bash[59677]: audit 2023-10-09T09:05:12.832846+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.16:0/898854043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:05:13.918 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:13.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:14.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:14.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:14.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:14.173 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:14.278 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:14.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:14.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:14.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:14.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:14.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:14.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:14.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:14.676 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:14.698 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:14.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:14.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:14 smithi016 bash[75179]: cluster 2023-10-09T09:05:14.564405+0000 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 15160/36447 objects degraded (41.595%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:14.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:14 smithi016 bash[77892]: cluster 2023-10-09T09:05:14.564405+0000 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 15160/36447 objects degraded (41.595%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:14.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:14.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:14.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:14.949 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:14 smithi094 bash[59677]: cluster 2023-10-09T09:05:14.564405+0000 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 15160/36447 objects degraded (41.595%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:14.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:14.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:14.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:15.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:15.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:15.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:15.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:15.196 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:15.226 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:15.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:15.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:15.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:15.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:15.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:15.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:15.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:15.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:15.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:15.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:15.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:15.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:15.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:15 smithi094 bash[59677]: cluster 2023-10-09T09:05:13.881131+0000 mgr.y (mgr.25421) 832 : cluster 0 pgmap v865: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 687 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 15160/33036 objects degraded (45.889%); 1733/33036 objects misplaced (5.246%); 985 KiB/s, 15 objects/s recovering 2023-10-09T09:05:15.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:15.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:15.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:15.855 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:15 smithi016 bash[75179]: cluster 2023-10-09T09:05:13.881131+0000 mgr.y (mgr.25421) 832 : cluster 0 pgmap v865: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 687 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 15160/33036 objects degraded (45.889%); 1733/33036 objects misplaced (5.246%); 985 KiB/s, 15 objects/s recovering 2023-10-09T09:05:15.855 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:05:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:05:15] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.33.4" 2023-10-09T09:05:15.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:15 smithi016 bash[77892]: cluster 2023-10-09T09:05:13.881131+0000 mgr.y (mgr.25421) 832 : cluster 0 pgmap v865: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 687 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 15160/33036 objects degraded (45.889%); 1733/33036 objects misplaced (5.246%); 985 KiB/s, 15 objects/s recovering 2023-10-09T09:05:15.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:15.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:15.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:15.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:16.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:16.110 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:16.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:16.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:16.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:16.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:16.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:16.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:16.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:16.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:16.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:16.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:16.613 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:16 smithi016 bash[75179]: cluster 2023-10-09T09:05:15.881855+0000 mgr.y (mgr.25421) 833 : cluster 0 pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 592 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 663 op/s; 15199/28470 objects degraded (53.386%); 1181/28470 objects misplaced (4.148%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:16.614 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:16 smithi016 bash[77892]: cluster 2023-10-09T09:05:15.881855+0000 mgr.y (mgr.25421) 833 : cluster 0 pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 592 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 663 op/s; 15199/28470 objects degraded (53.386%); 1181/28470 objects misplaced (4.148%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:16.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:16.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:16.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:16.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:16 smithi094 bash[59677]: cluster 2023-10-09T09:05:15.881855+0000 mgr.y (mgr.25421) 833 : cluster 0 pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 592 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 663 op/s; 15199/28470 objects degraded (53.386%); 1181/28470 objects misplaced (4.148%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:16.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:16.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:16.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:16.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:16.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:16.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:17.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:17.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:17.138 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:05:17.139 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:05:17.139 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-10-09T09:05:17.139 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:05:17.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.224 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:17.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:17.329 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:17.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:17.381 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:17.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:17.487 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:17.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:17.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.618 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:17.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:17.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:17.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:17.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:17.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:17.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:17.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:17.982 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:18.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:18.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:18.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:18.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:18.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:18.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:18.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:18.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:18.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:18.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:18.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:18.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:18.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:18.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:18.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:18.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:05:18.910 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:18.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:05:18.935 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:18.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:19.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:19.068 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:19.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:19.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:19.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:18 smithi016 bash[77892]: cluster 2023-10-09T09:05:17.882522+0000 mgr.y (mgr.25421) 834 : cluster 0 pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 496 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 15199/23841 objects degraded (63.752%); 1135/23841 objects misplaced (4.761%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:19.185 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:18 smithi016 bash[75179]: cluster 2023-10-09T09:05:17.882522+0000 mgr.y (mgr.25421) 834 : cluster 0 pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 496 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 15199/23841 objects degraded (63.752%); 1135/23841 objects misplaced (4.761%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:19.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:19.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:19.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:18 smithi094 bash[59677]: cluster 2023-10-09T09:05:17.882522+0000 mgr.y (mgr.25421) 834 : cluster 0 pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 496 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 15199/23841 objects degraded (63.752%); 1135/23841 objects misplaced (4.761%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:19.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:19.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:19.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:19.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:19.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:19.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:19.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:19.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:19.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:19.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:19.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:19.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:19.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:19.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:05:19.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:19.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:19.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:19.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:19.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:20.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:20.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:20.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:20.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:20.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:20.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:20.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:20.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:20 smithi016 bash[75179]: audit 2023-10-09T09:05:19.239922+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:20.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:20 smithi016 bash[75179]: audit 2023-10-09T09:05:19.244572+0000 mon.a (mon.0) 1132 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:20.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:20 smithi016 bash[75179]: cluster 2023-10-09T09:05:19.565356+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 15199/23841 objects degraded (63.752%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:20.392 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:20 smithi016 bash[77892]: audit 2023-10-09T09:05:19.239922+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:20.393 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:20 smithi016 bash[77892]: audit 2023-10-09T09:05:19.244572+0000 mon.a (mon.0) 1132 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:20.393 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:20 smithi016 bash[77892]: cluster 2023-10-09T09:05:19.565356+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 15199/23841 objects degraded (63.752%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:20.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:20.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:20.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:20.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:20 smithi094 bash[59677]: audit 2023-10-09T09:05:19.239922+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:20.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:20 smithi094 bash[59677]: audit 2023-10-09T09:05:19.244572+0000 mon.a (mon.0) 1132 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:20.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:20 smithi094 bash[59677]: cluster 2023-10-09T09:05:19.565356+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 15199/23841 objects degraded (63.752%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:20.558 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:20.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:20.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:20.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:20.858 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:20.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:20.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:20.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:20.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:21.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:21.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:21.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:21.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:21.260 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:21.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:21.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:21.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:21.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:21.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:21.522 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:21.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:21.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:21.637 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:21 smithi016 bash[75179]: cluster 2023-10-09T09:05:19.883094+0000 mgr.y (mgr.25421) 835 : cluster 0 pgmap v868: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 468 MiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 667 op/s; 15199/22509 objects degraded (67.524%); 1135/22509 objects misplaced (5.042%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:21.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:21 smithi016 bash[77892]: cluster 2023-10-09T09:05:19.883094+0000 mgr.y (mgr.25421) 835 : cluster 0 pgmap v868: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 468 MiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 667 op/s; 15199/22509 objects degraded (67.524%); 1135/22509 objects misplaced (5.042%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:21.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:21.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:21.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:21.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:21.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:21.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:21.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:21.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:21 smithi094 bash[59677]: cluster 2023-10-09T09:05:19.883094+0000 mgr.y (mgr.25421) 835 : cluster 0 pgmap v868: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 468 MiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 667 op/s; 15199/22509 objects degraded (67.524%); 1135/22509 objects misplaced (5.042%); 870 KiB/s, 16 objects/s recovering 2023-10-09T09:05:21.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:21.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:22.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:22.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:22.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:22.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:22.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:22.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:22.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:22.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:22.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:22.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:22.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:22.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:22.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:22.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:22.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:23.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:23.111 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:23.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:23.216 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:23.246 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:23.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:23.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:23.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:23.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:23.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:23.504 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:23.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:23.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:23.613 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:23.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:23.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:23 smithi094 bash[59677]: cluster 2023-10-09T09:05:21.884045+0000 mgr.y (mgr.25421) 836 : cluster 0 pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 331 MiB data, 4.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 683 op/s; 15128/15924 objects degraded (95.001%); 677/15924 objects misplaced (4.251%); 320 KiB/s, 14 objects/s recovering 2023-10-09T09:05:23.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:23.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:23 smithi016 bash[75179]: cluster 2023-10-09T09:05:21.884045+0000 mgr.y (mgr.25421) 836 : cluster 0 pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 331 MiB data, 4.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 683 op/s; 15128/15924 objects degraded (95.001%); 677/15924 objects misplaced (4.251%); 320 KiB/s, 14 objects/s recovering 2023-10-09T09:05:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:23 smithi016 bash[77892]: cluster 2023-10-09T09:05:21.884045+0000 mgr.y (mgr.25421) 836 : cluster 0 pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 331 MiB data, 4.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 683 op/s; 15128/15924 objects degraded (95.001%); 677/15924 objects misplaced (4.251%); 320 KiB/s, 14 objects/s recovering 2023-10-09T09:05:23.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:23.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:23.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:23.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:23.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:23.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:24.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:24.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:24.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:24.171 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:24.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:24.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:24.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:24.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:24.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:24.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:24.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:24.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:24.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:24.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:24.722 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:05:24.723 INFO:teuthology.orchestra.run.smithi016.stdout:{"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15128/12060 objects degraded (125.439%), 4 pgs degraded, 4 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1280,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":204,"num_osds":8,"num_up_osds":8,"osd_up_since":1696841235,"num_in_osds":5,"osd_in_since":1696842121,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":4020,"data_bytes":262412216,"bytes_used":4592492544,"bytes_avail":475382685696,"bytes_total":479975178240,"degraded_objects":15128,"degraded_total":12060,"degraded_ratio":1.2543946932006633,"misplaced_objects":642,"misplaced_total":12060,"misplaced_ratio":0.053233830845771136,"recovering_objects_per_sec":13,"recovering_bytes_per_sec":290042,"recovering_keys_per_sec":0,"num_objects_recovered":148,"num_bytes_recovered":3276800,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":719},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi016.ccdktz","status":"up:active","gid":14634}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"}},"servicemap":{"epoch":30,"modified":"2023-10-09T09:03:15.825457+0000","services":{}},"progress_events":{"73d2276e-8e1e-406f-ad35-c70c19f45358":{"message":"Global Recovery Event (110s)\n [=========================...] (remaining: 12s)","progress":0.89887642860412598,"add_to_ceph_s":true},"8402cf81-b0a2-4ab7-aed5-036e29e682ad":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 7m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T09:05:24.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:24.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:24.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:24 smithi094 bash[59677]: cluster 2023-10-09T09:05:24.566331+0000 mon.a (mon.0) 1134 : cluster 3 Health check update: Degraded data redundancy: 15128/15924 objects degraded (95.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:24.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:24.855 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:24.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:24.980 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:24 smithi016 bash[75179]: cluster 2023-10-09T09:05:24.566331+0000 mon.a (mon.0) 1134 : cluster 3 Health check update: Degraded data redundancy: 15128/15924 objects degraded (95.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:24.981 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:24 smithi016 bash[77892]: cluster 2023-10-09T09:05:24.566331+0000 mon.a (mon.0) 1134 : cluster 3 Health check update: Degraded data redundancy: 15128/15924 objects degraded (95.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:24.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:24.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:25.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:25.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:25.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:25.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:25.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:25.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:25.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:25.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:25.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:25.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:25.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:25.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:25.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:25.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:25.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:25.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:25 smithi016 bash[75179]: cluster 2023-10-09T09:05:23.884729+0000 mgr.y (mgr.25421) 837 : cluster 0 pgmap v870: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 250 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 719 op/s; 15128/12060 objects degraded (125.439%); 642/12060 objects misplaced (5.323%); 283 KiB/s, 13 objects/s recovering 2023-10-09T09:05:25.616 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:25 smithi016 bash[75179]: audit 2023-10-09T09:05:24.720843+0000 mon.a (mon.0) 1135 : audit 0 from='client.? 172.21.15.16:0/3360482997' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:05:25.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:25.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:25.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:25.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:25.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:25.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:25 smithi094 bash[59677]: cluster 2023-10-09T09:05:23.884729+0000 mgr.y (mgr.25421) 837 : cluster 0 pgmap v870: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 250 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 719 op/s; 15128/12060 objects degraded (125.439%); 642/12060 objects misplaced (5.323%); 283 KiB/s, 13 objects/s recovering 2023-10-09T09:05:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:25 smithi094 bash[59677]: audit 2023-10-09T09:05:24.720843+0000 mon.a (mon.0) 1135 : audit 0 from='client.? 172.21.15.16:0/3360482997' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:05:25.794 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:25.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:25.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:25.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:25.909 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:05:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:05:25] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-10-09T09:05:25.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:25 smithi016 bash[77892]: cluster 2023-10-09T09:05:23.884729+0000 mgr.y (mgr.25421) 837 : cluster 0 pgmap v870: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 250 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 719 op/s; 15128/12060 objects degraded (125.439%); 642/12060 objects misplaced (5.323%); 283 KiB/s, 13 objects/s recovering 2023-10-09T09:05:25.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:25 smithi016 bash[77892]: audit 2023-10-09T09:05:24.720843+0000 mon.a (mon.0) 1135 : audit 0 from='client.? 172.21.15.16:0/3360482997' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:05:25.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:25.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:26.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:26.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:26.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:26.196 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:26.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:26.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:26.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:26.458 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T09:05:26.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:05:26.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:26.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:26.609 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:26.610 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:26 smithi016 bash[75179]: cluster 2023-10-09T09:05:25.885908+0000 mgr.y (mgr.25421) 838 : cluster 0 pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 155 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 710 op/s; 15053/7467 objects degraded (201.594%); 217/7467 objects misplaced (2.906%); 267 KiB/s, 18 objects/s recovering 2023-10-09T09:05:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:26.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:26.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:26.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:26.854 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:26 smithi094 bash[59677]: cluster 2023-10-09T09:05:25.885908+0000 mgr.y (mgr.25421) 838 : cluster 0 pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 155 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 710 op/s; 15053/7467 objects degraded (201.594%); 217/7467 objects misplaced (2.906%); 267 KiB/s, 18 objects/s recovering 2023-10-09T09:05:26.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:26.869 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:26.869 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:26.872 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:26.907 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:26 smithi016 bash[77892]: cluster 2023-10-09T09:05:25.885908+0000 mgr.y (mgr.25421) 838 : cluster 0 pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 155 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 710 op/s; 15053/7467 objects degraded (201.594%); 217/7467 objects misplaced (2.906%); 267 KiB/s, 18 objects/s recovering 2023-10-09T09:05:26.973 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:26.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:26.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:26.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:27.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:27.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:27.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:27.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:27.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:27.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:27.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:27.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:27.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:27.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:27.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:27.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:27.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:27.526 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 88236 objects 2023-10-09T09:05:27.526 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :138.616 2023-10-09T09:05:27.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:27.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:27.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:27.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:27.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:27.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:27.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:27.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:27.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:27.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:27.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:27.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:27.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:28.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:28.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:28.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:28.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:28.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:28.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:28.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:28.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:28.417 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:28.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:28.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:28.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:28.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:28.676 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:28.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:28.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:28.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:28.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:28.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:28.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:28.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:29.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:29.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:28 smithi016 bash[77892]: cluster 2023-10-09T09:05:27.886631+0000 mgr.y (mgr.25421) 839 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 65 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 701 op/s; 15053/3189 objects degraded (472.029%); 176/3189 objects misplaced (5.519%); 0 B/s, 12 objects/s recovering 2023-10-09T09:05:29.078 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:28 smithi016 bash[75179]: cluster 2023-10-09T09:05:27.886631+0000 mgr.y (mgr.25421) 839 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 65 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 701 op/s; 15053/3189 objects degraded (472.029%); 176/3189 objects misplaced (5.519%); 0 B/s, 12 objects/s recovering 2023-10-09T09:05:29.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:29.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:29.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:29.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:28 smithi094 bash[59677]: cluster 2023-10-09T09:05:27.886631+0000 mgr.y (mgr.25421) 839 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 65 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 701 op/s; 15053/3189 objects degraded (472.029%); 176/3189 objects misplaced (5.519%); 0 B/s, 12 objects/s recovering 2023-10-09T09:05:29.234 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:29.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:29.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:29.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:29.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:29.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:29.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:29.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:29.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:29.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:29.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:29.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:29.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:29.673 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T09:05:29.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T09:05:29.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:29.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:29.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:29.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:29.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:29.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:29.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:29.945 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:29 smithi016 bash[77892]: cluster 2023-10-09T09:05:29.567537+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 15053/3189 objects degraded (472.029%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:29.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:30.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:30.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:30.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:30.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:30.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:29 smithi094 bash[59677]: cluster 2023-10-09T09:05:29.567537+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 15053/3189 objects degraded (472.029%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:30.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:29 smithi016 bash[75179]: cluster 2023-10-09T09:05:29.567537+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 15053/3189 objects degraded (472.029%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:30.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:30.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:30.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:30.379 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:30.416 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:30.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:30.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:30.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:30.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:30.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:30.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:30.822 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:30.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:30.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:30.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:30.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:30.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:30.975 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:31.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:31.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:31.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:31.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:31.301 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:31.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:31.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:31.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:31.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:31.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:31.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:31.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:31.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:31.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:31.797 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:31 smithi016 bash[75179]: cluster 2023-10-09T09:05:29.887255+0000 mgr.y (mgr.25421) 840 : cluster 0 pgmap v873: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 43 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 603 op/s; 15053/2100 objects degraded (716.810%); 176/2100 objects misplaced (8.381%); 0 B/s, 12 objects/s recovering 2023-10-09T09:05:31.798 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:31 smithi016 bash[77892]: cluster 2023-10-09T09:05:29.887255+0000 mgr.y (mgr.25421) 840 : cluster 0 pgmap v873: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 43 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 603 op/s; 15053/2100 objects degraded (716.810%); 176/2100 objects misplaced (8.381%); 0 B/s, 12 objects/s recovering 2023-10-09T09:05:31.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:31.838 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:31 smithi094 bash[59677]: cluster 2023-10-09T09:05:29.887255+0000 mgr.y (mgr.25421) 840 : cluster 0 pgmap v873: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 43 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 603 op/s; 15053/2100 objects degraded (716.810%); 176/2100 objects misplaced (8.381%); 0 B/s, 12 objects/s recovering 2023-10-09T09:05:31.856 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:31.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:31.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:31.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:31.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:32.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:32.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:32.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:32.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:32.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:32.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:32.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:32.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:32.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:32.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:32.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:32.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:32.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:32.616 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:32.647 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:32.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:32.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:32.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:32.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:32.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:32.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:32.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:33.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:33.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:33.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:33.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:33.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:33.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:33.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:33.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:33.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:33.447 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:33.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:33.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:33.586 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:33.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:33.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:33.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:33 smithi094 bash[59677]: cluster 2023-10-09T09:05:31.888414+0000 mgr.y (mgr.25421) 841 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 14727/72 objects degraded (20454.167%); 0 B/s, 39 objects/s recovering 2023-10-09T09:05:33.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:33.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:33.843 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:33 smithi016 bash[75179]: cluster 2023-10-09T09:05:31.888414+0000 mgr.y (mgr.25421) 841 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 14727/72 objects degraded (20454.167%); 0 B/s, 39 objects/s recovering 2023-10-09T09:05:33.843 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:33 smithi016 bash[77892]: cluster 2023-10-09T09:05:31.888414+0000 mgr.y (mgr.25421) 841 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 14727/72 objects degraded (20454.167%); 0 B/s, 39 objects/s recovering 2023-10-09T09:05:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:33.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:33.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:33.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:34.015 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:34.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:34.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:34.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:34.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:34.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:34.239 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:34.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:34.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:34.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:34.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:34.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:34.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:34.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:34.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:34.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:34.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:34.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:34.643 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:34.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:34.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:34.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:34.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:34.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:34.799 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:34 smithi016 bash[77892]: audit 2023-10-09T09:05:34.235497+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:34.799 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:34 smithi016 bash[77892]: cluster 2023-10-09T09:05:34.568776+0000 mon.a (mon.0) 1138 : cluster 3 Health check update: Degraded data redundancy: 14727/72 objects degraded (20454.167%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:34.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:34 smithi016 bash[75179]: audit 2023-10-09T09:05:34.235497+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:34.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:34 smithi016 bash[75179]: cluster 2023-10-09T09:05:34.568776+0000 mon.a (mon.0) 1138 : cluster 3 Health check update: Degraded data redundancy: 14727/72 objects degraded (20454.167%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:34.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:34.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:34.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:34.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:34.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:34 smithi094 bash[59677]: audit 2023-10-09T09:05:34.235497+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:34.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:34 smithi094 bash[59677]: cluster 2023-10-09T09:05:34.568776+0000 mon.a (mon.0) 1138 : cluster 3 Health check update: Degraded data redundancy: 14727/72 objects degraded (20454.167%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:34.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:35.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:35.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:35.161 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:35.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:35.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:35.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:35.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:35.516 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:35.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:35.648 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:35.681 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:35.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:35.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:35.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:35.917 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:35 smithi016 bash[75179]: cluster 2023-10-09T09:05:33.888978+0000 mgr.y (mgr.25421) 842 : cluster 0 pgmap v875: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 440 op/s; 14727/72 objects degraded (20454.167%); 0 B/s, 33 objects/s recovering 2023-10-09T09:05:35.917 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:05:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:05:35] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-10-09T09:05:35.918 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:35 smithi016 bash[77892]: cluster 2023-10-09T09:05:33.888978+0000 mgr.y (mgr.25421) 842 : cluster 0 pgmap v875: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 440 op/s; 14727/72 objects degraded (20454.167%); 0 B/s, 33 objects/s recovering 2023-10-09T09:05:35.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:35.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:35.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:35 smithi094 bash[59677]: cluster 2023-10-09T09:05:33.888978+0000 mgr.y (mgr.25421) 842 : cluster 0 pgmap v875: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 440 op/s; 14727/72 objects degraded (20454.167%); 0 B/s, 33 objects/s recovering 2023-10-09T09:05:36.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:36.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:36.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:36.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:36.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:36.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:36.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:36.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:36.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:36.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:36.687 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:36.715 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:36.818 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:36.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:36.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:36.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:36.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:36.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:37.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:37.110 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:37.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:37.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:37.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:37.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:37.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:37.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:37.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:37.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:37.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:37.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:37.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:37.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:37.625 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:37.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:37.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:37.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:37.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:37.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:37.866 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:37 smithi094 bash[59677]: cluster 2023-10-09T09:05:35.889623+0000 mgr.y (mgr.25421) 843 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 332 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 76 objects/s recovering 2023-10-09T09:05:37.884 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:37.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:37.988 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:37.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:37 smithi016 bash[77892]: cluster 2023-10-09T09:05:35.889623+0000 mgr.y (mgr.25421) 843 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 332 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 76 objects/s recovering 2023-10-09T09:05:38.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:37 smithi016 bash[75179]: cluster 2023-10-09T09:05:35.889623+0000 mgr.y (mgr.25421) 843 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 332 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 76 objects/s recovering 2023-10-09T09:05:38.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:38.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:38.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:38.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:38.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:38.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:38.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:38.411 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:38.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:38.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:38.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:38.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:38.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:38.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:38 smithi094 bash[59677]: cluster 2023-10-09T09:05:37.890137+0000 mgr.y (mgr.25421) 844 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 205 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 69 objects/s recovering 2023-10-09T09:05:38.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:38.805 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:38.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:38.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:38.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:38 smithi016 bash[75179]: cluster 2023-10-09T09:05:37.890137+0000 mgr.y (mgr.25421) 844 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 205 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 69 objects/s recovering 2023-10-09T09:05:38.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:38 smithi016 bash[77892]: cluster 2023-10-09T09:05:37.890137+0000 mgr.y (mgr.25421) 844 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 205 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 69 objects/s recovering 2023-10-09T09:05:38.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:38.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:39.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:39.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:39.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:39.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:39.301 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:39.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:39.440 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:39.474 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:39.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:39.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:39.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:39.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:39.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:39.735 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:39.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:39.851 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:39 smithi094 bash[59677]: cluster 2023-10-09T09:05:39.570420+0000 mon.a (mon.0) 1139 : cluster 3 Health check update: Degraded data redundancy: 14215/72 objects degraded (19743.056%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:39.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:39.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:39.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:39.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:39 smithi016 bash[75179]: cluster 2023-10-09T09:05:39.570420+0000 mon.a (mon.0) 1139 : cluster 3 Health check update: Degraded data redundancy: 14215/72 objects degraded (19743.056%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:39.983 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:39 smithi016 bash[77892]: cluster 2023-10-09T09:05:39.570420+0000 mon.a (mon.0) 1139 : cluster 3 Health check update: Degraded data redundancy: 14215/72 objects degraded (19743.056%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:39.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:39.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:39.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:40.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:40.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:40.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:40.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:40.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:40.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:40.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:40.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:40.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:40.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:40.481 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:40.516 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:40.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:40.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:40.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:40.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:40.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:40.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:40 smithi094 bash[59677]: cluster 2023-10-09T09:05:39.890571+0000 mgr.y (mgr.25421) 845 : cluster 0 pgmap v878: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 86 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 69 objects/s recovering 2023-10-09T09:05:40.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:40.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:40.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:40.891 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:40 smithi016 bash[77892]: cluster 2023-10-09T09:05:39.890571+0000 mgr.y (mgr.25421) 845 : cluster 0 pgmap v878: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 86 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 69 objects/s recovering 2023-10-09T09:05:40.892 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:40 smithi016 bash[75179]: cluster 2023-10-09T09:05:39.890571+0000 mgr.y (mgr.25421) 845 : cluster 0 pgmap v878: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 86 op/s; 14215/72 objects degraded (19743.056%); 0 B/s, 69 objects/s recovering 2023-10-09T09:05:40.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:40.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:41.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:41.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:41.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:41.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:41.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:41.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:41.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:41.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:41.554 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:41.589 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:41.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:41.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:41.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:41.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:41.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:41.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:41.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:41.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:42.081 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:42.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:42.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:42.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:42.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:42.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:42.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:42.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:42.373 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:42.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:42.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:42.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:42.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:42.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:42.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:42.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:42.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:42.741 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:42.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:42.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:42.876 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:42.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:42.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:43.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:43.046 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:43.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:43.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:43.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:42 smithi094 bash[59677]: cluster 2023-10-09T09:05:41.891255+0000 mgr.y (mgr.25421) 846 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 13634/72 objects degraded (18936.111%); 0 B/s, 118 objects/s recovering 2023-10-09T09:05:43.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:42 smithi016 bash[75179]: cluster 2023-10-09T09:05:41.891255+0000 mgr.y (mgr.25421) 846 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 13634/72 objects degraded (18936.111%); 0 B/s, 118 objects/s recovering 2023-10-09T09:05:43.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:42 smithi016 bash[77892]: cluster 2023-10-09T09:05:41.891255+0000 mgr.y (mgr.25421) 846 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 13634/72 objects degraded (18936.111%); 0 B/s, 118 objects/s recovering 2023-10-09T09:05:43.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:43.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:43.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:43.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:43.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:43.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:43.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:43.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:43.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:43.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:43.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:43.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:43.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:43.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:43.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:43.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:43.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:43.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:44.049 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:44.083 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:44.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:44.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:44.219 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:44.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:44.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:44.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:44.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:44.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:44.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:44.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:44.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:44.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:44.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:44.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:44.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:44.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:44.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:44.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:44.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:44.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:44.978 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:44.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:44 smithi094 bash[59677]: cluster 2023-10-09T09:05:43.891925+0000 mgr.y (mgr.25421) 847 : cluster 0 pgmap v880: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 13634/72 objects degraded (18936.111%); 0 B/s, 91 objects/s recovering 2023-10-09T09:05:44.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:44 smithi094 bash[59677]: cluster 2023-10-09T09:05:44.572010+0000 mon.a (mon.0) 1140 : cluster 3 Health check update: Degraded data redundancy: 13634/72 objects degraded (18936.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:45.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:45.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:45.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:45.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:45.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:45.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:44 smithi016 bash[75179]: cluster 2023-10-09T09:05:43.891925+0000 mgr.y (mgr.25421) 847 : cluster 0 pgmap v880: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 13634/72 objects degraded (18936.111%); 0 B/s, 91 objects/s recovering 2023-10-09T09:05:45.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:44 smithi016 bash[75179]: cluster 2023-10-09T09:05:44.572010+0000 mon.a (mon.0) 1140 : cluster 3 Health check update: Degraded data redundancy: 13634/72 objects degraded (18936.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:44 smithi016 bash[77892]: cluster 2023-10-09T09:05:43.891925+0000 mgr.y (mgr.25421) 847 : cluster 0 pgmap v880: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 13634/72 objects degraded (18936.111%); 0 B/s, 91 objects/s recovering 2023-10-09T09:05:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:44 smithi016 bash[77892]: cluster 2023-10-09T09:05:44.572010+0000 mon.a (mon.0) 1140 : cluster 3 Health check update: Degraded data redundancy: 13634/72 objects degraded (18936.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:45.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:45.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:45.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:45.401 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:45.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:45.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:45.533 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:45.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:45.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:45.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:45.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:45.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:45.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:45.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:45.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:45.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:45.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:45.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:45.947 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:05:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:05:45] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-10-09T09:05:46.015 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:46.051 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:46.155 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:46.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:45 smithi094 bash[59677]: audit 2023-10-09T09:05:45.007389+0000 mon.a (mon.0) 1141 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:45 smithi094 bash[59677]: audit 2023-10-09T09:05:45.430677+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:46.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:45 smithi094 bash[59677]: audit 2023-10-09T09:05:45.439063+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:46.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:45 smithi016 bash[75179]: audit 2023-10-09T09:05:45.007389+0000 mon.a (mon.0) 1141 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:46.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:45 smithi016 bash[75179]: audit 2023-10-09T09:05:45.430677+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:46.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:45 smithi016 bash[75179]: audit 2023-10-09T09:05:45.439063+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:46.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:45 smithi016 bash[77892]: audit 2023-10-09T09:05:45.007389+0000 mon.a (mon.0) 1141 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:46.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:45 smithi016 bash[77892]: audit 2023-10-09T09:05:45.430677+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:46.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:45 smithi016 bash[77892]: audit 2023-10-09T09:05:45.439063+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:46.290 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:46.321 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:46.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:46.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:46.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:46.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:46.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:46.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:46.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:46.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:46.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:46.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:46.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:46.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:47.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:47.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:47.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:47.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:47.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:47.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:47.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:47.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:47.565 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:47.598 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:47.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:47.713 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:47 smithi016 bash[75179]: cluster 2023-10-09T09:05:45.892535+0000 mgr.y (mgr.25421) 848 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 146 objects/s recovering 2023-10-09T09:05:47.714 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:47 smithi016 bash[77892]: cluster 2023-10-09T09:05:45.892535+0000 mgr.y (mgr.25421) 848 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 146 objects/s recovering 2023-10-09T09:05:47.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:47.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:47.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:47 smithi094 bash[59677]: cluster 2023-10-09T09:05:45.892535+0000 mgr.y (mgr.25421) 848 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 146 objects/s recovering 2023-10-09T09:05:47.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:47.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:47.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:47.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:47.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:47.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:47.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:48.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:48.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:48.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:48.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:48.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:48.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:48.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:48.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:48.377 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:48.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:48.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:48.481 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:48.513 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:48.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:48.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:48.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:48.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:48.746 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:48.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:48.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:48.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:48.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:48.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:48.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:48.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:49.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:49.039 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:49.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:49.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:49.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:49.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:49.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:49.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:49.506 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:49.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:49.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:49.617 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:49 smithi016 bash[75179]: cluster 2023-10-09T09:05:47.893503+0000 mgr.y (mgr.25421) 849 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 103 objects/s recovering 2023-10-09T09:05:49.618 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:49 smithi016 bash[75179]: audit 2023-10-09T09:05:49.235356+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:49.618 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:49 smithi016 bash[77892]: cluster 2023-10-09T09:05:47.893503+0000 mgr.y (mgr.25421) 849 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 103 objects/s recovering 2023-10-09T09:05:49.618 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:49 smithi016 bash[77892]: audit 2023-10-09T09:05:49.235356+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:49.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:49.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:49.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:49 smithi094 bash[59677]: cluster 2023-10-09T09:05:47.893503+0000 mgr.y (mgr.25421) 849 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 103 objects/s recovering 2023-10-09T09:05:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:49 smithi094 bash[59677]: audit 2023-10-09T09:05:49.235356+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:05:49.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:49.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:49.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:49.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:49.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:50.000 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:50.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:50.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:50.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:50.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:50.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:50.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:50.285 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:50.341 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:50.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:50.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:50.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:50.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:50.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:50.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:50.581 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:50 smithi094 bash[59677]: cluster 2023-10-09T09:05:49.572973+0000 mon.a (mon.0) 1145 : cluster 3 Health check update: Degraded data redundancy: 12971/72 objects degraded (18015.278%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:50.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:50.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:50.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:50.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:50.718 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:50 smithi016 bash[75179]: cluster 2023-10-09T09:05:49.572973+0000 mon.a (mon.0) 1145 : cluster 3 Health check update: Degraded data redundancy: 12971/72 objects degraded (18015.278%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:50.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:50 smithi016 bash[77892]: cluster 2023-10-09T09:05:49.572973+0000 mon.a (mon.0) 1145 : cluster 3 Health check update: Degraded data redundancy: 12971/72 objects degraded (18015.278%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:50.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:50.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:50.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:50.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:50.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:50.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:51.001 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:51.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:51.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:51.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:51.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:51.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:51.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:51.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:51.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:51.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:51.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:51.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:51.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:51.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:51.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:51.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:51.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:51.782 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:51.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:51.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:51 smithi016 bash[75179]: cluster 2023-10-09T09:05:49.894222+0000 mgr.y (mgr.25421) 850 : cluster 0 pgmap v883: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 103 objects/s recovering 2023-10-09T09:05:51.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:51 smithi016 bash[75179]: audit 2023-10-09T09:05:51.191246+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:51.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:51 smithi016 bash[75179]: audit 2023-10-09T09:05:51.197959+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:51.897 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:51 smithi016 bash[77892]: cluster 2023-10-09T09:05:49.894222+0000 mgr.y (mgr.25421) 850 : cluster 0 pgmap v883: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 103 objects/s recovering 2023-10-09T09:05:51.897 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:51 smithi016 bash[77892]: audit 2023-10-09T09:05:51.191246+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:51.898 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:51 smithi016 bash[77892]: audit 2023-10-09T09:05:51.197959+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:51.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:51.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:51.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:51 smithi094 bash[59677]: cluster 2023-10-09T09:05:49.894222+0000 mgr.y (mgr.25421) 850 : cluster 0 pgmap v883: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12971/72 objects degraded (18015.278%); 0 B/s, 103 objects/s recovering 2023-10-09T09:05:51.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:51 smithi094 bash[59677]: audit 2023-10-09T09:05:51.191246+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:51.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:51 smithi094 bash[59677]: audit 2023-10-09T09:05:51.197959+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:52.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:52.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:52.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:52.179 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:52.284 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:52.318 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:52.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:52.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:52.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:52.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:52.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:52.678 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:52.715 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:52.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:52.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:52.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:52.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:52.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:52.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:53.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:53.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:53.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:53.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:53.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:53.261 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:53.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:53.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:53.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:53.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:53.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:53.495 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:53.525 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:53.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:53.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:53.629 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:53.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:53.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:53.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:53.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:53 smithi016 bash[77892]: cluster 2023-10-09T09:05:51.895042+0000 mgr.y (mgr.25421) 851 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12087/72 objects degraded (16787.500%); 0 B/s, 177 objects/s recovering 2023-10-09T09:05:53.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:53 smithi016 bash[75179]: cluster 2023-10-09T09:05:51.895042+0000 mgr.y (mgr.25421) 851 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12087/72 objects degraded (16787.500%); 0 B/s, 177 objects/s recovering 2023-10-09T09:05:53.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:53.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:53 smithi094 bash[59677]: cluster 2023-10-09T09:05:51.895042+0000 mgr.y (mgr.25421) 851 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12087/72 objects degraded (16787.500%); 0 B/s, 177 objects/s recovering 2023-10-09T09:05:53.799 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:53.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:53.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:53.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:53.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:53.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:54.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:54.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:54.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:54.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:54.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:54.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:54.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:54.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:54.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:54.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:54.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:54.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:54.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:54.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:54.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:54.569 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:54.602 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:54.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:54.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:54.717 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:54 smithi094 bash[59677]: cluster 2023-10-09T09:05:54.573994+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 12087/72 objects degraded (16787.500%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:54.734 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:54.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:54.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:54.869 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:54.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:54.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:54.981 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:54 smithi016 bash[75179]: cluster 2023-10-09T09:05:54.573994+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 12087/72 objects degraded (16787.500%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:54.981 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:54 smithi016 bash[77892]: cluster 2023-10-09T09:05:54.573994+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 12087/72 objects degraded (16787.500%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:55.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:55.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:55.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:55.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:55.233 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:55.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:55.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:55.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:55.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:55.506 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:55.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:55.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:55.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:55.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:55.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:55.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:55.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:55.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:55.905 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:05:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:05:55] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.33.4" 2023-10-09T09:05:55.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:55 smithi016 bash[75179]: cluster 2023-10-09T09:05:53.895776+0000 mgr.y (mgr.25421) 852 : cluster 0 pgmap v885: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12087/72 objects degraded (16787.500%); 0 B/s, 128 objects/s recovering 2023-10-09T09:05:55.906 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:55 smithi016 bash[77892]: cluster 2023-10-09T09:05:53.895776+0000 mgr.y (mgr.25421) 852 : cluster 0 pgmap v885: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12087/72 objects degraded (16787.500%); 0 B/s, 128 objects/s recovering 2023-10-09T09:05:55.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:55.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:55.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:55.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:55 smithi094 bash[59677]: cluster 2023-10-09T09:05:53.895776+0000 mgr.y (mgr.25421) 852 : cluster 0 pgmap v885: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 12087/72 objects degraded (16787.500%); 0 B/s, 128 objects/s recovering 2023-10-09T09:05:56.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:56.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:56.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:56.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:56.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:56.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:56.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:56.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:56.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:56.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:56.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:56.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:56.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:56.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:56.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[75179]: cluster 2023-10-09T09:05:55.595453+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[75179]: cluster 2023-10-09T09:05:55.595760+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[75179]: cluster 2023-10-09T09:05:55.598327+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e205: 8 total, 8 up, 5 in 2023-10-09T09:05:56.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[75179]: cluster 2023-10-09T09:05:55.607974+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[75179]: cluster 2023-10-09T09:05:55.610467+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[75179]: cluster 2023-10-09T09:05:55.610586+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[75179]: cluster 2023-10-09T09:05:55.896528+0000 mgr.y (mgr.25421) 853 : cluster 0 pgmap v887: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 10738/72 objects degraded (14913.889%); 0 B/s, 223 objects/s recovering 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[77892]: cluster 2023-10-09T09:05:55.595453+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[77892]: cluster 2023-10-09T09:05:55.595760+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[77892]: cluster 2023-10-09T09:05:55.598327+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e205: 8 total, 8 up, 5 in 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[77892]: cluster 2023-10-09T09:05:55.607974+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[77892]: cluster 2023-10-09T09:05:55.610467+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[77892]: cluster 2023-10-09T09:05:55.610586+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:56 smithi016 bash[77892]: cluster 2023-10-09T09:05:55.896528+0000 mgr.y (mgr.25421) 853 : cluster 0 pgmap v887: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 10738/72 objects degraded (14913.889%); 0 B/s, 223 objects/s recovering 2023-10-09T09:05:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:56.784 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:56.810 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:56.914 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:56.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:56 smithi094 bash[59677]: cluster 2023-10-09T09:05:55.595453+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:56 smithi094 bash[59677]: cluster 2023-10-09T09:05:55.595760+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:56 smithi094 bash[59677]: cluster 2023-10-09T09:05:55.598327+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e205: 8 total, 8 up, 5 in 2023-10-09T09:05:56.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:56 smithi094 bash[59677]: cluster 2023-10-09T09:05:55.607974+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:56 smithi094 bash[59677]: cluster 2023-10-09T09:05:55.610467+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:56 smithi094 bash[59677]: cluster 2023-10-09T09:05:55.610586+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T09:05:56.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:56 smithi094 bash[59677]: cluster 2023-10-09T09:05:55.896528+0000 mgr.y (mgr.25421) 853 : cluster 0 pgmap v887: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 10738/72 objects degraded (14913.889%); 0 B/s, 223 objects/s recovering 2023-10-09T09:05:56.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:56.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:57.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:57.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:57.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:57.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:57.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:57.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:57.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:57.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:57.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:57.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:57.553 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:05:57.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:57.576 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:05:57.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:57.605 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:05:57.665 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:05:57.689 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:05:57.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:57.716 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:05:57.729 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:57.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:57.748 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:05:57.772 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:05:57.774 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.774 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.774 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.774 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.774 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.775 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.775 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.775 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:05:57.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:57.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:57.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[75179]: cluster 2023-10-09T09:05:56.605724+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e206: 8 total, 8 up, 5 in 2023-10-09T09:05:57.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[75179]: cluster 2023-10-09T09:05:56.608132+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[75179]: cluster 2023-10-09T09:05:56.611688+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[75179]: cluster 2023-10-09T09:05:56.630713+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[75179]: cluster 2023-10-09T09:05:56.703038+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[75179]: cluster 2023-10-09T09:05:56.703051+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[75179]: audit 2023-10-09T09:05:57.526048+0000 mon.a (mon.0) 1151 : audit 0 from='client.? 172.21.15.16:0/1699013787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:05:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:57.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[77892]: cluster 2023-10-09T09:05:56.605724+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e206: 8 total, 8 up, 5 in 2023-10-09T09:05:57.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[77892]: cluster 2023-10-09T09:05:56.608132+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[77892]: cluster 2023-10-09T09:05:56.611688+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[77892]: cluster 2023-10-09T09:05:56.630713+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[77892]: cluster 2023-10-09T09:05:56.703038+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[77892]: cluster 2023-10-09T09:05:56.703051+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:57 smithi016 bash[77892]: audit 2023-10-09T09:05:57.526048+0000 mon.a (mon.0) 1151 : audit 0 from='client.? 172.21.15.16:0/1699013787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:05:57.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:57.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:57 smithi094 bash[59677]: cluster 2023-10-09T09:05:56.605724+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e206: 8 total, 8 up, 5 in 2023-10-09T09:05:57.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:57 smithi094 bash[59677]: cluster 2023-10-09T09:05:56.608132+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:57 smithi094 bash[59677]: cluster 2023-10-09T09:05:56.611688+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:57 smithi094 bash[59677]: cluster 2023-10-09T09:05:56.630713+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.970 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:57 smithi094 bash[59677]: cluster 2023-10-09T09:05:56.703038+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.971 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:57 smithi094 bash[59677]: cluster 2023-10-09T09:05:56.703051+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T09:05:57.971 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:57 smithi094 bash[59677]: audit 2023-10-09T09:05:57.526048+0000 mon.a (mon.0) 1151 : audit 0 from='client.? 172.21.15.16:0/1699013787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:05:57.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:57.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:57.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:58.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:58.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:58.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:58.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:58.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:58.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:05:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:58.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:05:58.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:58.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:58.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:58.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:58.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:58.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:58.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:58.650 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:58.677 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:58.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[75179]: cluster 2023-10-09T09:05:57.609278+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[75179]: cluster 2023-10-09T09:05:57.612502+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e207: 8 total, 8 up, 5 in 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[75179]: cluster 2023-10-09T09:05:57.612616+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[75179]: cluster 2023-10-09T09:05:57.615566+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[75179]: cluster 2023-10-09T09:05:57.617965+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[75179]: cluster 2023-10-09T09:05:57.897282+0000 mgr.y (mgr.25421) 854 : cluster 0 pgmap v890: 89 pgs: 2 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 10738/72 objects degraded (14913.889%); 0 B/s, 224 objects/s recovering 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[77892]: cluster 2023-10-09T09:05:57.609278+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[77892]: cluster 2023-10-09T09:05:57.612502+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e207: 8 total, 8 up, 5 in 2023-10-09T09:05:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[77892]: cluster 2023-10-09T09:05:57.612616+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[77892]: cluster 2023-10-09T09:05:57.615566+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[77892]: cluster 2023-10-09T09:05:57.617965+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:58 smithi016 bash[77892]: cluster 2023-10-09T09:05:57.897282+0000 mgr.y (mgr.25421) 854 : cluster 0 pgmap v890: 89 pgs: 2 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 10738/72 objects degraded (14913.889%); 0 B/s, 224 objects/s recovering 2023-10-09T09:05:58.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:58.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:58.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:58.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:58.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:58.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:05:58.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:58.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:05:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:58 smithi094 bash[59677]: cluster 2023-10-09T09:05:57.609278+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:58 smithi094 bash[59677]: cluster 2023-10-09T09:05:57.612502+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e207: 8 total, 8 up, 5 in 2023-10-09T09:05:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:58 smithi094 bash[59677]: cluster 2023-10-09T09:05:57.612616+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:58 smithi094 bash[59677]: cluster 2023-10-09T09:05:57.615566+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:58 smithi094 bash[59677]: cluster 2023-10-09T09:05:57.617965+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T09:05:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:58 smithi094 bash[59677]: cluster 2023-10-09T09:05:57.897282+0000 mgr.y (mgr.25421) 854 : cluster 0 pgmap v890: 89 pgs: 2 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 10738/72 objects degraded (14913.889%); 0 B/s, 224 objects/s recovering 2023-10-09T09:05:59.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:59.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:59.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:05:59.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:59.190 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:59.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:59.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:05:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:05:59.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:59.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:05:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:05:59.555 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:59.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:05:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:05:59.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:59.702 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: cluster 2023-10-09T09:05:58.617062+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: cluster 2023-10-09T09:05:58.620386+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: cluster 2023-10-09T09:05:58.621356+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e208: 8 total, 8 up, 5 in 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: cluster 2023-10-09T09:05:58.622342+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: cluster 2023-10-09T09:05:58.624752+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: cluster 2023-10-09T09:05:58.627053+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: audit 2023-10-09T09:05:59.040554+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: audit 2023-10-09T09:05:59.046787+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: audit 2023-10-09T09:05:59.518847+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: audit 2023-10-09T09:05:59.520063+0000 mon.a (mon.0) 1157 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:59.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: audit 2023-10-09T09:05:59.526946+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.704 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:05:59 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.575139+0000 mon.a (mon.0) 1159 : cluster 3 Health check update: Degraded data redundancy: 10738/72 objects degraded (14913.889%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:59.723 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:05:59.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:05:59.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:59.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:05:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:05:59.864 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: cluster 2023-10-09T09:05:58.617062+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: cluster 2023-10-09T09:05:58.620386+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: cluster 2023-10-09T09:05:58.621356+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e208: 8 total, 8 up, 5 in 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: cluster 2023-10-09T09:05:58.622342+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: cluster 2023-10-09T09:05:58.624752+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: cluster 2023-10-09T09:05:58.627053+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: audit 2023-10-09T09:05:59.040554+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: audit 2023-10-09T09:05:59.046787+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: audit 2023-10-09T09:05:59.518847+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: audit 2023-10-09T09:05:59.520063+0000 mon.a (mon.0) 1157 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:59.865 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: audit 2023-10-09T09:05:59.526946+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.575139+0000 mon.a (mon.0) 1159 : cluster 3 Health check update: Degraded data redundancy: 10738/72 objects degraded (14913.889%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: cluster 2023-10-09T09:05:58.617062+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: cluster 2023-10-09T09:05:58.620386+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: cluster 2023-10-09T09:05:58.621356+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e208: 8 total, 8 up, 5 in 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: cluster 2023-10-09T09:05:58.622342+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: cluster 2023-10-09T09:05:58.624752+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: cluster 2023-10-09T09:05:58.627053+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:05:59.866 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: audit 2023-10-09T09:05:59.040554+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: audit 2023-10-09T09:05:59.046787+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: audit 2023-10-09T09:05:59.518847+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:59.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: audit 2023-10-09T09:05:59.520063+0000 mon.a (mon.0) 1157 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:59.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: audit 2023-10-09T09:05:59.526946+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:05:59.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:05:59 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.575139+0000 mon.a (mon.0) 1159 : cluster 3 Health check update: Degraded data redundancy: 10738/72 objects degraded (14913.889%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:05:59.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:59.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:05:59.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:05:59.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:00.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:00.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:00.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:00.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:00.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:00.243 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:00.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:00.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:00.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:00.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:00.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:00.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:00.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:00.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:00.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:00.631 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:00.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:00.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:00.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:00.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:00.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:00.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:00.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.120491+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:06:00.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.120512+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.629651+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e209: 8 total, 8 up, 5 in 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.633448+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.720695+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.720827+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.721021+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[75179]: cluster 2023-10-09T09:05:59.897691+0000 mgr.y (mgr.25421) 855 : cluster 0 pgmap v893: 89 pgs: 3 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 9898/72 objects degraded (13747.222%); 0 B/s, 209 objects/s recovering 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.120491+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:06:00.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.120512+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:06:00.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.629651+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e209: 8 total, 8 up, 5 in 2023-10-09T09:06:00.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.633448+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.720695+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.720827+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.721021+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:00 smithi016 bash[77892]: cluster 2023-10-09T09:05:59.897691+0000 mgr.y (mgr.25421) 855 : cluster 0 pgmap v893: 89 pgs: 3 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 9898/72 objects degraded (13747.222%); 0 B/s, 209 objects/s recovering 2023-10-09T09:06:00.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:00.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:00.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:00.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.120491+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:06:00.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.120512+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T09:06:00.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.629651+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e209: 8 total, 8 up, 5 in 2023-10-09T09:06:00.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.633448+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.720695+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.720827+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.721021+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T09:06:00.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:00 smithi094 bash[59677]: cluster 2023-10-09T09:05:59.897691+0000 mgr.y (mgr.25421) 855 : cluster 0 pgmap v893: 89 pgs: 3 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 9898/72 objects degraded (13747.222%); 0 B/s, 209 objects/s recovering 2023-10-09T09:06:01.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:01.057 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:06:01.057 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:06:01.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:01.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:01.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:01.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:01.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:01.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:01.298 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:06:01.301 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:06:01.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:01.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:01.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:01.525 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:01.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:01.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:01.652 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:06:01.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:01.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:01.765 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:01 smithi094 bash[59677]: audit 2023-10-09T09:06:01.276530+0000 mgr.y (mgr.25421) 856 : audit 0 from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:06:01.765 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:01 smithi094 bash[59677]: audit 2023-10-09T09:06:01.481823+0000 mon.a (mon.0) 1161 : audit 1 from='client.? 172.21.15.16:0/2751618388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:01.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:01.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:01.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:01 smithi016 bash[75179]: audit 2023-10-09T09:06:01.276530+0000 mgr.y (mgr.25421) 856 : audit 0 from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:06:01.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:01 smithi016 bash[75179]: audit 2023-10-09T09:06:01.481823+0000 mon.a (mon.0) 1161 : audit 1 from='client.? 172.21.15.16:0/2751618388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:01.895 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:01 smithi016 bash[77892]: audit 2023-10-09T09:06:01.276530+0000 mgr.y (mgr.25421) 856 : audit 0 from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:06:01.895 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:01 smithi016 bash[77892]: audit 2023-10-09T09:06:01.481823+0000 mon.a (mon.0) 1161 : audit 1 from='client.? 172.21.15.16:0/2751618388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:01.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:01.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:02.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:02.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:02.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:02.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:02.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:02.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:02.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:02.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:02.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:02.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:02.531 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:02.564 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:02.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:02.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[75179]: cluster 2023-10-09T09:06:01.634241+0000 mon.a (mon.0) 1162 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:06:02.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[75179]: cluster 2023-10-09T09:06:01.642276+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[75179]: audit 2023-10-09T09:06:01.645092+0000 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.16:0/2751618388' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:06:02.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[75179]: cluster 2023-10-09T09:06:01.645177+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e210: 8 total, 8 up, 5 in 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[75179]: cluster 2023-10-09T09:06:01.804827+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[75179]: cluster 2023-10-09T09:06:01.804883+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[75179]: cluster 2023-10-09T09:06:01.898423+0000 mgr.y (mgr.25421) 857 : cluster 0 pgmap v895: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 84 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 9505/72 objects degraded (13201.389%); 0 B/s, 287 objects/s recovering 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[77892]: cluster 2023-10-09T09:06:01.634241+0000 mon.a (mon.0) 1162 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[77892]: cluster 2023-10-09T09:06:01.642276+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[77892]: audit 2023-10-09T09:06:01.645092+0000 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.16:0/2751618388' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[77892]: cluster 2023-10-09T09:06:01.645177+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e210: 8 total, 8 up, 5 in 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[77892]: cluster 2023-10-09T09:06:01.804827+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[77892]: cluster 2023-10-09T09:06:01.804883+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:02 smithi016 bash[77892]: cluster 2023-10-09T09:06:01.898423+0000 mgr.y (mgr.25421) 857 : cluster 0 pgmap v895: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 84 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 9505/72 objects degraded (13201.389%); 0 B/s, 287 objects/s recovering 2023-10-09T09:06:02.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:02.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:02.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:02.803 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:02 smithi094 bash[59677]: cluster 2023-10-09T09:06:01.634241+0000 mon.a (mon.0) 1162 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:06:02.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:02 smithi094 bash[59677]: cluster 2023-10-09T09:06:01.642276+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:02 smithi094 bash[59677]: audit 2023-10-09T09:06:01.645092+0000 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.16:0/2751618388' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:06:02.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:02 smithi094 bash[59677]: cluster 2023-10-09T09:06:01.645177+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e210: 8 total, 8 up, 5 in 2023-10-09T09:06:02.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:02 smithi094 bash[59677]: cluster 2023-10-09T09:06:01.804827+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:02 smithi094 bash[59677]: cluster 2023-10-09T09:06:01.804883+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:02.804 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:02 smithi094 bash[59677]: cluster 2023-10-09T09:06:01.898423+0000 mgr.y (mgr.25421) 857 : cluster 0 pgmap v895: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 84 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 9505/72 objects degraded (13201.389%); 0 B/s, 287 objects/s recovering 2023-10-09T09:06:02.823 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:02.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:02.962 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:02.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:03.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:03.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:03.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:03.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:03.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:03.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:03.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:03.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:03.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:03.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:03.454 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:03.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:03.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:03.589 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:03.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:03.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:03.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:03.664 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_2' removed 2023-10-09T09:06:03.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:03.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:03.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:03.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:03.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:03.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:03 smithi094 bash[59677]: cluster 2023-10-09T09:06:01.805224+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:03.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:03 smithi094 bash[59677]: audit 2023-10-09T09:06:03.206385+0000 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.16:0/290254602' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:06:03.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:03.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:03.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:03.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:03 smithi016 bash[75179]: cluster 2023-10-09T09:06:01.805224+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:04.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:03 smithi016 bash[75179]: audit 2023-10-09T09:06:03.206385+0000 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.16:0/290254602' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:06:04.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:03 smithi016 bash[77892]: cluster 2023-10-09T09:06:01.805224+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T09:06:04.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:03 smithi016 bash[77892]: audit 2023-10-09T09:06:03.206385+0000 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.16:0/290254602' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:06:04.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:04.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:04.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:04.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:04.511 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:04.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:04.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:04.663 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: audit 2023-10-09T09:06:03.657152+0000 mon.a (mon.0) 1166 : audit 1 from='client.? 172.21.15.16:0/290254602' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:06:04.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: cluster 2023-10-09T09:06:03.657677+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e211: 8 total, 8 up, 5 in 2023-10-09T09:06:04.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: cluster 2023-10-09T09:06:03.755551+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: cluster 2023-10-09T09:06:03.758530+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: cluster 2023-10-09T09:06:03.761344+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: cluster 2023-10-09T09:06:03.898883+0000 mgr.y (mgr.25421) 858 : cluster 0 pgmap v897: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:04.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: audit 2023-10-09T09:06:04.235492+0000 mon.a (mon.0) 1168 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:04.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[77892]: cluster 2023-10-09T09:06:04.576168+0000 mon.a (mon.0) 1169 : cluster 3 Health check update: Degraded data redundancy: 9505/72 objects degraded (13201.389%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:06:04.673 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:04.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:04.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:04.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:04.803 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: audit 2023-10-09T09:06:03.657152+0000 mon.a (mon.0) 1166 : audit 1 from='client.? 172.21.15.16:0/290254602' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:06:04.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: cluster 2023-10-09T09:06:03.657677+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e211: 8 total, 8 up, 5 in 2023-10-09T09:06:04.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: cluster 2023-10-09T09:06:03.755551+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: cluster 2023-10-09T09:06:03.758530+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: cluster 2023-10-09T09:06:03.761344+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: cluster 2023-10-09T09:06:03.898883+0000 mgr.y (mgr.25421) 858 : cluster 0 pgmap v897: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:04.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: audit 2023-10-09T09:06:04.235492+0000 mon.a (mon.0) 1168 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:04.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:04 smithi016 bash[75179]: cluster 2023-10-09T09:06:04.576168+0000 mon.a (mon.0) 1169 : cluster 3 Health check update: Degraded data redundancy: 9505/72 objects degraded (13201.389%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:06:04.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:04.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:04.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:04.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: audit 2023-10-09T09:06:03.657152+0000 mon.a (mon.0) 1166 : audit 1 from='client.? 172.21.15.16:0/290254602' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:06:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: cluster 2023-10-09T09:06:03.657677+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e211: 8 total, 8 up, 5 in 2023-10-09T09:06:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: cluster 2023-10-09T09:06:03.755551+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: cluster 2023-10-09T09:06:03.758530+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: cluster 2023-10-09T09:06:03.761344+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T09:06:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: cluster 2023-10-09T09:06:03.898883+0000 mgr.y (mgr.25421) 858 : cluster 0 pgmap v897: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: audit 2023-10-09T09:06:04.235492+0000 mon.a (mon.0) 1168 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:04.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:04 smithi094 bash[59677]: cluster 2023-10-09T09:06:04.576168+0000 mon.a (mon.0) 1169 : cluster 3 Health check update: Degraded data redundancy: 9505/72 objects degraded (13201.389%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:06:05.038 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:05.070 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:05.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:05.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:05.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:05.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:05.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:05.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:05.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:05.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:05.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:05.467 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:05.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:05.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:05.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:05.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:05.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:05.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:05.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:05 smithi094 bash[59677]: cluster 2023-10-09T09:06:04.653098+0000 mon.a (mon.0) 1170 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9505/72 objects degraded (13201.389%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T09:06:05.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:05.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:05.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:05.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:05.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:05.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:05.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:05.991 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:05 smithi016 bash[77892]: cluster 2023-10-09T09:06:04.653098+0000 mon.a (mon.0) 1170 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9505/72 objects degraded (13201.389%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T09:06:05.992 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:05 smithi016 bash[75179]: cluster 2023-10-09T09:06:04.653098+0000 mon.a (mon.0) 1170 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9505/72 objects degraded (13201.389%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T09:06:05.992 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:06:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:06:05] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-09T09:06:06.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:06.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:06.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:06.146 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":893,"stamp":"2023-10-09T09:05:59.897483+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9898,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":176734,"num_write_kb":5649550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23456,"num_bytes_recovered":1025916238,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85727,"ondisk_log_size":85727,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":283,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":3592552,"kb_used_data":748672,"kb_used_omap":117,"kb_used_meta":2843722,"kb_avail":465133208,"statfs":{"total":479975178240,"available":476296404992,"internally_reserved":0,"allocated":766640128,"data_stored":763229001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":2911971986},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.001450"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026598+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:04:45.026598+0000","last_peered":"2023-10-09T09:04:45.026598+0000","last_clean":"2023-10-09T09:04:45.026598+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:04:45.026598+0000","last_undegraded":"2023-10-09T09:04:45.026598+0000","last_fullsized":"2023-10-09T09:04:45.026598+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":216,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.215755+0000","last_change":"2023-10-09T09:02:02.625626+0000","last_active":"2023-10-09T09:05:13.215755+0000","last_peered":"2023-10-09T09:05:13.215755+0000","last_clean":"2023-10-09T09:05:13.215755+0000","last_became_active":"2023-10-09T09:02:02.625312+0000","last_became_peered":"2023-10-09T09:02:02.625312+0000","last_unstale":"2023-10-09T09:05:13.215755+0000","last_undegraded":"2023-10-09T09:05:13.215755+0000","last_fullsized":"2023-10-09T09:05:13.215755+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:01.640943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.217771+0000","last_change":"2023-10-09T09:02:02.626274+0000","last_active":"2023-10-09T09:05:13.217771+0000","last_peered":"2023-10-09T09:05:13.217771+0000","last_clean":"2023-10-09T09:05:13.217771+0000","last_became_active":"2023-10-09T09:02:02.625973+0000","last_became_peered":"2023-10-09T09:02:02.625973+0000","last_unstale":"2023-10-09T09:05:13.217771+0000","last_undegraded":"2023-10-09T09:05:13.217771+0000","last_fullsized":"2023-10-09T09:05:13.217771+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:44.223794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":219,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.608021+0000","last_change":"2023-10-09T09:03:19.554441+0000","last_active":"2023-10-09T09:05:56.608021+0000","last_peered":"2023-10-09T09:05:56.608021+0000","last_clean":"2023-10-09T09:05:56.608021+0000","last_became_active":"2023-10-09T09:03:19.553967+0000","last_became_peered":"2023-10-09T09:03:19.553967+0000","last_unstale":"2023-10-09T09:05:56.608021+0000","last_undegraded":"2023-10-09T09:05:56.608021+0000","last_fullsized":"2023-10-09T09:05:56.608021+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:26.065993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":205,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.219282+0000","last_change":"2023-10-09T09:02:02.656821+0000","last_active":"2023-10-09T09:05:13.219282+0000","last_peered":"2023-10-09T09:05:13.219282+0000","last_clean":"2023-10-09T09:05:13.219282+0000","last_became_active":"2023-10-09T09:02:02.656523+0000","last_became_peered":"2023-10-09T09:02:02.656523+0000","last_unstale":"2023-10-09T09:05:13.219282+0000","last_undegraded":"2023-10-09T09:05:13.219282+0000","last_fullsized":"2023-10-09T09:05:13.219282+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:21.635518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216389+0000","last_change":"2023-10-09T09:03:19.582769+0000","last_active":"2023-10-09T09:05:13.216389+0000","last_peered":"2023-10-09T09:05:13.216389+0000","last_clean":"2023-10-09T09:05:13.216389+0000","last_became_active":"2023-10-09T09:03:19.577023+0000","last_became_peered":"2023-10-09T09:03:19.577023+0000","last_unstale":"2023-10-09T09:05:13.216389+0000","last_undegraded":"2023-10-09T09:05:13.216389+0000","last_fullsized":"2023-10-09T09:05:13.216389+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":181,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.605103+0000","last_change":"2023-10-09T09:01:36.280403+0000","last_active":"2023-10-09T09:05:56.605103+0000","last_peered":"2023-10-09T09:05:56.605103+0000","last_clean":"2023-10-09T09:05:56.605103+0000","last_became_active":"2023-10-09T09:01:36.280070+0000","last_became_peered":"2023-10-09T09:01:36.280070+0000","last_unstale":"2023-10-09T09:05:56.605103+0000","last_undegraded":"2023-10-09T09:05:56.605103+0000","last_fullsized":"2023-10-09T09:05:56.605103+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:33.796899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":217,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T09:05:55.600531+0000","last_change":"2023-10-09T09:02:02.630162+0000","last_active":"2023-10-09T09:05:55.600531+0000","last_peered":"2023-10-09T09:05:55.600531+0000","last_clean":"2023-10-09T09:05:55.600531+0000","last_became_active":"2023-10-09T09:02:02.629768+0000","last_became_peered":"2023-10-09T09:02:02.629768+0000","last_unstale":"2023-10-09T09:05:55.600531+0000","last_undegraded":"2023-10-09T09:05:55.600531+0000","last_fullsized":"2023-10-09T09:05:55.600531+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:36.859890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":23,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:45.813381+0000","last_change":"2023-10-09T09:03:19.562130+0000","last_active":"2023-10-09T09:05:45.813381+0000","last_peered":"2023-10-09T09:05:45.813381+0000","last_clean":"2023-10-09T09:05:45.813381+0000","last_became_active":"2023-10-09T09:03:19.557757+0000","last_became_peered":"2023-10-09T09:03:19.557757+0000","last_unstale":"2023-10-09T09:05:45.813381+0000","last_undegraded":"2023-10-09T09:05:45.813381+0000","last_fullsized":"2023-10-09T09:05:45.813381+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:46.546761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":178,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T09:05:55.601158+0000","last_change":"2023-10-09T09:01:36.279247+0000","last_active":"2023-10-09T09:05:55.601158+0000","last_peered":"2023-10-09T09:05:55.601158+0000","last_clean":"2023-10-09T09:05:55.601158+0000","last_became_active":"2023-10-09T09:01:36.278376+0000","last_became_peered":"2023-10-09T09:01:36.278376+0000","last_unstale":"2023-10-09T09:05:55.601158+0000","last_undegraded":"2023-10-09T09:05:55.601158+0000","last_fullsized":"2023-10-09T09:05:55.601158+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:19.166567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":162,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.215875+0000","last_change":"2023-10-09T08:59:53.544302+0000","last_active":"2023-10-09T09:05:13.215875+0000","last_peered":"2023-10-09T09:05:13.215875+0000","last_clean":"2023-10-09T09:05:13.215875+0000","last_became_active":"2023-10-09T08:59:53.531404+0000","last_became_peered":"2023-10-09T08:59:53.531404+0000","last_unstale":"2023-10-09T09:05:13.215875+0000","last_undegraded":"2023-10-09T09:05:13.215875+0000","last_fullsized":"2023-10-09T09:05:13.215875+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:48.670745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216048+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:05:13.216048+0000","last_peered":"2023-10-09T09:05:13.216048+0000","last_clean":"2023-10-09T09:05:13.216048+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:05:13.216048+0000","last_undegraded":"2023-10-09T09:05:13.216048+0000","last_fullsized":"2023-10-09T09:05:13.216048+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":86,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.027022+0000","last_change":"2023-10-09T09:00:31.867387+0000","last_active":"2023-10-09T09:04:45.027022+0000","last_peered":"2023-10-09T09:04:45.027022+0000","last_clean":"2023-10-09T09:04:45.027022+0000","last_became_active":"2023-10-09T09:00:31.859752+0000","last_became_peered":"2023-10-09T09:00:31.859752+0000","last_unstale":"2023-10-09T09:04:45.027022+0000","last_undegraded":"2023-10-09T09:04:45.027022+0000","last_fullsized":"2023-10-09T09:04:45.027022+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.027479+0000","last_change":"2023-10-09T09:02:02.630025+0000","last_active":"2023-10-09T09:04:45.027479+0000","last_peered":"2023-10-09T09:04:45.027479+0000","last_clean":"2023-10-09T09:04:45.027479+0000","last_became_active":"2023-10-09T09:02:02.629474+0000","last_became_peered":"2023-10-09T09:02:02.629474+0000","last_unstale":"2023-10-09T09:04:45.027479+0000","last_undegraded":"2023-10-09T09:04:45.027479+0000","last_fullsized":"2023-10-09T09:04:45.027479+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":78,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.027030+0000","last_change":"2023-10-09T09:00:31.959283+0000","last_active":"2023-10-09T09:04:45.027030+0000","last_peered":"2023-10-09T09:04:45.027030+0000","last_clean":"2023-10-09T09:04:45.027030+0000","last_became_active":"2023-10-09T09:00:31.861638+0000","last_became_peered":"2023-10-09T09:00:31.861638+0000","last_unstale":"2023-10-09T09:04:45.027030+0000","last_undegraded":"2023-10-09T09:04:45.027030+0000","last_fullsized":"2023-10-09T09:04:45.027030+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":198,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.215346+0000","last_change":"2023-10-09T09:02:02.636035+0000","last_active":"2023-10-09T09:05:13.215346+0000","last_peered":"2023-10-09T09:05:13.215346+0000","last_clean":"2023-10-09T09:05:13.215346+0000","last_became_active":"2023-10-09T09:02:02.635621+0000","last_became_peered":"2023-10-09T09:02:02.635621+0000","last_unstale":"2023-10-09T09:05:13.215346+0000","last_undegraded":"2023-10-09T09:05:13.215346+0000","last_fullsized":"2023-10-09T09:05:13.215346+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:46.294650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":208,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.607110+0000","last_change":"2023-10-09T09:03:19.564077+0000","last_active":"2023-10-09T09:05:56.607110+0000","last_peered":"2023-10-09T09:05:56.607110+0000","last_clean":"2023-10-09T09:05:56.607110+0000","last_became_active":"2023-10-09T09:03:19.563764+0000","last_became_peered":"2023-10-09T09:03:19.563764+0000","last_unstale":"2023-10-09T09:05:56.607110+0000","last_undegraded":"2023-10-09T09:05:56.607110+0000","last_fullsized":"2023-10-09T09:05:56.607110+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:38.043450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":162,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.215096+0000","last_change":"2023-10-09T09:01:36.279054+0000","last_active":"2023-10-09T09:05:13.215096+0000","last_peered":"2023-10-09T09:05:13.215096+0000","last_clean":"2023-10-09T09:05:13.215096+0000","last_became_active":"2023-10-09T09:01:36.278755+0000","last_became_peered":"2023-10-09T09:01:36.278755+0000","last_unstale":"2023-10-09T09:05:13.215096+0000","last_undegraded":"2023-10-09T09:05:13.215096+0000","last_fullsized":"2023-10-09T09:05:13.215096+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:09.089065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026770+0000","last_change":"2023-10-09T09:00:31.851267+0000","last_active":"2023-10-09T09:04:45.026770+0000","last_peered":"2023-10-09T09:04:45.026770+0000","last_clean":"2023-10-09T09:04:45.026770+0000","last_became_active":"2023-10-09T09:00:31.850918+0000","last_became_peered":"2023-10-09T09:00:31.850918+0000","last_unstale":"2023-10-09T09:04:45.026770+0000","last_undegraded":"2023-10-09T09:04:45.026770+0000","last_fullsized":"2023-10-09T09:04:45.026770+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":181,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.608147+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T09:05:56.608147+0000","last_peered":"2023-10-09T09:05:56.608147+0000","last_clean":"2023-10-09T09:05:56.608147+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T09:05:56.608147+0000","last_undegraded":"2023-10-09T09:05:56.608147+0000","last_fullsized":"2023-10-09T09:05:56.608147+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":83,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216403+0000","last_change":"2023-10-09T09:01:36.301852+0000","last_active":"2023-10-09T09:05:13.216403+0000","last_peered":"2023-10-09T09:05:13.216403+0000","last_clean":"2023-10-09T09:05:13.216403+0000","last_became_active":"2023-10-09T09:01:36.284416+0000","last_became_peered":"2023-10-09T09:01:36.284416+0000","last_unstale":"2023-10-09T09:05:13.216403+0000","last_undegraded":"2023-10-09T09:05:13.216403+0000","last_fullsized":"2023-10-09T09:05:13.216403+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:45.813392+0000","last_change":"2023-10-09T09:03:19.575211+0000","last_active":"2023-10-09T09:05:45.813392+0000","last_peered":"2023-10-09T09:05:45.813392+0000","last_clean":"2023-10-09T09:05:45.813392+0000","last_became_active":"2023-10-09T09:03:19.574829+0000","last_became_peered":"2023-10-09T09:03:19.574829+0000","last_unstale":"2023-10-09T09:05:45.813392+0000","last_undegraded":"2023-10-09T09:05:45.813392+0000","last_fullsized":"2023-10-09T09:05:45.813392+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:14.763790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"204'11038","reported_seq":11077,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:27.437211+0000","last_change":"2023-10-09T09:01:36.277698+0000","last_active":"2023-10-09T09:05:27.437211+0000","last_peered":"2023-10-09T09:05:27.437211+0000","last_clean":"2023-10-09T09:05:27.437211+0000","last_became_active":"2023-10-09T09:01:36.276985+0000","last_became_peered":"2023-10-09T09:01:36.276985+0000","last_unstale":"2023-10-09T09:05:27.437211+0000","last_undegraded":"2023-10-09T09:05:27.437211+0000","last_fullsized":"2023-10-09T09:05:27.437211+0000","mapping_epoch":176,"log_start":"192'5500","ondisk_log_start":"192'5500","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11038,"num_write_kb":353216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":87,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026940+0000","last_change":"2023-10-09T09:03:19.626156+0000","last_active":"2023-10-09T09:04:45.026940+0000","last_peered":"2023-10-09T09:04:45.026940+0000","last_clean":"2023-10-09T09:04:45.026940+0000","last_became_active":"2023-10-09T09:03:19.575061+0000","last_became_peered":"2023-10-09T09:03:19.575061+0000","last_unstale":"2023-10-09T09:04:45.026940+0000","last_undegraded":"2023-10-09T09:04:45.026940+0000","last_fullsized":"2023-10-09T09:04:45.026940+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216295+0000","last_change":"2023-10-09T09:03:19.582540+0000","last_active":"2023-10-09T09:05:13.216295+0000","last_peered":"2023-10-09T09:05:13.216295+0000","last_clean":"2023-10-09T09:05:13.216295+0000","last_became_active":"2023-10-09T09:03:19.577044+0000","last_became_peered":"2023-10-09T09:03:19.577044+0000","last_unstale":"2023-10-09T09:05:13.216295+0000","last_undegraded":"2023-10-09T09:05:13.216295+0000","last_fullsized":"2023-10-09T09:05:13.216295+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"204'10830","reported_seq":10871,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:27.498006+0000","last_change":"2023-10-09T09:01:16.814785+0000","last_active":"2023-10-09T09:05:27.498006+0000","last_peered":"2023-10-09T09:05:27.498006+0000","last_clean":"2023-10-09T09:05:27.498006+0000","last_became_active":"2023-10-09T09:01:16.814404+0000","last_became_peered":"2023-10-09T09:01:16.814404+0000","last_unstale":"2023-10-09T09:05:27.498006+0000","last_undegraded":"2023-10-09T09:05:27.498006+0000","last_fullsized":"2023-10-09T09:05:27.498006+0000","mapping_epoch":172,"log_start":"192'5600","ondisk_log_start":"192'5600","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5230,"log_dups_size":0,"ondisk_log_size":5230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10830,"num_write_kb":346560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":74,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.027411+0000","last_change":"2023-10-09T09:02:02.724809+0000","last_active":"2023-10-09T09:04:45.027411+0000","last_peered":"2023-10-09T09:04:45.027411+0000","last_clean":"2023-10-09T09:04:45.027411+0000","last_became_active":"2023-10-09T09:02:02.666202+0000","last_became_peered":"2023-10-09T09:02:02.666202+0000","last_unstale":"2023-10-09T09:04:45.027411+0000","last_undegraded":"2023-10-09T09:04:45.027411+0000","last_fullsized":"2023-10-09T09:04:45.027411+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026315+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:04:45.026315+0000","last_peered":"2023-10-09T09:04:45.026315+0000","last_clean":"2023-10-09T09:04:45.026315+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:04:45.026315+0000","last_undegraded":"2023-10-09T09:04:45.026315+0000","last_fullsized":"2023-10-09T09:04:45.026315+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"204'11098","reported_seq":11144,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.607854+0000","last_change":"2023-10-09T09:01:16.948996+0000","last_active":"2023-10-09T09:05:56.607854+0000","last_peered":"2023-10-09T09:05:56.607854+0000","last_clean":"2023-10-09T09:05:56.607854+0000","last_became_active":"2023-10-09T09:01:16.948583+0000","last_became_peered":"2023-10-09T09:01:16.948583+0000","last_unstale":"2023-10-09T09:05:56.607854+0000","last_undegraded":"2023-10-09T09:05:56.607854+0000","last_fullsized":"2023-10-09T09:05:56.607854+0000","mapping_epoch":172,"log_start":"192'5400","ondisk_log_start":"192'5400","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:21.355627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11098,"num_write_kb":355136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":114,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.217150+0000","last_change":"2023-10-09T09:01:36.305615+0000","last_active":"2023-10-09T09:05:13.217150+0000","last_peered":"2023-10-09T09:05:13.217150+0000","last_clean":"2023-10-09T09:05:13.217150+0000","last_became_active":"2023-10-09T09:01:36.292828+0000","last_became_peered":"2023-10-09T09:01:36.292828+0000","last_unstale":"2023-10-09T09:05:13.217150+0000","last_undegraded":"2023-10-09T09:05:13.217150+0000","last_fullsized":"2023-10-09T09:05:13.217150+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216251+0000","last_change":"2023-10-09T09:03:19.564369+0000","last_active":"2023-10-09T09:05:13.216251+0000","last_peered":"2023-10-09T09:05:13.216251+0000","last_clean":"2023-10-09T09:05:13.216251+0000","last_became_active":"2023-10-09T09:03:19.563989+0000","last_became_peered":"2023-10-09T09:03:19.563989+0000","last_unstale":"2023-10-09T09:05:13.216251+0000","last_undegraded":"2023-10-09T09:05:13.216251+0000","last_fullsized":"2023-10-09T09:05:13.216251+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"204'11274","reported_seq":14911,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T09:05:55.814507+0000","last_change":"2023-10-09T09:05:55.318868+0000","last_active":"2023-10-09T09:05:55.814507+0000","last_peered":"2023-10-09T09:05:55.814507+0000","last_clean":"2023-10-09T09:02:01.626664+0000","last_became_active":"2023-10-09T09:02:52.345464+0000","last_became_peered":"2023-10-09T09:02:52.345464+0000","last_unstale":"2023-10-09T09:05:55.814507+0000","last_undegraded":"2023-10-09T09:02:52.049405+0000","last_fullsized":"2023-10-09T09:02:51.903806+0000","mapping_epoch":190,"log_start":"195'6586","ondisk_log_start":"195'6586","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:38.670622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4994,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11274,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1789,"num_bytes_recovered":114753536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"3,6","objects":-4994}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.215479+0000","last_change":"2023-10-09T09:02:02.664617+0000","last_active":"2023-10-09T09:05:13.215479+0000","last_peered":"2023-10-09T09:05:13.215479+0000","last_clean":"2023-10-09T09:05:13.215479+0000","last_became_active":"2023-10-09T09:02:02.664305+0000","last_became_peered":"2023-10-09T09:02:02.664305+0000","last_unstale":"2023-10-09T09:05:13.215479+0000","last_undegraded":"2023-10-09T09:05:13.215479+0000","last_fullsized":"2023-10-09T09:05:13.215479+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:14.520177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":199,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.607353+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:05:56.607353+0000","last_peered":"2023-10-09T09:05:56.607353+0000","last_clean":"2023-10-09T09:05:56.607353+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:05:56.607353+0000","last_undegraded":"2023-10-09T09:05:56.607353+0000","last_fullsized":"2023-10-09T09:05:56.607353+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"204'10880","reported_seq":15837,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:27.511375+0000","last_change":"2023-10-09T09:04:01.588490+0000","last_active":"2023-10-09T09:05:27.511375+0000","last_peered":"2023-10-09T09:05:27.511375+0000","last_clean":"2023-10-09T09:05:27.511375+0000","last_became_active":"2023-10-09T09:04:01.574284+0000","last_became_peered":"2023-10-09T09:04:01.574284+0000","last_unstale":"2023-10-09T09:05:27.511375+0000","last_undegraded":"2023-10-09T09:05:27.511375+0000","last_fullsized":"2023-10-09T09:05:27.511375+0000","mapping_epoch":198,"log_start":"195'6232","ondisk_log_start":"195'6232","created":172,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4648,"log_dups_size":0,"ondisk_log_size":4648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:13.836976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10880,"num_write_kb":348160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2441,"num_bytes_recovered":155975680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":206,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:15.772155+0000","last_change":"2023-10-09T08:59:53.642095+0000","last_active":"2023-10-09T09:05:15.772155+0000","last_peered":"2023-10-09T09:05:15.772155+0000","last_clean":"2023-10-09T09:05:15.772155+0000","last_became_active":"2023-10-09T08:59:53.531588+0000","last_became_peered":"2023-10-09T08:59:53.531588+0000","last_unstale":"2023-10-09T09:05:15.772155+0000","last_undegraded":"2023-10-09T09:05:15.772155+0000","last_fullsized":"2023-10-09T09:05:15.772155+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.878230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026872+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:04:45.026872+0000","last_peered":"2023-10-09T09:04:45.026872+0000","last_clean":"2023-10-09T09:04:45.026872+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:04:45.026872+0000","last_undegraded":"2023-10-09T09:04:45.026872+0000","last_fullsized":"2023-10-09T09:04:45.026872+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"204'11052","reported_seq":19714,"reported_epoch":207,"state":"peering","last_fresh":"2023-10-09T09:05:57.658870+0000","last_change":"2023-10-09T09:05:57.659063+0000","last_active":"2023-10-09T09:05:57.658870+0000","last_peered":"2023-10-09T09:05:57.508726+0000","last_clean":"2023-10-09T09:05:57.508726+0000","last_became_active":"2023-10-09T09:03:20.662001+0000","last_became_peered":"2023-10-09T09:03:20.662001+0000","last_unstale":"2023-10-09T09:05:57.658870+0000","last_undegraded":"2023-10-09T09:05:57.658870+0000","last_fullsized":"2023-10-09T09:05:57.658870+0000","mapping_epoch":207,"log_start":"192'5300","ondisk_log_start":"192'5300","created":172,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:32.733979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11052,"num_write_kb":353664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4291,"num_bytes_recovered":281214976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,6],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":63,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216158+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:05:13.216158+0000","last_peered":"2023-10-09T09:05:13.216158+0000","last_clean":"2023-10-09T09:05:13.216158+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:05:13.216158+0000","last_undegraded":"2023-10-09T09:05:13.216158+0000","last_fullsized":"2023-10-09T09:05:13.216158+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.038541+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:04:45.038541+0000","last_peered":"2023-10-09T09:04:45.038541+0000","last_clean":"2023-10-09T09:04:45.038541+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:04:45.038541+0000","last_undegraded":"2023-10-09T09:04:45.038541+0000","last_fullsized":"2023-10-09T09:04:45.038541+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"204'11204","reported_seq":11263,"reported_epoch":204,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:05:27.516903+0000","last_change":"2023-10-09T09:03:20.697136+0000","last_active":"2023-10-09T09:05:27.516903+0000","last_peered":"2023-10-09T09:05:27.516903+0000","last_clean":"2023-10-09T09:03:19.287407+0000","last_became_active":"2023-10-09T09:03:20.665776+0000","last_became_peered":"2023-10-09T09:03:20.665776+0000","last_unstale":"2023-10-09T09:05:27.516903+0000","last_undegraded":"2023-10-09T09:05:27.516903+0000","last_fullsized":"2023-10-09T09:05:27.516903+0000","mapping_epoch":194,"log_start":"192'5700","ondisk_log_start":"192'5700","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5504,"log_dups_size":0,"ondisk_log_size":5504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11204,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","0","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026316+0000","last_change":"2023-10-09T09:03:19.563963+0000","last_active":"2023-10-09T09:04:45.026316+0000","last_peered":"2023-10-09T09:04:45.026316+0000","last_clean":"2023-10-09T09:04:45.026316+0000","last_became_active":"2023-10-09T09:03:19.563629+0000","last_became_peered":"2023-10-09T09:03:19.563629+0000","last_unstale":"2023-10-09T09:04:45.026316+0000","last_undegraded":"2023-10-09T09:04:45.026316+0000","last_fullsized":"2023-10-09T09:04:45.026316+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026687+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:04:45.026687+0000","last_peered":"2023-10-09T09:04:45.026687+0000","last_clean":"2023-10-09T09:04:45.026687+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:04:45.026687+0000","last_undegraded":"2023-10-09T09:04:45.026687+0000","last_fullsized":"2023-10-09T09:04:45.026687+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"204'10902","reported_seq":15822,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:27.483992+0000","last_change":"2023-10-09T09:02:38.910227+0000","last_active":"2023-10-09T09:05:27.483992+0000","last_peered":"2023-10-09T09:05:27.483992+0000","last_clean":"2023-10-09T09:05:27.483992+0000","last_became_active":"2023-10-09T09:02:38.903339+0000","last_became_peered":"2023-10-09T09:02:38.903339+0000","last_unstale":"2023-10-09T09:05:27.483992+0000","last_undegraded":"2023-10-09T09:05:27.483992+0000","last_fullsized":"2023-10-09T09:05:27.483992+0000","mapping_epoch":188,"log_start":"192'5700","ondisk_log_start":"192'5700","created":172,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5202,"log_dups_size":0,"ondisk_log_size":5202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10902,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2426,"num_bytes_recovered":158990336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:30.796667+0000","last_change":"2023-10-09T09:03:19.572389+0000","last_active":"2023-10-09T09:05:30.796667+0000","last_peered":"2023-10-09T09:05:30.796667+0000","last_clean":"2023-10-09T09:05:30.796667+0000","last_became_active":"2023-10-09T09:03:19.572091+0000","last_became_peered":"2023-10-09T09:03:19.572091+0000","last_unstale":"2023-10-09T09:05:30.796667+0000","last_undegraded":"2023-10-09T09:05:30.796667+0000","last_fullsized":"2023-10-09T09:05:30.796667+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:27.702718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":160,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.234211+0000","last_change":"2023-10-09T09:02:02.636996+0000","last_active":"2023-10-09T09:05:13.234211+0000","last_peered":"2023-10-09T09:05:13.234211+0000","last_clean":"2023-10-09T09:05:13.234211+0000","last_became_active":"2023-10-09T09:02:02.636702+0000","last_became_peered":"2023-10-09T09:02:02.636702+0000","last_unstale":"2023-10-09T09:05:13.234211+0000","last_undegraded":"2023-10-09T09:05:13.234211+0000","last_fullsized":"2023-10-09T09:05:13.234211+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:45.392645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"204'10846","reported_seq":10892,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.607787+0000","last_change":"2023-10-09T09:01:16.949929+0000","last_active":"2023-10-09T09:05:56.607787+0000","last_peered":"2023-10-09T09:05:56.607787+0000","last_clean":"2023-10-09T09:05:56.607787+0000","last_became_active":"2023-10-09T09:01:16.949317+0000","last_became_peered":"2023-10-09T09:01:16.949317+0000","last_unstale":"2023-10-09T09:05:56.607787+0000","last_undegraded":"2023-10-09T09:05:56.607787+0000","last_fullsized":"2023-10-09T09:05:56.607787+0000","mapping_epoch":172,"log_start":"192'5100","ondisk_log_start":"192'5100","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:27.017754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10846,"num_write_kb":347072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":149,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.605444+0000","last_change":"2023-10-09T09:03:19.556450+0000","last_active":"2023-10-09T09:05:56.605444+0000","last_peered":"2023-10-09T09:05:56.605444+0000","last_clean":"2023-10-09T09:05:56.605444+0000","last_became_active":"2023-10-09T09:03:19.556190+0000","last_became_peered":"2023-10-09T09:03:19.556190+0000","last_unstale":"2023-10-09T09:05:56.605444+0000","last_undegraded":"2023-10-09T09:05:56.605444+0000","last_fullsized":"2023-10-09T09:05:56.605444+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:23.704455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:50.815436+0000","last_change":"2023-10-09T09:01:36.285844+0000","last_active":"2023-10-09T09:05:50.815436+0000","last_peered":"2023-10-09T09:05:50.815436+0000","last_clean":"2023-10-09T09:05:50.815436+0000","last_became_active":"2023-10-09T09:01:36.285525+0000","last_became_peered":"2023-10-09T09:01:36.285525+0000","last_unstale":"2023-10-09T09:05:50.815436+0000","last_undegraded":"2023-10-09T09:05:50.815436+0000","last_fullsized":"2023-10-09T09:05:50.815436+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:05.624756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":183,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026754+0000","last_change":"2023-10-09T09:02:52.021749+0000","last_active":"2023-10-09T09:04:45.026754+0000","last_peered":"2023-10-09T09:04:45.026754+0000","last_clean":"2023-10-09T09:04:45.026754+0000","last_became_active":"2023-10-09T09:02:52.021416+0000","last_became_peered":"2023-10-09T09:02:52.021416+0000","last_unstale":"2023-10-09T09:04:45.026754+0000","last_undegraded":"2023-10-09T09:04:45.026754+0000","last_fullsized":"2023-10-09T09:04:45.026754+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"204'11012","reported_seq":11062,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:05:55.600511+0000","last_change":"2023-10-09T09:02:53.102143+0000","last_active":"2023-10-09T09:05:55.600511+0000","last_peered":"2023-10-09T09:05:55.600511+0000","last_clean":"2023-10-09T09:02:50.898920+0000","last_became_active":"2023-10-09T09:02:53.090937+0000","last_became_peered":"2023-10-09T09:02:53.090937+0000","last_unstale":"2023-10-09T09:05:55.600511+0000","last_undegraded":"2023-10-09T09:02:52.940746+0000","last_fullsized":"2023-10-09T09:02:52.912781+0000","mapping_epoch":191,"log_start":"195'6301","ondisk_log_start":"195'6301","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4711,"log_dups_size":0,"ondisk_log_size":4711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:08.317143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11012,"num_write_kb":352384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3","objects":-4904}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216049+0000","last_change":"2023-10-09T09:03:19.564803+0000","last_active":"2023-10-09T09:05:13.216049+0000","last_peered":"2023-10-09T09:05:13.216049+0000","last_clean":"2023-10-09T09:05:13.216049+0000","last_became_active":"2023-10-09T09:03:19.564520+0000","last_became_peered":"2023-10-09T09:03:19.564520+0000","last_unstale":"2023-10-09T09:05:13.216049+0000","last_undegraded":"2023-10-09T09:05:13.216049+0000","last_fullsized":"2023-10-09T09:05:13.216049+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.027157+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:04:45.027157+0000","last_peered":"2023-10-09T09:04:45.027157+0000","last_clean":"2023-10-09T09:04:45.027157+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:04:45.027157+0000","last_undegraded":"2023-10-09T09:04:45.027157+0000","last_fullsized":"2023-10-09T09:04:45.027157+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":173,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.610534+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:05:56.610534+0000","last_peered":"2023-10-09T09:05:56.610534+0000","last_clean":"2023-10-09T09:05:56.610534+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:05:56.610534+0000","last_undegraded":"2023-10-09T09:05:56.610534+0000","last_fullsized":"2023-10-09T09:05:56.610534+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"204'11082","reported_seq":16207,"reported_epoch":204,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:05:27.458219+0000","last_change":"2023-10-09T09:04:20.760516+0000","last_active":"2023-10-09T09:05:27.458219+0000","last_peered":"2023-10-09T09:05:27.458219+0000","last_clean":"2023-10-09T09:02:01.574862+0000","last_became_active":"2023-10-09T09:03:20.671903+0000","last_became_peered":"2023-10-09T09:03:20.671903+0000","last_unstale":"2023-10-09T09:05:27.458219+0000","last_undegraded":"2023-10-09T09:05:27.458219+0000","last_fullsized":"2023-10-09T09:05:27.458219+0000","mapping_epoch":194,"log_start":"192'5600","ondisk_log_start":"192'5600","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5482,"log_dups_size":0,"ondisk_log_size":5482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11082,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2531,"num_bytes_recovered":154402816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[4,2,0],"avail_no_missing":["4","0","1","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":232,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.609986+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:05:56.609986+0000","last_peered":"2023-10-09T09:05:56.609986+0000","last_clean":"2023-10-09T09:05:56.609986+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:05:56.609986+0000","last_undegraded":"2023-10-09T09:05:56.609986+0000","last_fullsized":"2023-10-09T09:05:56.609986+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216075+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:05:13.216075+0000","last_peered":"2023-10-09T09:05:13.216075+0000","last_clean":"2023-10-09T09:05:13.216075+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:05:13.216075+0000","last_undegraded":"2023-10-09T09:05:13.216075+0000","last_fullsized":"2023-10-09T09:05:13.216075+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":213,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.604974+0000","last_change":"2023-10-09T09:03:19.575546+0000","last_active":"2023-10-09T09:05:56.604974+0000","last_peered":"2023-10-09T09:05:56.604974+0000","last_clean":"2023-10-09T09:05:56.604974+0000","last_became_active":"2023-10-09T09:03:19.575075+0000","last_became_peered":"2023-10-09T09:03:19.575075+0000","last_unstale":"2023-10-09T09:05:56.604974+0000","last_undegraded":"2023-10-09T09:05:56.604974+0000","last_fullsized":"2023-10-09T09:05:56.604974+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:14.350886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"204'11062","reported_seq":11112,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:27.512076+0000","last_change":"2023-10-09T09:01:36.284533+0000","last_active":"2023-10-09T09:05:27.512076+0000","last_peered":"2023-10-09T09:05:27.512076+0000","last_clean":"2023-10-09T09:05:27.512076+0000","last_became_active":"2023-10-09T09:01:36.284124+0000","last_became_peered":"2023-10-09T09:01:36.284124+0000","last_unstale":"2023-10-09T09:05:27.512076+0000","last_undegraded":"2023-10-09T09:05:27.512076+0000","last_fullsized":"2023-10-09T09:05:27.512076+0000","mapping_epoch":176,"log_start":"192'5800","ondisk_log_start":"192'5800","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5262,"log_dups_size":0,"ondisk_log_size":5262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11062,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":138,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.609997+0000","last_change":"2023-10-09T09:01:36.274911+0000","last_active":"2023-10-09T09:05:56.609997+0000","last_peered":"2023-10-09T09:05:56.609997+0000","last_clean":"2023-10-09T09:05:56.609997+0000","last_became_active":"2023-10-09T09:01:36.274560+0000","last_became_peered":"2023-10-09T09:01:36.274560+0000","last_unstale":"2023-10-09T09:05:56.609997+0000","last_undegraded":"2023-10-09T09:05:56.609997+0000","last_fullsized":"2023-10-09T09:05:56.609997+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:55.305969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":446,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.604134+0000","last_change":"2023-10-09T09:02:05.534070+0000","last_active":"2023-10-09T09:05:56.604134+0000","last_peered":"2023-10-09T09:05:56.604134+0000","last_clean":"2023-10-09T09:05:56.604134+0000","last_became_active":"2023-10-09T09:02:05.533567+0000","last_became_peered":"2023-10-09T09:02:05.533567+0000","last_unstale":"2023-10-09T09:05:56.604134+0000","last_undegraded":"2023-10-09T09:05:56.604134+0000","last_fullsized":"2023-10-09T09:05:56.604134+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:59.568077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:25.794874+0000","last_change":"2023-10-09T09:03:19.562659+0000","last_active":"2023-10-09T09:05:25.794874+0000","last_peered":"2023-10-09T09:05:25.794874+0000","last_clean":"2023-10-09T09:05:25.794874+0000","last_became_active":"2023-10-09T09:03:19.562284+0000","last_became_peered":"2023-10-09T09:03:19.562284+0000","last_unstale":"2023-10-09T09:05:25.794874+0000","last_undegraded":"2023-10-09T09:05:25.794874+0000","last_fullsized":"2023-10-09T09:05:25.794874+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:26.537954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.217501+0000","last_change":"2023-10-09T09:00:31.859121+0000","last_active":"2023-10-09T09:05:13.217501+0000","last_peered":"2023-10-09T09:05:13.217501+0000","last_clean":"2023-10-09T09:05:13.217501+0000","last_became_active":"2023-10-09T09:00:31.858675+0000","last_became_peered":"2023-10-09T09:00:31.858675+0000","last_unstale":"2023-10-09T09:05:13.217501+0000","last_undegraded":"2023-10-09T09:05:13.217501+0000","last_fullsized":"2023-10-09T09:05:13.217501+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"204'11038","reported_seq":16053,"reported_epoch":207,"state":"peering","last_fresh":"2023-10-09T09:05:57.616130+0000","last_change":"2023-10-09T09:05:57.616337+0000","last_active":"2023-10-09T09:05:57.616130+0000","last_peered":"2023-10-09T09:05:57.451611+0000","last_clean":"2023-10-09T09:02:01.567140+0000","last_became_active":"2023-10-09T09:02:06.809579+0000","last_became_peered":"2023-10-09T09:02:06.809579+0000","last_unstale":"2023-10-09T09:05:57.616130+0000","last_undegraded":"2023-10-09T09:05:57.616130+0000","last_fullsized":"2023-10-09T09:05:57.616130+0000","mapping_epoch":207,"log_start":"192'5200","ondisk_log_start":"192'5200","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:28.772753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11038,"num_write_kb":353216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2474,"num_bytes_recovered":107085824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3,4],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":100,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.604320+0000","last_change":"2023-10-09T09:02:02.649488+0000","last_active":"2023-10-09T09:05:56.604320+0000","last_peered":"2023-10-09T09:05:56.604320+0000","last_clean":"2023-10-09T09:05:56.604320+0000","last_became_active":"2023-10-09T09:02:02.645550+0000","last_became_peered":"2023-10-09T09:02:02.645550+0000","last_unstale":"2023-10-09T09:05:56.604320+0000","last_undegraded":"2023-10-09T09:05:56.604320+0000","last_fullsized":"2023-10-09T09:05:56.604320+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:40.192731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":236,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.608287+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:05:56.608287+0000","last_peered":"2023-10-09T09:05:56.608287+0000","last_clean":"2023-10-09T09:05:56.608287+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:05:56.608287+0000","last_undegraded":"2023-10-09T09:05:56.608287+0000","last_fullsized":"2023-10-09T09:05:56.608287+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":197,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.607894+0000","last_change":"2023-10-09T09:02:02.657733+0000","last_active":"2023-10-09T09:05:56.607894+0000","last_peered":"2023-10-09T09:05:56.607894+0000","last_clean":"2023-10-09T09:05:56.607894+0000","last_became_active":"2023-10-09T09:02:02.656701+0000","last_became_peered":"2023-10-09T09:02:02.656701+0000","last_unstale":"2023-10-09T09:05:56.607894+0000","last_undegraded":"2023-10-09T09:05:56.607894+0000","last_fullsized":"2023-10-09T09:05:56.607894+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:39.552401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"204'11124","reported_seq":21057,"reported_epoch":205,"state":"peering","last_fresh":"2023-10-09T09:05:55.609065+0000","last_change":"2023-10-09T09:05:55.609341+0000","last_active":"2023-10-09T09:05:55.609065+0000","last_peered":"2023-10-09T09:05:55.317569+0000","last_clean":"2023-10-09T09:02:01.581848+0000","last_became_active":"2023-10-09T09:02:52.219467+0000","last_became_peered":"2023-10-09T09:02:52.219467+0000","last_unstale":"2023-10-09T09:05:55.609065+0000","last_undegraded":"2023-10-09T09:05:55.609065+0000","last_fullsized":"2023-10-09T09:05:55.609065+0000","mapping_epoch":205,"log_start":"195'6422","ondisk_log_start":"195'6422","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":4702,"log_dups_size":0,"ondisk_log_size":4702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:16.807354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11124,"num_write_kb":355968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4936,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,4,6],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":180,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.604497+0000","last_change":"2023-10-09T09:02:02.644204+0000","last_active":"2023-10-09T09:05:56.604497+0000","last_peered":"2023-10-09T09:05:56.604497+0000","last_clean":"2023-10-09T09:05:56.604497+0000","last_became_active":"2023-10-09T09:02:02.643713+0000","last_became_peered":"2023-10-09T09:02:02.643713+0000","last_unstale":"2023-10-09T09:05:56.604497+0000","last_undegraded":"2023-10-09T09:05:56.604497+0000","last_fullsized":"2023-10-09T09:05:56.604497+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:20.811353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":175,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.610463+0000","last_change":"2023-10-09T09:02:02.647803+0000","last_active":"2023-10-09T09:05:56.610463+0000","last_peered":"2023-10-09T09:05:56.610463+0000","last_clean":"2023-10-09T09:05:56.610463+0000","last_became_active":"2023-10-09T09:02:02.646911+0000","last_became_peered":"2023-10-09T09:02:02.646911+0000","last_unstale":"2023-10-09T09:05:56.610463+0000","last_undegraded":"2023-10-09T09:05:56.610463+0000","last_fullsized":"2023-10-09T09:05:56.610463+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:35.294072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":194,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.607691+0000","last_change":"2023-10-09T09:02:52.016015+0000","last_active":"2023-10-09T09:05:56.607691+0000","last_peered":"2023-10-09T09:05:56.607691+0000","last_clean":"2023-10-09T09:05:56.607691+0000","last_became_active":"2023-10-09T09:02:52.015543+0000","last_became_peered":"2023-10-09T09:02:52.015543+0000","last_unstale":"2023-10-09T09:05:56.607691+0000","last_undegraded":"2023-10-09T09:05:56.607691+0000","last_fullsized":"2023-10-09T09:05:56.607691+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.410980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"204'11020","reported_seq":11075,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.605552+0000","last_change":"2023-10-09T09:01:36.279450+0000","last_active":"2023-10-09T09:05:56.605552+0000","last_peered":"2023-10-09T09:05:56.605552+0000","last_clean":"2023-10-09T09:05:56.605552+0000","last_became_active":"2023-10-09T09:01:36.279123+0000","last_became_peered":"2023-10-09T09:01:36.279123+0000","last_unstale":"2023-10-09T09:05:56.605552+0000","last_undegraded":"2023-10-09T09:05:56.605552+0000","last_fullsized":"2023-10-09T09:05:56.605552+0000","mapping_epoch":176,"log_start":"192'5300","ondisk_log_start":"192'5300","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5720,"log_dups_size":0,"ondisk_log_size":5720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:21.843271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11020,"num_write_kb":352640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":205,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.234213+0000","last_change":"2023-10-09T09:01:36.292450+0000","last_active":"2023-10-09T09:05:13.234213+0000","last_peered":"2023-10-09T09:05:13.234213+0000","last_clean":"2023-10-09T09:05:13.234213+0000","last_became_active":"2023-10-09T09:01:36.273029+0000","last_became_peered":"2023-10-09T09:01:36.273029+0000","last_unstale":"2023-10-09T09:05:13.234213+0000","last_undegraded":"2023-10-09T09:05:13.234213+0000","last_fullsized":"2023-10-09T09:05:13.234213+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:26.065684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216709+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:05:13.216709+0000","last_peered":"2023-10-09T09:05:13.216709+0000","last_clean":"2023-10-09T09:05:13.216709+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:05:13.216709+0000","last_undegraded":"2023-10-09T09:05:13.216709+0000","last_fullsized":"2023-10-09T09:05:13.216709+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:20.791911+0000","last_change":"2023-10-09T09:03:19.572475+0000","last_active":"2023-10-09T09:05:20.791911+0000","last_peered":"2023-10-09T09:05:20.791911+0000","last_clean":"2023-10-09T09:05:20.791911+0000","last_became_active":"2023-10-09T09:03:19.572081+0000","last_became_peered":"2023-10-09T09:03:19.572081+0000","last_unstale":"2023-10-09T09:05:20.791911+0000","last_undegraded":"2023-10-09T09:05:20.791911+0000","last_fullsized":"2023-10-09T09:05:20.791911+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:04.771178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"204'11012","reported_seq":16056,"reported_epoch":206,"state":"active+recovery_wait","last_fresh":"2023-10-09T09:05:56.610840+0000","last_change":"2023-10-09T09:04:45.074782+0000","last_active":"2023-10-09T09:05:56.610840+0000","last_peered":"2023-10-09T09:05:56.610840+0000","last_clean":"2023-10-09T09:02:01.579768+0000","last_became_active":"2023-10-09T09:04:45.039242+0000","last_became_peered":"2023-10-09T09:04:45.039242+0000","last_unstale":"2023-10-09T09:05:56.610840+0000","last_undegraded":"2023-10-09T09:05:56.610840+0000","last_fullsized":"2023-10-09T09:05:56.610840+0000","mapping_epoch":202,"log_start":"192'5200","ondisk_log_start":"192'5200","created":172,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:15.789712+0000","last_clean_scrub_stamp":"2023-10-09T09:01:15.789712+0000","objects_scrubbed":0,"log_size":5812,"log_dups_size":0,"ondisk_log_size":5812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:39.547884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11012,"num_write_kb":352321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2490,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":["2","1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:13.216492+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:05:13.216492+0000","last_peered":"2023-10-09T09:05:13.216492+0000","last_clean":"2023-10-09T09:05:13.216492+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:05:13.216492+0000","last_undegraded":"2023-10-09T09:05:13.216492+0000","last_fullsized":"2023-10-09T09:05:13.216492+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":216,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.608012+0000","last_change":"2023-10-09T09:02:02.674657+0000","last_active":"2023-10-09T09:05:56.608012+0000","last_peered":"2023-10-09T09:05:56.608012+0000","last_clean":"2023-10-09T09:05:56.608012+0000","last_became_active":"2023-10-09T09:02:02.657392+0000","last_became_peered":"2023-10-09T09:02:02.657392+0000","last_unstale":"2023-10-09T09:05:56.608012+0000","last_undegraded":"2023-10-09T09:05:56.608012+0000","last_fullsized":"2023-10-09T09:05:56.608012+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:16.286583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026560+0000","last_change":"2023-10-09T09:01:36.280537+0000","last_active":"2023-10-09T09:04:45.026560+0000","last_peered":"2023-10-09T09:04:45.026560+0000","last_clean":"2023-10-09T09:04:45.026560+0000","last_became_active":"2023-10-09T09:01:36.280149+0000","last_became_peered":"2023-10-09T09:01:36.280149+0000","last_unstale":"2023-10-09T09:04:45.026560+0000","last_undegraded":"2023-10-09T09:04:45.026560+0000","last_fullsized":"2023-10-09T09:04:45.026560+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":215,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.607521+0000","last_change":"2023-10-09T09:03:19.554339+0000","last_active":"2023-10-09T09:05:56.607521+0000","last_peered":"2023-10-09T09:05:56.607521+0000","last_clean":"2023-10-09T09:05:56.607521+0000","last_became_active":"2023-10-09T09:03:19.553953+0000","last_became_peered":"2023-10-09T09:03:19.553953+0000","last_unstale":"2023-10-09T09:05:56.607521+0000","last_undegraded":"2023-10-09T09:05:56.607521+0000","last_fullsized":"2023-10-09T09:05:56.607521+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:50.601515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":37,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.610384+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:05:56.610384+0000","last_peered":"2023-10-09T09:05:56.610384+0000","last_clean":"2023-10-09T09:05:56.610384+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:05:56.610384+0000","last_undegraded":"2023-10-09T09:05:56.610384+0000","last_fullsized":"2023-10-09T09:05:56.610384+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":206,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026593+0000","last_change":"2023-10-09T09:02:52.022788+0000","last_active":"2023-10-09T09:04:45.026593+0000","last_peered":"2023-10-09T09:04:45.026593+0000","last_clean":"2023-10-09T09:04:45.026593+0000","last_became_active":"2023-10-09T09:02:52.016418+0000","last_became_peered":"2023-10-09T09:02:52.016418+0000","last_unstale":"2023-10-09T09:04:45.026593+0000","last_undegraded":"2023-10-09T09:04:45.026593+0000","last_fullsized":"2023-10-09T09:04:45.026593+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026874+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:04:45.026874+0000","last_peered":"2023-10-09T09:04:45.026874+0000","last_clean":"2023-10-09T09:04:45.026874+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:04:45.026874+0000","last_undegraded":"2023-10-09T09:04:45.026874+0000","last_fullsized":"2023-10-09T09:04:45.026874+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":191,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026633+0000","last_change":"2023-10-09T09:03:19.575218+0000","last_active":"2023-10-09T09:04:45.026633+0000","last_peered":"2023-10-09T09:04:45.026633+0000","last_clean":"2023-10-09T09:04:45.026633+0000","last_became_active":"2023-10-09T09:03:19.557578+0000","last_became_peered":"2023-10-09T09:03:19.557578+0000","last_unstale":"2023-10-09T09:04:45.026633+0000","last_undegraded":"2023-10-09T09:04:45.026633+0000","last_fullsized":"2023-10-09T09:04:45.026633+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":57,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026368+0000","last_change":"2023-10-09T08:59:53.522794+0000","last_active":"2023-10-09T09:04:45.026368+0000","last_peered":"2023-10-09T09:04:45.026368+0000","last_clean":"2023-10-09T09:04:45.026368+0000","last_became_active":"2023-10-09T08:59:53.522419+0000","last_became_peered":"2023-10-09T08:59:53.522419+0000","last_unstale":"2023-10-09T09:04:45.026368+0000","last_undegraded":"2023-10-09T09:04:45.026368+0000","last_fullsized":"2023-10-09T09:04:45.026368+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":206,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:20.793220+0000","last_change":"2023-10-09T09:00:31.845771+0000","last_active":"2023-10-09T09:05:20.793220+0000","last_peered":"2023-10-09T09:05:20.793220+0000","last_clean":"2023-10-09T09:05:20.793220+0000","last_became_active":"2023-10-09T09:00:31.844775+0000","last_became_peered":"2023-10-09T09:00:31.844775+0000","last_unstale":"2023-10-09T09:05:20.793220+0000","last_undegraded":"2023-10-09T09:05:20.793220+0000","last_fullsized":"2023-10-09T09:05:20.793220+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:33.452539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":23,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T09:05:45.813280+0000","last_change":"2023-10-09T09:03:19.561924+0000","last_active":"2023-10-09T09:05:45.813280+0000","last_peered":"2023-10-09T09:05:45.813280+0000","last_clean":"2023-10-09T09:05:45.813280+0000","last_became_active":"2023-10-09T09:03:19.557692+0000","last_became_peered":"2023-10-09T09:03:19.557692+0000","last_unstale":"2023-10-09T09:05:45.813280+0000","last_undegraded":"2023-10-09T09:05:45.813280+0000","last_fullsized":"2023-10-09T09:05:45.813280+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:39.940109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":267,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T09:05:56.605225+0000","last_change":"2023-10-09T09:02:52.019620+0000","last_active":"2023-10-09T09:05:56.605225+0000","last_peered":"2023-10-09T09:05:56.605225+0000","last_clean":"2023-10-09T09:05:56.605225+0000","last_became_active":"2023-10-09T09:02:52.014489+0000","last_became_peered":"2023-10-09T09:02:52.014489+0000","last_unstale":"2023-10-09T09:05:56.605225+0000","last_undegraded":"2023-10-09T09:05:56.605225+0000","last_fullsized":"2023-10-09T09:05:56.605225+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:39.550600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":84,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T09:04:45.026117+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:04:45.026117+0000","last_peered":"2023-10-09T09:04:45.026117+0000","last_clean":"2023-10-09T09:04:45.026117+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:04:45.026117+0000","last_undegraded":"2023-10-09T09:04:45.026117+0000","last_fullsized":"2023-10-09T09:04:45.026117+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":13518,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9898,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176474,"num_write_kb":5647105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23378,"num_bytes_recovered":1023541248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1743847424,"data_stored":1743847424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85533,"ondisk_log_size":85533,"up":48,"acting":46,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430625,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626858,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1052012,"kb_used_data":156836,"kb_used_omap":0,"kb_used_meta":895168,"kb_avail":92693140,"statfs":{"total":95995035648,"available":94917775360,"internally_reserved":0,"allocated":160600064,"data_stored":160111154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916652032},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.786,"15min":2.3410000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":3.6259999999999999,"5min":58.621000000000002,"15min":153.63800000000001},"last":0.999},{"interface":"front","average":{"1min":1.365,"5min":2.0289999999999999,"15min":2.3540000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.6539999999999999,"5min":58.590000000000003,"15min":153.72200000000001},"last":0.65800000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:05:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.8440000000000001,"15min":1.8620000000000001},"min":{"1min":0.33000000000000002,"5min":0.32000000000000001,"15min":0.28299999999999997},"max":{"1min":1.0369999999999999,"5min":109.453,"15min":109.453},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":2.073,"15min":1.9339999999999999},"min":{"1min":0.441,"5min":0.316,"15min":0.314},"max":{"1min":1.9219999999999999,"5min":111.47499999999999,"15min":111.47499999999999},"last":0.76900000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:05:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.0189999999999999,"15min":1.4179999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29799999999999999},"max":{"1min":5.6820000000000004,"5min":5.6820000000000004,"15min":8.3979999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.177,"15min":1.48},"min":{"1min":0.377,"5min":0.30199999999999999,"15min":0.28100000000000003},"max":{"1min":3.431,"5min":9.1549999999999994,"15min":12.194000000000001},"last":0.95399999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:05:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.23,"15min":2.0870000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.7130000000000001,"5min":6.992,"15min":57.636000000000003},"last":1.089},{"interface":"front","average":{"1min":1.056,"5min":1.599,"15min":1.9059999999999999},"min":{"1min":0.49099999999999999,"5min":0.46500000000000002,"15min":0.34699999999999998},"max":{"1min":3.871,"5min":9.4429999999999996,"15min":58.393000000000001},"last":0.91400000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.4159999999999999,"15min":1.5369999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.0600000000000001,"5min":32.280999999999999,"15min":32.280999999999999},"last":1.056},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.2669999999999999,"15min":1.1699999999999999},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.30599999999999999},"max":{"1min":1.446,"5min":34.689,"15min":34.689},"last":1.034}]},{"osd":5,"last update":"Mon Oct 9 09:05:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.0429999999999999,"15min":1.29},"min":{"1min":0.41399999999999998,"5min":0.40300000000000002,"15min":0.35499999999999998},"max":{"1min":3.5609999999999999,"5min":9.218,"15min":12.404999999999999},"last":1.133},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.90900000000000003,"15min":1.1279999999999999},"min":{"1min":0.50900000000000001,"5min":0.41599999999999998,"15min":0.33900000000000002},"max":{"1min":1.407,"5min":4.2610000000000001,"15min":8.8569999999999993},"last":1.165}]},{"osd":7,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.0529999999999999,"15min":1.4139999999999999},"min":{"1min":0.47599999999999998,"5min":0.434,"15min":0.40500000000000003},"max":{"1min":2.0289999999999999,"5min":9.2460000000000004,"15min":11.382},"last":1.204},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.93799999999999994,"15min":1.2350000000000001},"min":{"1min":0.46500000000000002,"5min":0.45400000000000001,"15min":0.377},"max":{"1min":3.4140000000000001,"5min":4.3259999999999996,"15min":9.4800000000000004},"last":1.121}]}]},{"osd":1,"up_from":0,"seq":356482285806,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449332,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429140,"kb_used_data":2764,"kb_used_omap":28,"kb_used_meta":426339,"kb_avail":93316012,"statfs":{"total":95995035648,"available":95555596288,"internally_reserved":0,"allocated":2830336,"data_stored":2026660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":436571984},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.1779999999999999,"15min":1.609},"min":{"1min":0.308,"5min":0.22,"15min":0.21199999999999999},"max":{"1min":3.0510000000000002,"5min":6.7649999999999997,"15min":37.380000000000003},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.073,"5min":1.133,"15min":1.651},"min":{"1min":0.33700000000000002,"5min":0.254,"15min":0.21199999999999999},"max":{"1min":5.7400000000000002,"5min":10.102,"15min":38.012999999999998},"last":0.69499999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":3.4039999999999999,"15min":2.4580000000000002},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":9.1869999999999994,"5min":71.322999999999993,"15min":71.322999999999993},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.4449999999999998,"5min":5.8140000000000001,"15min":3.3559999999999999},"min":{"1min":0.36799999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.7290000000000001,"5min":219.00700000000001,"15min":219.00700000000001},"last":0.443}]},{"osd":3,"last update":"Mon Oct 9 09:05:38 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":4.431,"15min":3.4660000000000002},"min":{"1min":0.437,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":7.2069999999999999,"5min":219.041,"15min":219.041},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.6479999999999999,"5min":2.0510000000000002,"15min":2.6389999999999998},"min":{"1min":0.40600000000000003,"5min":0.28899999999999998,"15min":0.25},"max":{"1min":9.8409999999999993,"5min":20.466000000000001,"15min":107.66500000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.5449999999999999,"15min":1.51},"min":{"1min":0.46999999999999997,"5min":0.40200000000000002,"15min":0.30199999999999999},"max":{"1min":7.0259999999999998,"5min":10.178000000000001,"15min":10.178000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.431,"15min":1.6830000000000001},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.375,"5min":10.526,"15min":11.654},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.522,"15min":1.423},"min":{"1min":0.42899999999999999,"5min":0.40500000000000003,"15min":0.34499999999999997},"max":{"1min":4.3630000000000004,"5min":9.3030000000000008,"15min":9.3689999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.754,"5min":1.5960000000000001,"15min":1.6140000000000001},"min":{"1min":0.497,"5min":0.46000000000000002,"15min":0.33700000000000002},"max":{"1min":8.5739999999999998,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.79600000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4039999999999999,"5min":2.411,"15min":1.869},"min":{"1min":0.39600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":4.3319999999999999,"5min":62.850999999999999,"15min":62.850999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.6359999999999999,"5min":2.4860000000000002,"15min":2.085},"min":{"1min":0.52200000000000002,"5min":0.434,"15min":0.36699999999999999},"max":{"1min":6.907,"5min":62.816000000000003,"15min":62.816000000000003},"last":0.75}]},{"osd":7,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.6000000000000001,"15min":1.6599999999999999},"min":{"1min":0.54800000000000004,"5min":0.495,"15min":0.39400000000000002},"max":{"1min":6.9550000000000001,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.579,"15min":1.859},"min":{"1min":0.40999999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.3300000000000001,"5min":14.359,"15min":14.359},"last":0.60999999999999999}]}]},{"osd":2,"up_from":88,"seq":377957122280,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":504964,"kb_used_data":2812,"kb_used_omap":81,"kb_used_meta":502062,"kb_avail":93240188,"statfs":{"total":95995035648,"available":95477952512,"internally_reserved":0,"allocated":2879488,"data_stored":2036304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":514111706},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":5.3529999999999998,"5min":3.694,"15min":3.3170000000000002},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.191},"max":{"1min":68.457999999999998,"5min":68.457999999999998,"15min":134.458},"last":0.23799999999999999},{"interface":"front","average":{"1min":5.5579999999999998,"5min":3.5609999999999999,"15min":2.9430000000000001},"min":{"1min":0.40899999999999997,"5min":0.26000000000000001,"15min":0.221},"max":{"1min":68.162999999999997,"5min":68.162999999999997,"15min":138.417},"last":0.38100000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":2.2749999999999999,"15min":2.2999999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.21299999999999999},"max":{"1min":3.379,"5min":57.939999999999998,"15min":57.939999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.385,"5min":1.917,"15min":2.0880000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5519999999999996,"5min":56.667999999999999,"15min":57.122999999999998},"last":0.41499999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:05:53 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":4.2939999999999996,"15min":2.6850000000000001},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":6.665,"5min":112.47499999999999,"15min":112.47499999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.0049999999999999,"5min":4.718,"15min":3.048},"min":{"1min":0.39000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":17.274000000000001,"5min":112.929,"15min":112.929},"last":0.443}]},{"osd":4,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9529999999999998,"5min":1.9350000000000001,"15min":1.966},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":17.408999999999999,"5min":17.408999999999999,"15min":17.408999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.784,"5min":1.764,"15min":1.845},"min":{"1min":0.45000000000000001,"5min":0.38800000000000001,"15min":0.314},"max":{"1min":6.2160000000000002,"5min":9.8439999999999994,"15min":14.071999999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.3740000000000001,"15min":1.472},"min":{"1min":0.46500000000000002,"5min":0.41499999999999998,"15min":0.33900000000000002},"max":{"1min":5.5899999999999999,"5min":11.138,"15min":12.411},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.925,"15min":1.96},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":17.375,"5min":17.375,"15min":17.375},"last":0.52400000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.7490000000000001,"15min":1.8660000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.33900000000000002},"max":{"1min":5.327,"5min":9.9139999999999997,"15min":10.456},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.5649999999999999,"15min":1.5960000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.39300000000000002},"max":{"1min":5.6230000000000002,"5min":11.869999999999999,"15min":12.381},"last":0.68500000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":5.0410000000000004,"5min":2.4430000000000001,"15min":2.0779999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.38800000000000001},"max":{"1min":67.491,"5min":67.491,"15min":67.491},"last":0.498},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.534,"15min":1.542},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.38500000000000001},"max":{"1min":5.6070000000000002,"5min":11.933,"15min":12.438000000000001},"last":0.748}]}]},{"osd":3,"up_from":93,"seq":399431958754,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428532,"kb_used_data":2156,"kb_used_omap":7,"kb_used_meta":426360,"kb_avail":93316620,"statfs":{"total":95995035648,"available":95556218880,"internally_reserved":0,"allocated":2207744,"data_stored":1426064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":436593256},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":13.754,"5min":3.8029999999999999,"15min":2.4329999999999998},"min":{"1min":0.27300000000000002,"5min":0.19500000000000001,"15min":0.19400000000000001},"max":{"1min":233.22900000000001,"5min":233.22900000000001,"15min":233.22900000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":14.622999999999999,"5min":4.1299999999999999,"15min":2.5659999999999998},"min":{"1min":0.29599999999999999,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":237.447,"5min":237.447,"15min":237.447},"last":0.39600000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":4.7190000000000003,"15min":2.8039999999999998},"min":{"1min":0.32100000000000001,"5min":0.31,"15min":0.22500000000000001},"max":{"1min":4.4279999999999999,"5min":326.75700000000001,"15min":326.75700000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.498,"5min":4.4400000000000004,"15min":2.6880000000000002},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":0.95699999999999996,"5min":327.024,"15min":327.024},"last":0.68000000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.4279999999999999,"15min":2.6070000000000002},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.255},"max":{"1min":5.7709999999999999,"5min":43.683999999999997,"15min":75.641999999999996},"last":1.071},{"interface":"front","average":{"1min":1.2869999999999999,"5min":2.0150000000000001,"15min":2.4060000000000001},"min":{"1min":0.40100000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2229999999999999,"5min":45.027999999999999,"15min":75.207999999999998},"last":0.79100000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":1.1850000000000001,"15min":1.474},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.03,"5min":8.9879999999999995,"15min":12.877000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.587,"5min":1.5760000000000001,"15min":1.7290000000000001},"min":{"1min":0.42199999999999999,"5min":0.36399999999999999,"15min":0.33500000000000002},"max":{"1min":7.4269999999999996,"5min":9.4649999999999999,"15min":10.486000000000001},"last":0.77100000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.4259999999999999,"15min":1.6240000000000001},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":7.4580000000000002,"5min":9.4990000000000006,"15min":10.577},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.5780000000000001,"15min":1.736},"min":{"1min":0.45900000000000002,"5min":0.44900000000000001,"15min":0.34300000000000003},"max":{"1min":4.5309999999999997,"5min":7.5590000000000002,"15min":12.204000000000001},"last":1.228}]},{"osd":6,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":1.657,"15min":1.8160000000000001},"min":{"1min":0.46100000000000002,"5min":0.308,"15min":0.308},"max":{"1min":7.5300000000000002,"5min":9.1370000000000005,"15min":10.537000000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.891,"15min":1.9079999999999999},"min":{"1min":0.49299999999999999,"5min":0.40699999999999997,"15min":0.35099999999999998},"max":{"1min":4.4710000000000001,"5min":9.5090000000000003,"15min":12.275},"last":1.1759999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.663,"15min":1.7589999999999999},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.41299999999999998},"max":{"1min":4.5490000000000004,"5min":7.5899999999999999,"15min":12.244},"last":0.876},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.3700000000000001,"15min":1.6439999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.39000000000000001},"max":{"1min":3.714,"5min":9.1560000000000006,"15min":13.339},"last":1.1160000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019324,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1177904,"kb_used_data":584104,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":92567248,"statfs":{"total":95995035648,"available":94788861952,"internally_reserved":0,"allocated":598122496,"data_stored":597628819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:05:22 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":2.9710000000000001,"15min":2.3180000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.5720000000000001,"5min":141.07400000000001,"15min":141.07400000000001},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.9449999999999998,"15min":2.3500000000000001},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":5.5510000000000002,"5min":141.15600000000001,"15min":141.15600000000001},"last":0.79100000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:05:48 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.72999999999999998,"15min":1.3740000000000001},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":1.571,"5min":2.8199999999999998,"15min":8.9169999999999998},"last":0.877},{"interface":"front","average":{"1min":0.82899999999999996,"5min":1.0209999999999999,"15min":1.464},"min":{"1min":0.318,"5min":0.318,"15min":0.29199999999999998},"max":{"1min":3.2759999999999998,"5min":7.6879999999999997,"15min":11.712999999999999},"last":1.0349999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:05:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":2.988,"15min":2.2959999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30499999999999999},"max":{"1min":3.9990000000000001,"5min":142.816,"15min":142.816},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.381,"5min":2.6469999999999998,"15min":1.9430000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.6100000000000003,"5min":143.81800000000001,"15min":143.81800000000001},"last":1.165}]},{"osd":3,"last update":"Mon Oct 9 09:05:48 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":2.71,"15min":2.1459999999999999},"min":{"1min":0.41099999999999998,"5min":0.31900000000000001,"15min":0.29499999999999998},"max":{"1min":6.1159999999999997,"5min":143.13399999999999,"15min":143.13399999999999},"last":1.083},{"interface":"front","average":{"1min":1.472,"5min":3.2519999999999998,"15min":2.177},"min":{"1min":0.46400000000000002,"5min":0.41299999999999998,"15min":0.35399999999999998},"max":{"1min":6.6230000000000002,"5min":152.37299999999999,"15min":152.37299999999999},"last":1.0129999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:05:07 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.0429999999999999,"15min":1.226},"min":{"1min":0.35099999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.4960000000000004,"5min":7.7439999999999998,"15min":11.762},"last":1.548},{"interface":"front","average":{"1min":1.3029999999999999,"5min":0.94799999999999995,"15min":1.2929999999999999},"min":{"1min":0.42199999999999999,"5min":0.37,"15min":0.28699999999999998},"max":{"1min":4.1379999999999999,"5min":4.1379999999999999,"15min":12.85},"last":1.2849999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.91000000000000003,"15min":1.3939999999999999},"min":{"1min":0.40600000000000003,"5min":0.35699999999999998,"15min":0.32100000000000001},"max":{"1min":2.02,"5min":5.1020000000000003,"15min":10.657999999999999},"last":1.5109999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.137,"15min":1.448},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.4039999999999999,"5min":7.6470000000000002,"15min":14.755000000000001},"last":1.4319999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:05:22 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.96599999999999997,"15min":1.413},"min":{"1min":0.46300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.9039999999999999,"5min":3.9039999999999999,"15min":13.933999999999999},"last":1.363},{"interface":"front","average":{"1min":1.105,"5min":0.95699999999999996,"15min":1.5700000000000001},"min":{"1min":0.505,"5min":0.36199999999999999,"15min":0.33200000000000002},"max":{"1min":4.984,"5min":5.1520000000000001,"15min":13.911},"last":1.389}]}]},{"osd":5,"up_from":0,"seq":154618823092,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":989462528,"data_stored":989462528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":596246528,"data_stored":596246528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":158138368,"data_stored":158138368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:06:06.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:06:06.185 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:06:06.185 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:06:06.186 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T09:06:06.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T09:06:06.246 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:06.280 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:06.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:06.419 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:06.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:06.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:06.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:06.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:06.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:06.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:06.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:06.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:06:06.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:06.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:06 smithi016 bash[75179]: cluster 2023-10-09T09:06:05.899894+0000 mgr.y (mgr.25421) 859 : cluster 0 pgmap v898: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:06.806 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:06 smithi016 bash[77892]: cluster 2023-10-09T09:06:05.899894+0000 mgr.y (mgr.25421) 859 : cluster 0 pgmap v898: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:06.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:06.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:06.935 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:06 smithi094 bash[59677]: cluster 2023-10-09T09:06:05.899894+0000 mgr.y (mgr.25421) 859 : cluster 0 pgmap v898: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:06.954 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:06.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:07.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:07.091 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:07.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:07.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:07.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:07.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:07.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:07.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:07.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:07.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:07.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:07.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:07.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:07.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:07.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:07.633 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:07.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:07.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:07.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:07.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:07.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:07.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:07.907 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:07.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:07.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:08.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:08.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:08.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:08.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:06:08.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:08.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:08.312 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:08.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:08.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:08.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:08.552 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:08.584 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:08.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:08.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:08.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:08.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:08.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:08.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:08.993 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:08.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:08.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:09.097 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:09.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:09.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:09.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:09.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:06:09.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:09.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:08 smithi094 bash[59677]: cluster 2023-10-09T09:06:07.900706+0000 mgr.y (mgr.25421) 860 : cluster 0 pgmap v899: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:09.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:08 smithi016 bash[77892]: cluster 2023-10-09T09:06:07.900706+0000 mgr.y (mgr.25421) 860 : cluster 0 pgmap v899: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:09.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:08 smithi016 bash[75179]: cluster 2023-10-09T09:06:07.900706+0000 mgr.y (mgr.25421) 860 : cluster 0 pgmap v899: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:09.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:09.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:09.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:09.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:09.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:09.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:09.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:09.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:09.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:09.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:09.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:09.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:09.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:09.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:09.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:09.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:09.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:09.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:09.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:10.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:10.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:10.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:10.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:10.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:10.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:10.325 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:10.357 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:10.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:10.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:10.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:10.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:10.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:10.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:10.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:10.632 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:10.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:10.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:10.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:10.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:10.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:10.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:10.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:11.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:11.019 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:11.020 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:11.020 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:11.025 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:10 smithi016 bash[75179]: cluster 2023-10-09T09:06:09.901396+0000 mgr.y (mgr.25421) 861 : cluster 0 pgmap v900: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:11.025 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:10 smithi016 bash[75179]: audit 2023-10-09T09:06:09.902709+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:11.026 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:10 smithi016 bash[77892]: cluster 2023-10-09T09:06:09.901396+0000 mgr.y (mgr.25421) 861 : cluster 0 pgmap v900: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:11.026 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:10 smithi016 bash[77892]: audit 2023-10-09T09:06:09.902709+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:11.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:11.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:11.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:11.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:11.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:10 smithi094 bash[59677]: cluster 2023-10-09T09:06:09.901396+0000 mgr.y (mgr.25421) 861 : cluster 0 pgmap v900: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:11.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:10 smithi094 bash[59677]: audit 2023-10-09T09:06:09.902709+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:11.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:11.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:11.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:11.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:11.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:11.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:11.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:11.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:11.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:11.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:11.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:11.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:11.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:11.933 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:11.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:11.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:11.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:12.034 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:12.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:12.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:12.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:12.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:12.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:12.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:12.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:12.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:12.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:12.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:12.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:12.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:12.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:12.620 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:12.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:12.724 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:12.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:12.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:12.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:12.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:12.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:12.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:12.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:12 smithi094 bash[59677]: cluster 2023-10-09T09:06:11.901971+0000 mgr.y (mgr.25421) 862 : cluster 0 pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:12.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:12 smithi016 bash[77892]: cluster 2023-10-09T09:06:11.901971+0000 mgr.y (mgr.25421) 862 : cluster 0 pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:12.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:12 smithi016 bash[75179]: cluster 2023-10-09T09:06:11.901971+0000 mgr.y (mgr.25421) 862 : cluster 0 pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:13.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:13.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:13.048 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:13.117 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:13.150 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:13.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:13.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:13.389 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:13.427 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:13.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:13.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:13.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:13.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:13.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:13.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:13.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:13.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:13.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:13.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:14.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:14.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:14.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:14.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:14.196 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:14.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:14.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:14.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:14.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:14.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:14.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:14.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:14.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:14.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:14.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:14.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:14.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:14.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:14.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:14.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:14.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:14.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:14.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:14.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:14.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:14.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:14.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:15.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:15.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:15.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:15.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:15.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:15.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:15.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:15.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:15.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:15.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:15.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:15.665 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:15.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:15 smithi016 bash[75179]: cluster 2023-10-09T09:06:13.902798+0000 mgr.y (mgr.25421) 863 : cluster 0 pgmap v902: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:15.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:15 smithi016 bash[77892]: cluster 2023-10-09T09:06:13.902798+0000 mgr.y (mgr.25421) 863 : cluster 0 pgmap v902: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:15.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:15.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:15.898 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:15.911 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:15 smithi094 bash[59677]: cluster 2023-10-09T09:06:13.902798+0000 mgr.y (mgr.25421) 863 : cluster 0 pgmap v902: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:15.930 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:15.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:06:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:06:15] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-09T09:06:16.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:16.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:16.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:16.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:16.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:16.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:16.304 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:16.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:16.437 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:16.467 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:16.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:16.602 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:16.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:16.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:16.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:16.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:16.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:16.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:16.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:16.964 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:16.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:16.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:17.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:17.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:17.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:17.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:17.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:17.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:17.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:17.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:17.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:17.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:17.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:17.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:17.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:17.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:17.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:17.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:17 smithi016 bash[75179]: cluster 2023-10-09T09:06:15.903689+0000 mgr.y (mgr.25421) 864 : cluster 0 pgmap v903: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:17.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:17 smithi016 bash[77892]: cluster 2023-10-09T09:06:15.903689+0000 mgr.y (mgr.25421) 864 : cluster 0 pgmap v903: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:17.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:17.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:17.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:17 smithi094 bash[59677]: cluster 2023-10-09T09:06:15.903689+0000 mgr.y (mgr.25421) 864 : cluster 0 pgmap v903: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:17.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:18.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:18.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:18.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:18.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:18.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:18.237 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:18.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:18.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:18.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:18.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:18.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:18.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:18.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:18.626 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:18.658 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:18.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:18.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:18.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:18.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:18.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:19.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:19.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:19.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:19.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:19.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:19.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:19.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:19.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:19.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:19.443 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:19.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:19.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:19.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:19.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:19.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:19.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:19.817 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:19 smithi016 bash[75179]: cluster 2023-10-09T09:06:17.904471+0000 mgr.y (mgr.25421) 865 : cluster 0 pgmap v904: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:19.817 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:19 smithi016 bash[75179]: audit 2023-10-09T09:06:19.240149+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:19.817 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:19 smithi016 bash[75179]: audit 2023-10-09T09:06:19.241178+0000 mon.a (mon.0) 1173 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:19.817 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:19 smithi016 bash[77892]: cluster 2023-10-09T09:06:17.904471+0000 mgr.y (mgr.25421) 865 : cluster 0 pgmap v904: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:19.817 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:19 smithi016 bash[77892]: audit 2023-10-09T09:06:19.240149+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:19.817 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:19 smithi016 bash[77892]: audit 2023-10-09T09:06:19.241178+0000 mon.a (mon.0) 1173 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:19.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:19.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:19.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:19.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:19.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:19 smithi094 bash[59677]: cluster 2023-10-09T09:06:17.904471+0000 mgr.y (mgr.25421) 865 : cluster 0 pgmap v904: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:19.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:19 smithi094 bash[59677]: audit 2023-10-09T09:06:19.240149+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:19.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:19 smithi094 bash[59677]: audit 2023-10-09T09:06:19.241178+0000 mon.a (mon.0) 1173 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:20.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:20.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:20.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:20.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:20.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:20.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:20.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:20.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:20.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:20.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:20.445 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:20.478 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:20.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:20.582 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:20.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:20.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:20.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:20.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:20 smithi094 bash[59677]: cluster 2023-10-09T09:06:19.905149+0000 mgr.y (mgr.25421) 866 : cluster 0 pgmap v905: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:20.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:20.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:20.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:20.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:20.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:20.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:20.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:20.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:20 smithi016 bash[75179]: cluster 2023-10-09T09:06:19.905149+0000 mgr.y (mgr.25421) 866 : cluster 0 pgmap v905: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:21.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:20 smithi016 bash[77892]: cluster 2023-10-09T09:06:19.905149+0000 mgr.y (mgr.25421) 866 : cluster 0 pgmap v905: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:21.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:21.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:21.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:21.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:21.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:21.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:21.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:21.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:21.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:21.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:21.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:21.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:21.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:21.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:21.827 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:21.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:21.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:21.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:21.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:22.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:22.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:22.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:22.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:22.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:22.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:22.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:22.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:22.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:22.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:22.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:22.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:22.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:22.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:22.586 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:22.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:22.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:22.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:22.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:22.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:22.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:22.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:22.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:22.892 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:22.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:22.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:22.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:23.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:23.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:23.142 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:22 smithi094 bash[59677]: cluster 2023-10-09T09:06:21.906486+0000 mgr.y (mgr.25421) 867 : cluster 0 pgmap v906: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:23.161 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:23.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:23.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:22 smithi016 bash[75179]: cluster 2023-10-09T09:06:21.906486+0000 mgr.y (mgr.25421) 867 : cluster 0 pgmap v906: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:23.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:22 smithi016 bash[77892]: cluster 2023-10-09T09:06:21.906486+0000 mgr.y (mgr.25421) 867 : cluster 0 pgmap v906: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:23.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:23.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:23.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:23.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:23.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:23.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:23.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:23.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:23.677 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:23.781 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:23.813 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:23.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:23.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:23.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:23.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:24.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:24.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:24.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:24.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:24.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:24.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:24.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:24.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:24.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:24.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:24.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:24.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:24.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:24.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:24.612 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:24.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:24.716 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:24.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:24.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:24.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:24.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:24.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:24.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:24.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:25.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:25.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:25.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:25.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:25.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:25.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:25.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:25.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:25.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:25.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:25.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:25.502 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:06:25.502 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:06:25.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:25.548 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:25.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:25.663 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:25 smithi016 bash[77892]: cluster 2023-10-09T09:06:23.907012+0000 mgr.y (mgr.25421) 868 : cluster 0 pgmap v907: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:25.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:25 smithi016 bash[75179]: cluster 2023-10-09T09:06:23.907012+0000 mgr.y (mgr.25421) 868 : cluster 0 pgmap v907: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:25.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:25.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:25.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:25.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:25.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:25.915 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:06:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:06:25] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-09T09:06:25.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:25.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:25.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:25.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:25 smithi094 bash[59677]: cluster 2023-10-09T09:06:23.907012+0000 mgr.y (mgr.25421) 868 : cluster 0 pgmap v907: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:26.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:26.063 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:26.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:26.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:26.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:26.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:26.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:26.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:26.430 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:26.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:26.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:26.589 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:06:26.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:26.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:26.603 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:06:26.608 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_3' created 2023-10-09T09:06:26.614 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:06:26.636 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:06:26.667 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:06:26.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:26.697 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:06:26.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:26 smithi094 bash[59677]: audit 2023-10-09T09:06:25.727413+0000 mon.a (mon.0) 1174 : audit 1 from='client.? 172.21.15.16:0/1950201946' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T09:06:26.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:26 smithi094 bash[59677]: audit 2023-10-09T09:06:26.519684+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.16:0/2373315573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:06:26.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:26 smithi094 bash[59677]: audit 2023-10-09T09:06:26.520235+0000 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:06:26.710 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:26 smithi094 bash[59677]: audit 2023-10-09T09:06:26.563060+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.16:0/1464903676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:06:26.721 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:06:26.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:26.742 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:06:26.763 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:06:26.765 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.765 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.765 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.766 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.766 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.766 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.766 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.766 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:06:26.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:26.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:26.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:26.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[75179]: audit 2023-10-09T09:06:25.727413+0000 mon.a (mon.0) 1174 : audit 1 from='client.? 172.21.15.16:0/1950201946' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T09:06:26.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[75179]: audit 2023-10-09T09:06:26.519684+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.16:0/2373315573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:06:26.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[75179]: audit 2023-10-09T09:06:26.520235+0000 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:06:26.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[75179]: audit 2023-10-09T09:06:26.563060+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.16:0/1464903676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:06:26.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[77892]: audit 2023-10-09T09:06:25.727413+0000 mon.a (mon.0) 1174 : audit 1 from='client.? 172.21.15.16:0/1950201946' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T09:06:26.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[77892]: audit 2023-10-09T09:06:26.519684+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.16:0/2373315573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:06:26.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[77892]: audit 2023-10-09T09:06:26.520235+0000 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:06:26.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:26 smithi016 bash[77892]: audit 2023-10-09T09:06:26.563060+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.16:0/1464903676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:06:26.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:26.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:26.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:26.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:27.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:27.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:27.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:27.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:27.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:27.239 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:27.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:27.325 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:06:27.326 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:06:27.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:27.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:27.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:27.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:27.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:27.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:27.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:27.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:27.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:27.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:27.740 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:27.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[75179]: cluster 2023-10-09T09:06:25.907459+0000 mgr.y (mgr.25421) 869 : cluster 0 pgmap v908: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:27.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[75179]: cluster 2023-10-09T09:06:26.587794+0000 mon.a (mon.0) 1176 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:06:27.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[75179]: cluster 2023-10-09T09:06:26.594840+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T09:06:27.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[75179]: audit 2023-10-09T09:06:26.598038+0000 mon.a (mon.0) 1177 : audit 1 from='client.? 172.21.15.16:0/1950201946' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T09:06:27.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[75179]: audit 2023-10-09T09:06:26.598090+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:06:27.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[75179]: cluster 2023-10-09T09:06:26.598135+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e212: 8 total, 8 up, 5 in 2023-10-09T09:06:27.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[75179]: audit 2023-10-09T09:06:27.310677+0000 mgr.y (mgr.25421) 870 : audit 0 from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:06:27.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[77892]: cluster 2023-10-09T09:06:25.907459+0000 mgr.y (mgr.25421) 869 : cluster 0 pgmap v908: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:27.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[77892]: cluster 2023-10-09T09:06:26.587794+0000 mon.a (mon.0) 1176 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:06:27.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[77892]: cluster 2023-10-09T09:06:26.594840+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T09:06:27.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[77892]: audit 2023-10-09T09:06:26.598038+0000 mon.a (mon.0) 1177 : audit 1 from='client.? 172.21.15.16:0/1950201946' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T09:06:27.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[77892]: audit 2023-10-09T09:06:26.598090+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:06:27.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[77892]: cluster 2023-10-09T09:06:26.598135+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e212: 8 total, 8 up, 5 in 2023-10-09T09:06:27.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:27 smithi016 bash[77892]: audit 2023-10-09T09:06:27.310677+0000 mgr.y (mgr.25421) 870 : audit 0 from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:06:27.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:27.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:27.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:27.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:27.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:27 smithi094 bash[59677]: cluster 2023-10-09T09:06:25.907459+0000 mgr.y (mgr.25421) 869 : cluster 0 pgmap v908: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:27 smithi094 bash[59677]: cluster 2023-10-09T09:06:26.587794+0000 mon.a (mon.0) 1176 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:06:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:27 smithi094 bash[59677]: cluster 2023-10-09T09:06:26.594840+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T09:06:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:27 smithi094 bash[59677]: audit 2023-10-09T09:06:26.598038+0000 mon.a (mon.0) 1177 : audit 1 from='client.? 172.21.15.16:0/1950201946' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T09:06:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:27 smithi094 bash[59677]: audit 2023-10-09T09:06:26.598090+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:06:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:27 smithi094 bash[59677]: cluster 2023-10-09T09:06:26.598135+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e212: 8 total, 8 up, 5 in 2023-10-09T09:06:27.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:27 smithi094 bash[59677]: audit 2023-10-09T09:06:27.310677+0000 mgr.y (mgr.25421) 870 : audit 0 from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:06:28.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:28.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:06:28.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:28.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:28.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:28.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:28.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:28.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:28.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:28.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:28.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:28.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:28.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:28.677 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:28.711 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:28.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[75179]: cluster 2023-10-09T09:06:27.603237+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[75179]: cluster 2023-10-09T09:06:27.603852+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[75179]: cluster 2023-10-09T09:06:27.605439+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[75179]: cluster 2023-10-09T09:06:27.605563+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e213: 8 total, 8 up, 5 in 2023-10-09T09:06:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[75179]: cluster 2023-10-09T09:06:27.605968+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[75179]: cluster 2023-10-09T09:06:27.908147+0000 mgr.y (mgr.25421) 871 : cluster 0 pgmap v911: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[77892]: cluster 2023-10-09T09:06:27.603237+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[77892]: cluster 2023-10-09T09:06:27.603852+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[77892]: cluster 2023-10-09T09:06:27.605439+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[77892]: cluster 2023-10-09T09:06:27.605563+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e213: 8 total, 8 up, 5 in 2023-10-09T09:06:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[77892]: cluster 2023-10-09T09:06:27.605968+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:28 smithi016 bash[77892]: cluster 2023-10-09T09:06:27.908147+0000 mgr.y (mgr.25421) 871 : cluster 0 pgmap v911: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:28.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:28.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:28.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:28.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:28.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:28 smithi094 bash[59677]: cluster 2023-10-09T09:06:27.603237+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:28 smithi094 bash[59677]: cluster 2023-10-09T09:06:27.603852+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:28 smithi094 bash[59677]: cluster 2023-10-09T09:06:27.605439+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:28 smithi094 bash[59677]: cluster 2023-10-09T09:06:27.605563+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e213: 8 total, 8 up, 5 in 2023-10-09T09:06:28.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:28 smithi094 bash[59677]: cluster 2023-10-09T09:06:27.605968+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T09:06:28.966 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:28 smithi094 bash[59677]: cluster 2023-10-09T09:06:27.908147+0000 mgr.y (mgr.25421) 871 : cluster 0 pgmap v911: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:28.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:28.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:29.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:29.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:29.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:29.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:29.217 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:29.249 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:29.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:29.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:29.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:29.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:29.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:29.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:29.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:29.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:29.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:29.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:29.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:29.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:29.759 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:29.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:29.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:29.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:29.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:29.906 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:29 smithi016 bash[77892]: cluster 2023-10-09T09:06:28.612138+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T09:06:29.906 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:29 smithi016 bash[77892]: cluster 2023-10-09T09:06:28.615198+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e214: 8 total, 8 up, 5 in 2023-10-09T09:06:29.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:29 smithi016 bash[75179]: cluster 2023-10-09T09:06:28.612138+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T09:06:29.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:29 smithi016 bash[75179]: cluster 2023-10-09T09:06:28.615198+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e214: 8 total, 8 up, 5 in 2023-10-09T09:06:29.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:29.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:29.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:29 smithi094 bash[59677]: cluster 2023-10-09T09:06:28.612138+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T09:06:29.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:29 smithi094 bash[59677]: cluster 2023-10-09T09:06:28.615198+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e214: 8 total, 8 up, 5 in 2023-10-09T09:06:30.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:30.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:30.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:30.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:30.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:30.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:30.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:30.450 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:30.555 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:30.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:30.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:30.694 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:30.707 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:30 smithi094 bash[59677]: cluster 2023-10-09T09:06:28.824032+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T09:06:30.707 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:30 smithi094 bash[59677]: cluster 2023-10-09T09:06:29.908598+0000 mgr.y (mgr.25421) 872 : cluster 0 pgmap v913: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:30.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:30.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:30.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T09:06:30.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:30.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:30.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:30.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:30.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:30 smithi016 bash[77892]: cluster 2023-10-09T09:06:28.824032+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T09:06:30.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:30 smithi016 bash[77892]: cluster 2023-10-09T09:06:29.908598+0000 mgr.y (mgr.25421) 872 : cluster 0 pgmap v913: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:30.967 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:30 smithi016 bash[75179]: cluster 2023-10-09T09:06:28.824032+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T09:06:30.967 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:30 smithi016 bash[75179]: cluster 2023-10-09T09:06:29.908598+0000 mgr.y (mgr.25421) 872 : cluster 0 pgmap v913: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:30.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:30.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:30.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:31.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:31.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:31.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:31.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:31.147 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":909,"stamp":"2023-10-09T09:06:26.594468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":2374990,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":3487140,"kb_used_data":612924,"kb_used_omap":117,"kb_used_meta":2874058,"kb_avail":465238620,"statfs":{"total":479975178240,"available":476404346880,"internally_reserved":0,"allocated":627634176,"data_stored":624137576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":2943036050},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.691386"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810959+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:06:01.810959+0000","last_peered":"2023-10-09T09:06:01.810959+0000","last_clean":"2023-10-09T09:06:01.810959+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:06:01.810959+0000","last_undegraded":"2023-10-09T09:06:01.810959+0000","last_fullsized":"2023-10-09T09:06:01.810959+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":222,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.666040+0000","last_change":"2023-10-09T09:02:02.625626+0000","last_active":"2023-10-09T09:06:01.666040+0000","last_peered":"2023-10-09T09:06:01.666040+0000","last_clean":"2023-10-09T09:06:01.666040+0000","last_became_active":"2023-10-09T09:02:02.625312+0000","last_became_peered":"2023-10-09T09:02:02.625312+0000","last_unstale":"2023-10-09T09:06:01.666040+0000","last_undegraded":"2023-10-09T09:06:01.666040+0000","last_fullsized":"2023-10-09T09:06:01.666040+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:01.640943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":175,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665476+0000","last_change":"2023-10-09T09:02:02.626274+0000","last_active":"2023-10-09T09:06:01.665476+0000","last_peered":"2023-10-09T09:06:01.665476+0000","last_clean":"2023-10-09T09:06:01.665476+0000","last_became_active":"2023-10-09T09:02:02.625973+0000","last_became_peered":"2023-10-09T09:02:02.625973+0000","last_unstale":"2023-10-09T09:06:01.665476+0000","last_undegraded":"2023-10-09T09:06:01.665476+0000","last_fullsized":"2023-10-09T09:06:01.665476+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:44.223794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":223,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806662+0000","last_change":"2023-10-09T09:03:19.554441+0000","last_active":"2023-10-09T09:06:01.806662+0000","last_peered":"2023-10-09T09:06:01.806662+0000","last_clean":"2023-10-09T09:06:01.806662+0000","last_became_active":"2023-10-09T09:03:19.553967+0000","last_became_peered":"2023-10-09T09:03:19.553967+0000","last_unstale":"2023-10-09T09:06:01.806662+0000","last_undegraded":"2023-10-09T09:06:01.806662+0000","last_fullsized":"2023-10-09T09:06:01.806662+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:26.065993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":211,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665941+0000","last_change":"2023-10-09T09:02:02.656821+0000","last_active":"2023-10-09T09:06:01.665941+0000","last_peered":"2023-10-09T09:06:01.665941+0000","last_clean":"2023-10-09T09:06:01.665941+0000","last_became_active":"2023-10-09T09:02:02.656523+0000","last_became_peered":"2023-10-09T09:02:02.656523+0000","last_unstale":"2023-10-09T09:06:01.665941+0000","last_undegraded":"2023-10-09T09:06:01.665941+0000","last_fullsized":"2023-10-09T09:06:01.665941+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:21.635518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.813932+0000","last_change":"2023-10-09T09:03:19.582769+0000","last_active":"2023-10-09T09:06:01.813932+0000","last_peered":"2023-10-09T09:06:01.813932+0000","last_clean":"2023-10-09T09:06:01.813932+0000","last_became_active":"2023-10-09T09:03:19.577023+0000","last_became_peered":"2023-10-09T09:03:19.577023+0000","last_unstale":"2023-10-09T09:06:01.813932+0000","last_undegraded":"2023-10-09T09:06:01.813932+0000","last_fullsized":"2023-10-09T09:06:01.813932+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":185,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806686+0000","last_change":"2023-10-09T09:01:36.280403+0000","last_active":"2023-10-09T09:06:01.806686+0000","last_peered":"2023-10-09T09:06:01.806686+0000","last_clean":"2023-10-09T09:06:01.806686+0000","last_became_active":"2023-10-09T09:01:36.280070+0000","last_became_peered":"2023-10-09T09:01:36.280070+0000","last_unstale":"2023-10-09T09:06:01.806686+0000","last_undegraded":"2023-10-09T09:06:01.806686+0000","last_fullsized":"2023-10-09T09:06:01.806686+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:33.796899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":223,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.666023+0000","last_change":"2023-10-09T09:02:02.630162+0000","last_active":"2023-10-09T09:06:01.666023+0000","last_peered":"2023-10-09T09:06:01.666023+0000","last_clean":"2023-10-09T09:06:01.666023+0000","last_became_active":"2023-10-09T09:02:02.629768+0000","last_became_peered":"2023-10-09T09:02:02.629768+0000","last_unstale":"2023-10-09T09:06:01.666023+0000","last_undegraded":"2023-10-09T09:06:01.666023+0000","last_fullsized":"2023-10-09T09:06:01.666023+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:36.859890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":29,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.666462+0000","last_change":"2023-10-09T09:03:19.562130+0000","last_active":"2023-10-09T09:06:01.666462+0000","last_peered":"2023-10-09T09:06:01.666462+0000","last_clean":"2023-10-09T09:06:01.666462+0000","last_became_active":"2023-10-09T09:03:19.557757+0000","last_became_peered":"2023-10-09T09:03:19.557757+0000","last_unstale":"2023-10-09T09:06:01.666462+0000","last_undegraded":"2023-10-09T09:06:01.666462+0000","last_fullsized":"2023-10-09T09:06:01.666462+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:46.546761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665380+0000","last_change":"2023-10-09T09:01:36.279247+0000","last_active":"2023-10-09T09:06:01.665380+0000","last_peered":"2023-10-09T09:06:01.665380+0000","last_clean":"2023-10-09T09:06:01.665380+0000","last_became_active":"2023-10-09T09:01:36.278376+0000","last_became_peered":"2023-10-09T09:01:36.278376+0000","last_unstale":"2023-10-09T09:06:01.665380+0000","last_undegraded":"2023-10-09T09:06:01.665380+0000","last_fullsized":"2023-10-09T09:06:01.665380+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:19.166567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":169,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:03.679898+0000","last_change":"2023-10-09T08:59:53.544302+0000","last_active":"2023-10-09T09:06:03.679898+0000","last_peered":"2023-10-09T09:06:03.679898+0000","last_clean":"2023-10-09T09:06:03.679898+0000","last_became_active":"2023-10-09T08:59:53.531404+0000","last_became_peered":"2023-10-09T08:59:53.531404+0000","last_unstale":"2023-10-09T09:06:03.679898+0000","last_undegraded":"2023-10-09T09:06:03.679898+0000","last_fullsized":"2023-10-09T09:06:03.679898+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:48.670745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.812932+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:06:01.812932+0000","last_peered":"2023-10-09T09:06:01.812932+0000","last_clean":"2023-10-09T09:06:01.812932+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:06:01.812932+0000","last_undegraded":"2023-10-09T09:06:01.812932+0000","last_fullsized":"2023-10-09T09:06:01.812932+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":93,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811396+0000","last_change":"2023-10-09T09:00:31.867387+0000","last_active":"2023-10-09T09:06:01.811396+0000","last_peered":"2023-10-09T09:06:01.811396+0000","last_clean":"2023-10-09T09:06:01.811396+0000","last_became_active":"2023-10-09T09:00:31.859752+0000","last_became_peered":"2023-10-09T09:00:31.859752+0000","last_unstale":"2023-10-09T09:06:01.811396+0000","last_undegraded":"2023-10-09T09:06:01.811396+0000","last_fullsized":"2023-10-09T09:06:01.811396+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":82,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810695+0000","last_change":"2023-10-09T09:02:02.630025+0000","last_active":"2023-10-09T09:06:01.810695+0000","last_peered":"2023-10-09T09:06:01.810695+0000","last_clean":"2023-10-09T09:06:01.810695+0000","last_became_active":"2023-10-09T09:02:02.629474+0000","last_became_peered":"2023-10-09T09:02:02.629474+0000","last_unstale":"2023-10-09T09:06:01.810695+0000","last_undegraded":"2023-10-09T09:06:01.810695+0000","last_fullsized":"2023-10-09T09:06:01.810695+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":85,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811009+0000","last_change":"2023-10-09T09:00:31.959283+0000","last_active":"2023-10-09T09:06:01.811009+0000","last_peered":"2023-10-09T09:06:01.811009+0000","last_clean":"2023-10-09T09:06:01.811009+0000","last_became_active":"2023-10-09T09:00:31.861638+0000","last_became_peered":"2023-10-09T09:00:31.861638+0000","last_unstale":"2023-10-09T09:06:01.811009+0000","last_undegraded":"2023-10-09T09:06:01.811009+0000","last_fullsized":"2023-10-09T09:06:01.811009+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":205,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.666168+0000","last_change":"2023-10-09T09:02:02.636035+0000","last_active":"2023-10-09T09:06:01.666168+0000","last_peered":"2023-10-09T09:06:01.666168+0000","last_clean":"2023-10-09T09:06:01.666168+0000","last_became_active":"2023-10-09T09:02:02.635621+0000","last_became_peered":"2023-10-09T09:02:02.635621+0000","last_unstale":"2023-10-09T09:06:01.666168+0000","last_undegraded":"2023-10-09T09:06:01.666168+0000","last_fullsized":"2023-10-09T09:06:01.666168+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:46.294650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":212,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806022+0000","last_change":"2023-10-09T09:03:19.564077+0000","last_active":"2023-10-09T09:06:01.806022+0000","last_peered":"2023-10-09T09:06:01.806022+0000","last_clean":"2023-10-09T09:06:01.806022+0000","last_became_active":"2023-10-09T09:03:19.563764+0000","last_became_peered":"2023-10-09T09:03:19.563764+0000","last_unstale":"2023-10-09T09:06:01.806022+0000","last_undegraded":"2023-10-09T09:06:01.806022+0000","last_fullsized":"2023-10-09T09:06:01.806022+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:38.043450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":169,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:03.679793+0000","last_change":"2023-10-09T09:01:36.279054+0000","last_active":"2023-10-09T09:06:03.679793+0000","last_peered":"2023-10-09T09:06:03.679793+0000","last_clean":"2023-10-09T09:06:03.679793+0000","last_became_active":"2023-10-09T09:01:36.278755+0000","last_became_peered":"2023-10-09T09:01:36.278755+0000","last_unstale":"2023-10-09T09:06:03.679793+0000","last_undegraded":"2023-10-09T09:06:03.679793+0000","last_fullsized":"2023-10-09T09:06:03.679793+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:09.089065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810790+0000","last_change":"2023-10-09T09:00:31.851267+0000","last_active":"2023-10-09T09:06:01.810790+0000","last_peered":"2023-10-09T09:06:01.810790+0000","last_clean":"2023-10-09T09:06:01.810790+0000","last_became_active":"2023-10-09T09:00:31.850918+0000","last_became_peered":"2023-10-09T09:00:31.850918+0000","last_unstale":"2023-10-09T09:06:01.810790+0000","last_undegraded":"2023-10-09T09:06:01.810790+0000","last_fullsized":"2023-10-09T09:06:01.810790+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":185,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806742+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T09:06:01.806742+0000","last_peered":"2023-10-09T09:06:01.806742+0000","last_clean":"2023-10-09T09:06:01.806742+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T09:06:01.806742+0000","last_undegraded":"2023-10-09T09:06:01.806742+0000","last_fullsized":"2023-10-09T09:06:01.806742+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.812740+0000","last_change":"2023-10-09T09:01:36.301852+0000","last_active":"2023-10-09T09:06:01.812740+0000","last_peered":"2023-10-09T09:06:01.812740+0000","last_clean":"2023-10-09T09:06:01.812740+0000","last_became_active":"2023-10-09T09:01:36.284416+0000","last_became_peered":"2023-10-09T09:01:36.284416+0000","last_unstale":"2023-10-09T09:06:01.812740+0000","last_undegraded":"2023-10-09T09:06:01.812740+0000","last_fullsized":"2023-10-09T09:06:01.812740+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":29,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665773+0000","last_change":"2023-10-09T09:03:19.575211+0000","last_active":"2023-10-09T09:06:01.665773+0000","last_peered":"2023-10-09T09:06:01.665773+0000","last_clean":"2023-10-09T09:06:01.665773+0000","last_became_active":"2023-10-09T09:03:19.574829+0000","last_became_peered":"2023-10-09T09:03:19.574829+0000","last_unstale":"2023-10-09T09:06:01.665773+0000","last_undegraded":"2023-10-09T09:06:01.665773+0000","last_fullsized":"2023-10-09T09:06:01.665773+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:14.763790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811608+0000","last_change":"2023-10-09T09:03:19.626156+0000","last_active":"2023-10-09T09:06:01.811608+0000","last_peered":"2023-10-09T09:06:01.811608+0000","last_clean":"2023-10-09T09:06:01.811608+0000","last_became_active":"2023-10-09T09:03:19.575061+0000","last_became_peered":"2023-10-09T09:03:19.575061+0000","last_unstale":"2023-10-09T09:06:01.811608+0000","last_undegraded":"2023-10-09T09:06:01.811608+0000","last_fullsized":"2023-10-09T09:06:01.811608+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":188,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.812559+0000","last_change":"2023-10-09T09:03:19.582540+0000","last_active":"2023-10-09T09:06:01.812559+0000","last_peered":"2023-10-09T09:06:01.812559+0000","last_clean":"2023-10-09T09:06:01.812559+0000","last_became_active":"2023-10-09T09:03:19.577044+0000","last_became_peered":"2023-10-09T09:03:19.577044+0000","last_unstale":"2023-10-09T09:06:01.812559+0000","last_undegraded":"2023-10-09T09:06:01.812559+0000","last_fullsized":"2023-10-09T09:06:01.812559+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":81,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811705+0000","last_change":"2023-10-09T09:02:02.724809+0000","last_active":"2023-10-09T09:06:01.811705+0000","last_peered":"2023-10-09T09:06:01.811705+0000","last_clean":"2023-10-09T09:06:01.811705+0000","last_became_active":"2023-10-09T09:02:02.666202+0000","last_became_peered":"2023-10-09T09:02:02.666202+0000","last_unstale":"2023-10-09T09:06:01.811705+0000","last_undegraded":"2023-10-09T09:06:01.811705+0000","last_fullsized":"2023-10-09T09:06:01.811705+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":91,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810539+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:06:01.810539+0000","last_peered":"2023-10-09T09:06:01.810539+0000","last_clean":"2023-10-09T09:06:01.810539+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:06:01.810539+0000","last_undegraded":"2023-10-09T09:06:01.810539+0000","last_fullsized":"2023-10-09T09:06:01.810539+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":120,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.813661+0000","last_change":"2023-10-09T09:01:36.305615+0000","last_active":"2023-10-09T09:06:01.813661+0000","last_peered":"2023-10-09T09:06:01.813661+0000","last_clean":"2023-10-09T09:06:01.813661+0000","last_became_active":"2023-10-09T09:01:36.292828+0000","last_became_peered":"2023-10-09T09:01:36.292828+0000","last_unstale":"2023-10-09T09:06:01.813661+0000","last_undegraded":"2023-10-09T09:06:01.813661+0000","last_fullsized":"2023-10-09T09:06:01.813661+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.813119+0000","last_change":"2023-10-09T09:03:19.564369+0000","last_active":"2023-10-09T09:06:01.813119+0000","last_peered":"2023-10-09T09:06:01.813119+0000","last_clean":"2023-10-09T09:06:01.813119+0000","last_became_active":"2023-10-09T09:03:19.563989+0000","last_became_peered":"2023-10-09T09:03:19.563989+0000","last_unstale":"2023-10-09T09:06:01.813119+0000","last_undegraded":"2023-10-09T09:06:01.813119+0000","last_fullsized":"2023-10-09T09:06:01.813119+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":179,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:03.679640+0000","last_change":"2023-10-09T09:02:02.664617+0000","last_active":"2023-10-09T09:06:03.679640+0000","last_peered":"2023-10-09T09:06:03.679640+0000","last_clean":"2023-10-09T09:06:03.679640+0000","last_became_active":"2023-10-09T09:02:02.664305+0000","last_became_peered":"2023-10-09T09:02:02.664305+0000","last_unstale":"2023-10-09T09:06:03.679640+0000","last_undegraded":"2023-10-09T09:06:03.679640+0000","last_fullsized":"2023-10-09T09:06:03.679640+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:14.520177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":204,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:02.672777+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:06:02.672777+0000","last_peered":"2023-10-09T09:06:02.672777+0000","last_clean":"2023-10-09T09:06:02.672777+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:06:02.672777+0000","last_undegraded":"2023-10-09T09:06:02.672777+0000","last_fullsized":"2023-10-09T09:06:02.672777+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":212,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665665+0000","last_change":"2023-10-09T08:59:53.642095+0000","last_active":"2023-10-09T09:06:01.665665+0000","last_peered":"2023-10-09T09:06:01.665665+0000","last_clean":"2023-10-09T09:06:01.665665+0000","last_became_active":"2023-10-09T08:59:53.531588+0000","last_became_peered":"2023-10-09T08:59:53.531588+0000","last_unstale":"2023-10-09T09:06:01.665665+0000","last_undegraded":"2023-10-09T09:06:01.665665+0000","last_fullsized":"2023-10-09T09:06:01.665665+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.878230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811106+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:06:01.811106+0000","last_peered":"2023-10-09T09:06:01.811106+0000","last_clean":"2023-10-09T09:06:01.811106+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:06:01.811106+0000","last_undegraded":"2023-10-09T09:06:01.811106+0000","last_fullsized":"2023-10-09T09:06:01.811106+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.813044+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:06:01.813044+0000","last_peered":"2023-10-09T09:06:01.813044+0000","last_clean":"2023-10-09T09:06:01.813044+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:06:01.813044+0000","last_undegraded":"2023-10-09T09:06:01.813044+0000","last_fullsized":"2023-10-09T09:06:01.813044+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.812044+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:06:01.812044+0000","last_peered":"2023-10-09T09:06:01.812044+0000","last_clean":"2023-10-09T09:06:01.812044+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:06:01.812044+0000","last_undegraded":"2023-10-09T09:06:01.812044+0000","last_fullsized":"2023-10-09T09:06:01.812044+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":82,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810477+0000","last_change":"2023-10-09T09:03:19.563963+0000","last_active":"2023-10-09T09:06:01.810477+0000","last_peered":"2023-10-09T09:06:01.810477+0000","last_clean":"2023-10-09T09:06:01.810477+0000","last_became_active":"2023-10-09T09:03:19.563629+0000","last_became_peered":"2023-10-09T09:03:19.563629+0000","last_unstale":"2023-10-09T09:06:01.810477+0000","last_undegraded":"2023-10-09T09:06:01.810477+0000","last_fullsized":"2023-10-09T09:06:01.810477+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":82,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811356+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:06:01.811356+0000","last_peered":"2023-10-09T09:06:01.811356+0000","last_clean":"2023-10-09T09:06:01.811356+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:06:01.811356+0000","last_undegraded":"2023-10-09T09:06:01.811356+0000","last_fullsized":"2023-10-09T09:06:01.811356+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":29,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665224+0000","last_change":"2023-10-09T09:03:19.572389+0000","last_active":"2023-10-09T09:06:01.665224+0000","last_peered":"2023-10-09T09:06:01.665224+0000","last_clean":"2023-10-09T09:06:01.665224+0000","last_became_active":"2023-10-09T09:03:19.572091+0000","last_became_peered":"2023-10-09T09:03:19.572091+0000","last_unstale":"2023-10-09T09:06:01.665224+0000","last_undegraded":"2023-10-09T09:06:01.665224+0000","last_fullsized":"2023-10-09T09:06:01.665224+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:27.702718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.667061+0000","last_change":"2023-10-09T09:02:02.636996+0000","last_active":"2023-10-09T09:06:01.667061+0000","last_peered":"2023-10-09T09:06:01.667061+0000","last_clean":"2023-10-09T09:06:01.667061+0000","last_became_active":"2023-10-09T09:02:02.636702+0000","last_became_peered":"2023-10-09T09:02:02.636702+0000","last_unstale":"2023-10-09T09:06:01.667061+0000","last_undegraded":"2023-10-09T09:06:01.667061+0000","last_fullsized":"2023-10-09T09:06:01.667061+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:45.392645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":154,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:04.381603+0000","last_change":"2023-10-09T09:03:19.556450+0000","last_active":"2023-10-09T09:06:04.381603+0000","last_peered":"2023-10-09T09:06:04.381603+0000","last_clean":"2023-10-09T09:06:04.381603+0000","last_became_active":"2023-10-09T09:03:19.556190+0000","last_became_peered":"2023-10-09T09:03:19.556190+0000","last_unstale":"2023-10-09T09:06:04.381603+0000","last_undegraded":"2023-10-09T09:06:04.381603+0000","last_fullsized":"2023-10-09T09:06:04.381603+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:23.704455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":149,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.666214+0000","last_change":"2023-10-09T09:01:36.285844+0000","last_active":"2023-10-09T09:06:01.666214+0000","last_peered":"2023-10-09T09:06:01.666214+0000","last_clean":"2023-10-09T09:06:01.666214+0000","last_became_active":"2023-10-09T09:01:36.285525+0000","last_became_peered":"2023-10-09T09:01:36.285525+0000","last_unstale":"2023-10-09T09:06:01.666214+0000","last_undegraded":"2023-10-09T09:06:01.666214+0000","last_fullsized":"2023-10-09T09:06:01.666214+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:05.624756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":190,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811350+0000","last_change":"2023-10-09T09:02:52.021749+0000","last_active":"2023-10-09T09:06:01.811350+0000","last_peered":"2023-10-09T09:06:01.811350+0000","last_clean":"2023-10-09T09:06:01.811350+0000","last_became_active":"2023-10-09T09:02:52.021416+0000","last_became_peered":"2023-10-09T09:02:52.021416+0000","last_unstale":"2023-10-09T09:06:01.811350+0000","last_undegraded":"2023-10-09T09:06:01.811350+0000","last_fullsized":"2023-10-09T09:06:01.811350+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":178,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.812974+0000","last_change":"2023-10-09T09:03:19.564803+0000","last_active":"2023-10-09T09:06:01.812974+0000","last_peered":"2023-10-09T09:06:01.812974+0000","last_clean":"2023-10-09T09:06:01.812974+0000","last_became_active":"2023-10-09T09:03:19.564520+0000","last_became_peered":"2023-10-09T09:03:19.564520+0000","last_unstale":"2023-10-09T09:06:01.812974+0000","last_undegraded":"2023-10-09T09:06:01.812974+0000","last_fullsized":"2023-10-09T09:06:01.812974+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811131+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:06:01.811131+0000","last_peered":"2023-10-09T09:06:01.811131+0000","last_clean":"2023-10-09T09:06:01.811131+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:06:01.811131+0000","last_undegraded":"2023-10-09T09:06:01.811131+0000","last_fullsized":"2023-10-09T09:06:01.811131+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":178,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:03.659036+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:06:03.659036+0000","last_peered":"2023-10-09T09:06:03.659036+0000","last_clean":"2023-10-09T09:06:03.659036+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:06:03.659036+0000","last_undegraded":"2023-10-09T09:06:03.659036+0000","last_fullsized":"2023-10-09T09:06:03.659036+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":236,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.646287+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:06:01.646287+0000","last_peered":"2023-10-09T09:06:01.646287+0000","last_clean":"2023-10-09T09:06:01.646287+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:06:01.646287+0000","last_undegraded":"2023-10-09T09:06:01.646287+0000","last_fullsized":"2023-10-09T09:06:01.646287+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":171,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.812563+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:06:01.812563+0000","last_peered":"2023-10-09T09:06:01.812563+0000","last_clean":"2023-10-09T09:06:01.812563+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:06:01.812563+0000","last_undegraded":"2023-10-09T09:06:01.812563+0000","last_fullsized":"2023-10-09T09:06:01.812563+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":218,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:04.381421+0000","last_change":"2023-10-09T09:03:19.575546+0000","last_active":"2023-10-09T09:06:04.381421+0000","last_peered":"2023-10-09T09:06:04.381421+0000","last_clean":"2023-10-09T09:06:04.381421+0000","last_became_active":"2023-10-09T09:03:19.575075+0000","last_became_peered":"2023-10-09T09:03:19.575075+0000","last_unstale":"2023-10-09T09:06:04.381421+0000","last_undegraded":"2023-10-09T09:06:04.381421+0000","last_fullsized":"2023-10-09T09:06:04.381421+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:14.350886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":142,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.646564+0000","last_change":"2023-10-09T09:01:36.274911+0000","last_active":"2023-10-09T09:06:01.646564+0000","last_peered":"2023-10-09T09:06:01.646564+0000","last_clean":"2023-10-09T09:06:01.646564+0000","last_became_active":"2023-10-09T09:01:36.274560+0000","last_became_peered":"2023-10-09T09:01:36.274560+0000","last_unstale":"2023-10-09T09:06:01.646564+0000","last_undegraded":"2023-10-09T09:06:01.646564+0000","last_fullsized":"2023-10-09T09:06:01.646564+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:55.305969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":451,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:04.258065+0000","last_change":"2023-10-09T09:02:05.534070+0000","last_active":"2023-10-09T09:06:04.258065+0000","last_peered":"2023-10-09T09:06:04.258065+0000","last_clean":"2023-10-09T09:06:04.258065+0000","last_became_active":"2023-10-09T09:02:05.533567+0000","last_became_peered":"2023-10-09T09:02:05.533567+0000","last_unstale":"2023-10-09T09:06:04.258065+0000","last_undegraded":"2023-10-09T09:06:04.258065+0000","last_fullsized":"2023-10-09T09:06:04.258065+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:59.568077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.666549+0000","last_change":"2023-10-09T09:03:19.562659+0000","last_active":"2023-10-09T09:06:01.666549+0000","last_peered":"2023-10-09T09:06:01.666549+0000","last_clean":"2023-10-09T09:06:01.666549+0000","last_became_active":"2023-10-09T09:03:19.562284+0000","last_became_peered":"2023-10-09T09:03:19.562284+0000","last_unstale":"2023-10-09T09:06:01.666549+0000","last_undegraded":"2023-10-09T09:06:01.666549+0000","last_fullsized":"2023-10-09T09:06:01.666549+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:26.537954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.813808+0000","last_change":"2023-10-09T09:00:31.859121+0000","last_active":"2023-10-09T09:06:01.813808+0000","last_peered":"2023-10-09T09:06:01.813808+0000","last_clean":"2023-10-09T09:06:01.813808+0000","last_became_active":"2023-10-09T09:00:31.858675+0000","last_became_peered":"2023-10-09T09:00:31.858675+0000","last_unstale":"2023-10-09T09:06:01.813808+0000","last_undegraded":"2023-10-09T09:06:01.813808+0000","last_fullsized":"2023-10-09T09:06:01.813808+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":105,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:02.672712+0000","last_change":"2023-10-09T09:02:02.649488+0000","last_active":"2023-10-09T09:06:02.672712+0000","last_peered":"2023-10-09T09:06:02.672712+0000","last_clean":"2023-10-09T09:06:02.672712+0000","last_became_active":"2023-10-09T09:02:02.645550+0000","last_became_peered":"2023-10-09T09:02:02.645550+0000","last_unstale":"2023-10-09T09:06:02.672712+0000","last_undegraded":"2023-10-09T09:06:02.672712+0000","last_fullsized":"2023-10-09T09:06:02.672712+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:40.192731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":240,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.805734+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:06:01.805734+0000","last_peered":"2023-10-09T09:06:01.805734+0000","last_clean":"2023-10-09T09:06:01.805734+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:06:01.805734+0000","last_undegraded":"2023-10-09T09:06:01.805734+0000","last_fullsized":"2023-10-09T09:06:01.805734+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":201,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806571+0000","last_change":"2023-10-09T09:02:02.657733+0000","last_active":"2023-10-09T09:06:01.806571+0000","last_peered":"2023-10-09T09:06:01.806571+0000","last_clean":"2023-10-09T09:06:01.806571+0000","last_became_active":"2023-10-09T09:02:02.656701+0000","last_became_peered":"2023-10-09T09:02:02.656701+0000","last_unstale":"2023-10-09T09:06:01.806571+0000","last_undegraded":"2023-10-09T09:06:01.806571+0000","last_fullsized":"2023-10-09T09:06:01.806571+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:39.552401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":184,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806742+0000","last_change":"2023-10-09T09:02:02.644204+0000","last_active":"2023-10-09T09:06:01.806742+0000","last_peered":"2023-10-09T09:06:01.806742+0000","last_clean":"2023-10-09T09:06:01.806742+0000","last_became_active":"2023-10-09T09:02:02.643713+0000","last_became_peered":"2023-10-09T09:02:02.643713+0000","last_unstale":"2023-10-09T09:06:01.806742+0000","last_undegraded":"2023-10-09T09:06:01.806742+0000","last_fullsized":"2023-10-09T09:06:01.806742+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:20.811353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":180,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T09:06:03.657292+0000","last_change":"2023-10-09T09:02:02.647803+0000","last_active":"2023-10-09T09:06:03.657292+0000","last_peered":"2023-10-09T09:06:03.657292+0000","last_clean":"2023-10-09T09:06:03.657292+0000","last_became_active":"2023-10-09T09:02:02.646911+0000","last_became_peered":"2023-10-09T09:02:02.646911+0000","last_unstale":"2023-10-09T09:06:03.657292+0000","last_undegraded":"2023-10-09T09:06:03.657292+0000","last_fullsized":"2023-10-09T09:06:03.657292+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:35.294072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":198,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806222+0000","last_change":"2023-10-09T09:02:52.016015+0000","last_active":"2023-10-09T09:06:01.806222+0000","last_peered":"2023-10-09T09:06:01.806222+0000","last_clean":"2023-10-09T09:06:01.806222+0000","last_became_active":"2023-10-09T09:02:52.015543+0000","last_became_peered":"2023-10-09T09:02:52.015543+0000","last_unstale":"2023-10-09T09:06:01.806222+0000","last_undegraded":"2023-10-09T09:06:01.806222+0000","last_fullsized":"2023-10-09T09:06:01.806222+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.410980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":211,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.667146+0000","last_change":"2023-10-09T09:01:36.292450+0000","last_active":"2023-10-09T09:06:01.667146+0000","last_peered":"2023-10-09T09:06:01.667146+0000","last_clean":"2023-10-09T09:06:01.667146+0000","last_became_active":"2023-10-09T09:01:36.273029+0000","last_became_peered":"2023-10-09T09:01:36.273029+0000","last_unstale":"2023-10-09T09:06:01.667146+0000","last_undegraded":"2023-10-09T09:06:01.667146+0000","last_fullsized":"2023-10-09T09:06:01.667146+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:26.065684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":173,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.813835+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:06:01.813835+0000","last_peered":"2023-10-09T09:06:01.813835+0000","last_clean":"2023-10-09T09:06:01.813835+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:06:01.813835+0000","last_undegraded":"2023-10-09T09:06:01.813835+0000","last_fullsized":"2023-10-09T09:06:01.813835+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":29,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665257+0000","last_change":"2023-10-09T09:03:19.572475+0000","last_active":"2023-10-09T09:06:01.665257+0000","last_peered":"2023-10-09T09:06:01.665257+0000","last_clean":"2023-10-09T09:06:01.665257+0000","last_became_active":"2023-10-09T09:03:19.572081+0000","last_became_peered":"2023-10-09T09:03:19.572081+0000","last_unstale":"2023-10-09T09:06:01.665257+0000","last_undegraded":"2023-10-09T09:06:01.665257+0000","last_fullsized":"2023-10-09T09:06:01.665257+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:04.771178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:06:26.587297+0000","last_change":"2023-10-09T09:06:26.587297+0000","last_active":"2023-10-09T09:06:26.587297+0000","last_peered":"2023-10-09T09:06:26.587297+0000","last_clean":"2023-10-09T09:06:26.587297+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:06:26.587297+0000","last_undegraded":"2023-10-09T09:06:26.587297+0000","last_fullsized":"2023-10-09T09:06:26.587297+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":60,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.814088+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:06:01.814088+0000","last_peered":"2023-10-09T09:06:01.814088+0000","last_clean":"2023-10-09T09:06:01.814088+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:06:01.814088+0000","last_undegraded":"2023-10-09T09:06:01.814088+0000","last_fullsized":"2023-10-09T09:06:01.814088+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":220,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806591+0000","last_change":"2023-10-09T09:02:02.674657+0000","last_active":"2023-10-09T09:06:01.806591+0000","last_peered":"2023-10-09T09:06:01.806591+0000","last_clean":"2023-10-09T09:06:01.806591+0000","last_became_active":"2023-10-09T09:02:02.657392+0000","last_became_peered":"2023-10-09T09:02:02.657392+0000","last_unstale":"2023-10-09T09:06:01.806591+0000","last_undegraded":"2023-10-09T09:06:01.806591+0000","last_fullsized":"2023-10-09T09:06:01.806591+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:16.286583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":82,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810721+0000","last_change":"2023-10-09T09:01:36.280537+0000","last_active":"2023-10-09T09:06:01.810721+0000","last_peered":"2023-10-09T09:06:01.810721+0000","last_clean":"2023-10-09T09:06:01.810721+0000","last_became_active":"2023-10-09T09:01:36.280149+0000","last_became_peered":"2023-10-09T09:01:36.280149+0000","last_unstale":"2023-10-09T09:06:01.810721+0000","last_undegraded":"2023-10-09T09:06:01.810721+0000","last_fullsized":"2023-10-09T09:06:01.810721+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":219,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806186+0000","last_change":"2023-10-09T09:03:19.554339+0000","last_active":"2023-10-09T09:06:01.806186+0000","last_peered":"2023-10-09T09:06:01.806186+0000","last_clean":"2023-10-09T09:06:01.806186+0000","last_became_active":"2023-10-09T09:03:19.553953+0000","last_became_peered":"2023-10-09T09:03:19.553953+0000","last_unstale":"2023-10-09T09:06:01.806186+0000","last_undegraded":"2023-10-09T09:06:01.806186+0000","last_fullsized":"2023-10-09T09:06:01.806186+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:50.601515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.646721+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:06:01.646721+0000","last_peered":"2023-10-09T09:06:01.646721+0000","last_clean":"2023-10-09T09:06:01.646721+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:06:01.646721+0000","last_undegraded":"2023-10-09T09:06:01.646721+0000","last_fullsized":"2023-10-09T09:06:01.646721+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":213,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811008+0000","last_change":"2023-10-09T09:02:52.022788+0000","last_active":"2023-10-09T09:06:01.811008+0000","last_peered":"2023-10-09T09:06:01.811008+0000","last_clean":"2023-10-09T09:06:01.811008+0000","last_became_active":"2023-10-09T09:02:52.016418+0000","last_became_peered":"2023-10-09T09:02:52.016418+0000","last_unstale":"2023-10-09T09:06:01.811008+0000","last_undegraded":"2023-10-09T09:06:01.811008+0000","last_fullsized":"2023-10-09T09:06:01.811008+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810838+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:06:01.810838+0000","last_peered":"2023-10-09T09:06:01.810838+0000","last_clean":"2023-10-09T09:06:01.810838+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:06:01.810838+0000","last_undegraded":"2023-10-09T09:06:01.810838+0000","last_fullsized":"2023-10-09T09:06:01.810838+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":198,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811379+0000","last_change":"2023-10-09T09:03:19.575218+0000","last_active":"2023-10-09T09:06:01.811379+0000","last_peered":"2023-10-09T09:06:01.811379+0000","last_clean":"2023-10-09T09:06:01.811379+0000","last_became_active":"2023-10-09T09:03:19.557578+0000","last_became_peered":"2023-10-09T09:03:19.557578+0000","last_unstale":"2023-10-09T09:06:01.811379+0000","last_undegraded":"2023-10-09T09:06:01.811379+0000","last_fullsized":"2023-10-09T09:06:01.811379+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":64,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.811746+0000","last_change":"2023-10-09T08:59:53.522794+0000","last_active":"2023-10-09T09:06:01.811746+0000","last_peered":"2023-10-09T09:06:01.811746+0000","last_clean":"2023-10-09T09:06:01.811746+0000","last_became_active":"2023-10-09T08:59:53.522419+0000","last_became_peered":"2023-10-09T08:59:53.522419+0000","last_unstale":"2023-10-09T09:06:01.811746+0000","last_undegraded":"2023-10-09T09:06:01.811746+0000","last_fullsized":"2023-10-09T09:06:01.811746+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":212,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.666171+0000","last_change":"2023-10-09T09:00:31.845771+0000","last_active":"2023-10-09T09:06:01.666171+0000","last_peered":"2023-10-09T09:06:01.666171+0000","last_clean":"2023-10-09T09:06:01.666171+0000","last_became_active":"2023-10-09T09:00:31.844775+0000","last_became_peered":"2023-10-09T09:00:31.844775+0000","last_unstale":"2023-10-09T09:06:01.666171+0000","last_undegraded":"2023-10-09T09:06:01.666171+0000","last_fullsized":"2023-10-09T09:06:01.666171+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:33.452539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":29,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.665646+0000","last_change":"2023-10-09T09:03:19.561924+0000","last_active":"2023-10-09T09:06:01.665646+0000","last_peered":"2023-10-09T09:06:01.665646+0000","last_clean":"2023-10-09T09:06:01.665646+0000","last_became_active":"2023-10-09T09:03:19.557692+0000","last_became_peered":"2023-10-09T09:03:19.557692+0000","last_unstale":"2023-10-09T09:06:01.665646+0000","last_undegraded":"2023-10-09T09:06:01.665646+0000","last_fullsized":"2023-10-09T09:06:01.665646+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:39.940109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":271,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.806718+0000","last_change":"2023-10-09T09:02:52.019620+0000","last_active":"2023-10-09T09:06:01.806718+0000","last_peered":"2023-10-09T09:06:01.806718+0000","last_clean":"2023-10-09T09:06:01.806718+0000","last_became_active":"2023-10-09T09:02:52.014489+0000","last_became_peered":"2023-10-09T09:02:52.014489+0000","last_unstale":"2023-10-09T09:06:01.806718+0000","last_undegraded":"2023-10-09T09:06:01.806718+0000","last_fullsized":"2023-10-09T09:06:01.806718+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:39.550600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":91,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T09:06:01.810072+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:06:01.810072+0000","last_peered":"2023-10-09T09:06:01.810072+0000","last_clean":"2023-10-09T09:06:01.810072+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:06:01.810072+0000","last_undegraded":"2023-10-09T09:06:01.810072+0000","last_fullsized":"2023-10-09T09:06:01.810072+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":13518,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":139264,"data_stored":22108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430631,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626864,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":996520,"kb_used_data":101344,"kb_used_omap":0,"kb_used_meta":895168,"kb_avail":92748632,"statfs":{"total":95995035648,"available":94974599168,"internally_reserved":0,"allocated":103776256,"data_stored":103280604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916652032},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.786,"15min":2.3410000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":3.6259999999999999,"5min":58.621000000000002,"15min":153.63800000000001},"last":1.153},{"interface":"front","average":{"1min":1.365,"5min":2.0289999999999999,"15min":2.3540000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.6539999999999999,"5min":58.590000000000003,"15min":153.72200000000001},"last":1.1040000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:05:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.8440000000000001,"15min":1.8620000000000001},"min":{"1min":0.33000000000000002,"5min":0.32000000000000001,"15min":0.28299999999999997},"max":{"1min":1.0369999999999999,"5min":109.453,"15min":109.453},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.77900000000000003,"5min":2.073,"15min":1.9339999999999999},"min":{"1min":0.441,"5min":0.316,"15min":0.314},"max":{"1min":1.9219999999999999,"5min":111.47499999999999,"15min":111.47499999999999},"last":1.262}]},{"osd":2,"last update":"Mon Oct 9 09:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":1.04,"15min":1.393},"min":{"1min":0.45800000000000002,"5min":0.31900000000000001,"15min":0.29799999999999999},"max":{"1min":1.125,"5min":5.6820000000000004,"15min":8.3979999999999997},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.998,"5min":1.1719999999999999,"15min":1.488},"min":{"1min":0.49399999999999999,"5min":0.30199999999999999,"15min":0.28100000000000003},"max":{"1min":1.881,"5min":9.1549999999999994,"15min":12.194000000000001},"last":1.204}]},{"osd":3,"last update":"Mon Oct 9 09:05:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.23,"15min":2.0870000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.7130000000000001,"5min":6.992,"15min":57.636000000000003},"last":1.075},{"interface":"front","average":{"1min":1.056,"5min":1.599,"15min":1.9059999999999999},"min":{"1min":0.49099999999999999,"5min":0.46500000000000002,"15min":0.34699999999999998},"max":{"1min":3.871,"5min":9.4429999999999996,"15min":58.393000000000001},"last":0.92500000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.4159999999999999,"15min":1.5369999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.0600000000000001,"5min":32.280999999999999,"15min":32.280999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.2669999999999999,"15min":1.1699999999999999},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.30599999999999999},"max":{"1min":1.446,"5min":34.689,"15min":34.689},"last":1.1259999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:05:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.0429999999999999,"15min":1.29},"min":{"1min":0.41399999999999998,"5min":0.40300000000000002,"15min":0.35499999999999998},"max":{"1min":3.5609999999999999,"5min":9.218,"15min":12.404999999999999},"last":1.167},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.90900000000000003,"15min":1.1279999999999999},"min":{"1min":0.50900000000000001,"5min":0.41599999999999998,"15min":0.33900000000000002},"max":{"1min":1.407,"5min":4.2610000000000001,"15min":8.8569999999999993},"last":1.401}]},{"osd":7,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.0529999999999999,"15min":1.4139999999999999},"min":{"1min":0.47599999999999998,"5min":0.434,"15min":0.40500000000000003},"max":{"1min":2.0289999999999999,"5min":9.2460000000000004,"15min":11.382},"last":1.3069999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.93799999999999994,"15min":1.2350000000000001},"min":{"1min":0.46500000000000002,"5min":0.45400000000000001,"15min":0.377},"max":{"1min":3.4140000000000001,"5min":4.3259999999999996,"15min":9.4800000000000004},"last":1.05}]}]},{"osd":1,"up_from":0,"seq":356482285811,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449337,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429200,"kb_used_data":2824,"kb_used_omap":28,"kb_used_meta":426339,"kb_avail":93315952,"statfs":{"total":95995035648,"available":95555534848,"internally_reserved":0,"allocated":2891776,"data_stored":2066656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":436571984},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.1779999999999999,"15min":1.609},"min":{"1min":0.308,"5min":0.22,"15min":0.21199999999999999},"max":{"1min":3.0510000000000002,"5min":6.7649999999999997,"15min":37.380000000000003},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.073,"5min":1.133,"15min":1.651},"min":{"1min":0.33700000000000002,"5min":0.254,"15min":0.21199999999999999},"max":{"1min":5.7400000000000002,"5min":10.102,"15min":38.012999999999998},"last":0.55900000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":3.4039999999999999,"15min":2.4580000000000002},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":9.1869999999999994,"5min":71.322999999999993,"15min":71.322999999999993},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.4449999999999998,"5min":5.8140000000000001,"15min":3.3559999999999999},"min":{"1min":0.36799999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.7290000000000001,"5min":219.00700000000001,"15min":219.00700000000001},"last":0.63}]},{"osd":3,"last update":"Mon Oct 9 09:05:38 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":4.431,"15min":3.4660000000000002},"min":{"1min":0.437,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":7.2069999999999999,"5min":219.041,"15min":219.041},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.6479999999999999,"5min":2.0510000000000002,"15min":2.6389999999999998},"min":{"1min":0.40600000000000003,"5min":0.28899999999999998,"15min":0.25},"max":{"1min":9.8409999999999993,"5min":20.466000000000001,"15min":107.66500000000001},"last":0.69999999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.5449999999999999,"15min":1.51},"min":{"1min":0.46999999999999997,"5min":0.40200000000000002,"15min":0.30199999999999999},"max":{"1min":7.0259999999999998,"5min":10.178000000000001,"15min":10.178000000000001},"last":1.022},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.431,"15min":1.6830000000000001},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.375,"5min":10.526,"15min":11.654},"last":0.90700000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.522,"15min":1.423},"min":{"1min":0.42899999999999999,"5min":0.40500000000000003,"15min":0.34499999999999997},"max":{"1min":4.3630000000000004,"5min":9.3030000000000008,"15min":9.3689999999999998},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.754,"5min":1.5960000000000001,"15min":1.6140000000000001},"min":{"1min":0.497,"5min":0.46000000000000002,"15min":0.33700000000000002},"max":{"1min":8.5739999999999998,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.83899999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4039999999999999,"5min":2.411,"15min":1.869},"min":{"1min":0.39600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":4.3319999999999999,"5min":62.850999999999999,"15min":62.850999999999999},"last":1.181},{"interface":"front","average":{"1min":1.6359999999999999,"5min":2.4860000000000002,"15min":2.085},"min":{"1min":0.52200000000000002,"5min":0.434,"15min":0.36699999999999999},"max":{"1min":6.907,"5min":62.816000000000003,"15min":62.816000000000003},"last":1.157}]},{"osd":7,"last update":"Mon Oct 9 09:05:35 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.6000000000000001,"15min":1.6599999999999999},"min":{"1min":0.54800000000000004,"5min":0.495,"15min":0.39400000000000002},"max":{"1min":6.9550000000000001,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.579,"15min":1.859},"min":{"1min":0.40999999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.3300000000000001,"5min":14.359,"15min":14.359},"last":1.0069999999999999}]}]},{"osd":2,"up_from":88,"seq":377957122285,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":513216,"kb_used_data":2872,"kb_used_omap":81,"kb_used_meta":510254,"kb_avail":93231936,"statfs":{"total":95995035648,"available":95469502464,"internally_reserved":0,"allocated":2940928,"data_stored":2076572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":522500314},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":5.3529999999999998,"5min":3.694,"15min":3.3170000000000002},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.191},"max":{"1min":68.457999999999998,"5min":68.457999999999998,"15min":134.458},"last":0.93000000000000005},{"interface":"front","average":{"1min":5.5579999999999998,"5min":3.5609999999999999,"15min":2.9430000000000001},"min":{"1min":0.40899999999999997,"5min":0.26000000000000001,"15min":0.221},"max":{"1min":68.162999999999997,"5min":68.162999999999997,"15min":138.417},"last":0.59099999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":2.2749999999999999,"15min":2.2999999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.21299999999999999},"max":{"1min":3.379,"5min":57.939999999999998,"15min":57.939999999999998},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.385,"5min":1.917,"15min":2.0880000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5519999999999996,"5min":56.667999999999999,"15min":57.122999999999998},"last":0.86599999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:05:53 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":4.2939999999999996,"15min":2.6850000000000001},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":6.665,"5min":112.47499999999999,"15min":112.47499999999999},"last":0.76700000000000002},{"interface":"front","average":{"1min":2.0049999999999999,"5min":4.718,"15min":3.048},"min":{"1min":0.39000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":17.274000000000001,"5min":112.929,"15min":112.929},"last":0.73199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9529999999999998,"5min":1.9350000000000001,"15min":1.966},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":17.408999999999999,"5min":17.408999999999999,"15min":17.408999999999999},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.784,"5min":1.764,"15min":1.845},"min":{"1min":0.45000000000000001,"5min":0.38800000000000001,"15min":0.314},"max":{"1min":6.2160000000000002,"5min":9.8439999999999994,"15min":14.071999999999999},"last":1.0589999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.3740000000000001,"15min":1.472},"min":{"1min":0.46500000000000002,"5min":0.41499999999999998,"15min":0.33900000000000002},"max":{"1min":5.5899999999999999,"5min":11.138,"15min":12.411},"last":1.133},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.925,"15min":1.96},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":17.375,"5min":17.375,"15min":17.375},"last":1.2350000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.7490000000000001,"15min":1.8660000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.33900000000000002},"max":{"1min":5.327,"5min":9.9139999999999997,"15min":10.456},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.5649999999999999,"15min":1.5960000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.39300000000000002},"max":{"1min":5.6230000000000002,"5min":11.869999999999999,"15min":12.381},"last":1.083}]},{"osd":7,"last update":"Mon Oct 9 09:05:37 2023","interfaces":[{"interface":"back","average":{"1min":5.0410000000000004,"5min":2.4430000000000001,"15min":2.0779999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.38800000000000001},"max":{"1min":67.491,"5min":67.491,"15min":67.491},"last":1.385},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.534,"15min":1.542},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.38500000000000001},"max":{"1min":5.6070000000000002,"5min":11.933,"15min":12.438000000000001},"last":1.1799999999999999}]}]},{"osd":3,"up_from":93,"seq":399431958760,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":446656,"kb_used_data":2232,"kb_used_omap":7,"kb_used_meta":444408,"kb_avail":93298496,"statfs":{"total":95995035648,"available":95537659904,"internally_reserved":0,"allocated":2285568,"data_stored":1474515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":455074408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":13.754,"5min":3.8029999999999999,"15min":2.4329999999999998},"min":{"1min":0.27300000000000002,"5min":0.19500000000000001,"15min":0.19400000000000001},"max":{"1min":233.22900000000001,"5min":233.22900000000001,"15min":233.22900000000001},"last":0.318},{"interface":"front","average":{"1min":14.622999999999999,"5min":4.1299999999999999,"15min":2.5659999999999998},"min":{"1min":0.29599999999999999,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":237.447,"5min":237.447,"15min":237.447},"last":0.56699999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":4.7190000000000003,"15min":2.8039999999999998},"min":{"1min":0.32100000000000001,"5min":0.31,"15min":0.22500000000000001},"max":{"1min":4.4279999999999999,"5min":326.75700000000001,"15min":326.75700000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.498,"5min":4.4400000000000004,"15min":2.6880000000000002},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":0.95699999999999996,"5min":327.024,"15min":327.024},"last":0.39700000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.4279999999999999,"15min":2.6070000000000002},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.255},"max":{"1min":5.7709999999999999,"5min":43.683999999999997,"15min":75.641999999999996},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.2869999999999999,"5min":2.0150000000000001,"15min":2.4060000000000001},"min":{"1min":0.40100000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2229999999999999,"5min":45.027999999999999,"15min":75.207999999999998},"last":0.443}]},{"osd":4,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":1.1850000000000001,"15min":1.474},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.03,"5min":8.9879999999999995,"15min":12.877000000000001},"last":0.746},{"interface":"front","average":{"1min":1.587,"5min":1.5760000000000001,"15min":1.7290000000000001},"min":{"1min":0.42199999999999999,"5min":0.36399999999999999,"15min":0.33500000000000002},"max":{"1min":7.4269999999999996,"5min":9.4649999999999999,"15min":10.486000000000001},"last":0.755}]},{"osd":5,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.4259999999999999,"15min":1.6240000000000001},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":7.4580000000000002,"5min":9.4990000000000006,"15min":10.577},"last":1.036},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.5780000000000001,"15min":1.736},"min":{"1min":0.45900000000000002,"5min":0.44900000000000001,"15min":0.34300000000000003},"max":{"1min":4.5309999999999997,"5min":7.5590000000000002,"15min":12.204000000000001},"last":0.82299999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":1.657,"15min":1.8160000000000001},"min":{"1min":0.46100000000000002,"5min":0.308,"15min":0.308},"max":{"1min":7.5300000000000002,"5min":9.1370000000000005,"15min":10.537000000000001},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.891,"15min":1.9079999999999999},"min":{"1min":0.49299999999999999,"5min":0.40699999999999997,"15min":0.35099999999999998},"max":{"1min":4.4710000000000001,"5min":9.5090000000000003,"15min":12.275},"last":0.93500000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:05:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.663,"15min":1.7589999999999999},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.41299999999999998},"max":{"1min":4.5490000000000004,"5min":7.5899999999999999,"15min":12.244},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.3700000000000001,"15min":1.6439999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.39000000000000001},"max":{"1min":3.714,"5min":9.1560000000000006,"15min":13.339},"last":0.78800000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019330,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101548,"kb_used_data":503652,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":92643604,"statfs":{"total":95995035648,"available":94867050496,"internally_reserved":0,"allocated":515739648,"data_stored":515239229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:06:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":2.9609999999999999,"15min":2.242},"min":{"1min":0.55200000000000005,"5min":0.248,"15min":0.248},"max":{"1min":1.159,"5min":141.07400000000001,"15min":141.07400000000001},"last":0.627},{"interface":"front","average":{"1min":0.76900000000000002,"5min":2.8540000000000001,"15min":2.2690000000000001},"min":{"1min":0.38100000000000001,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":1.139,"5min":141.15600000000001,"15min":141.15600000000001},"last":1.014}]},{"osd":1,"last update":"Mon Oct 9 09:05:48 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.72999999999999998,"15min":1.3740000000000001},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":1.571,"5min":2.8199999999999998,"15min":8.9169999999999998},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.82899999999999996,"5min":1.0209999999999999,"15min":1.464},"min":{"1min":0.318,"5min":0.318,"15min":0.29199999999999998},"max":{"1min":3.2759999999999998,"5min":7.6879999999999997,"15min":11.712999999999999},"last":0.85099999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:06:22 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":2.972,"15min":2.262},"min":{"1min":0.622,"5min":0.34899999999999998,"15min":0.30499999999999999},"max":{"1min":1.1259999999999999,"5min":142.816,"15min":142.816},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":2.6629999999999998,"15min":1.9410000000000001},"min":{"1min":0.434,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.165,"5min":143.81800000000001,"15min":143.81800000000001},"last":0.94799999999999995}]},{"osd":3,"last update":"Mon Oct 9 09:05:48 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":2.71,"15min":2.1459999999999999},"min":{"1min":0.41099999999999998,"5min":0.31900000000000001,"15min":0.29499999999999998},"max":{"1min":6.1159999999999997,"5min":143.13399999999999,"15min":143.13399999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.472,"5min":3.2519999999999998,"15min":2.177},"min":{"1min":0.46400000000000002,"5min":0.41299999999999998,"15min":0.35399999999999998},"max":{"1min":6.6230000000000002,"5min":152.37299999999999,"15min":152.37299999999999},"last":0.88}]},{"osd":5,"last update":"Mon Oct 9 09:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.083,"15min":1.238},"min":{"1min":0.51500000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.3639999999999999,"5min":7.7439999999999998,"15min":11.762},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.95099999999999996,"15min":1.29},"min":{"1min":0.46200000000000002,"5min":0.37,"15min":0.28699999999999998},"max":{"1min":1.2849999999999999,"5min":4.1379999999999999,"15min":12.85},"last":1.069}]},{"osd":6,"last update":"Mon Oct 9 09:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.91000000000000003,"15min":1.3939999999999999},"min":{"1min":0.40600000000000003,"5min":0.35699999999999998,"15min":0.32100000000000001},"max":{"1min":2.02,"5min":5.1020000000000003,"15min":10.657999999999999},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.123,"5min":1.137,"15min":1.448},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.4039999999999999,"5min":7.6470000000000002,"15min":14.755000000000001},"last":1}]},{"osd":7,"last update":"Mon Oct 9 09:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":0.97899999999999998,"15min":1.4139999999999999},"min":{"1min":0.50800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.4490000000000001,"5min":3.9039999999999999,"15min":13.933999999999999},"last":1.139},{"interface":"front","average":{"1min":1.083,"5min":0.96499999999999997,"15min":1.5669999999999999},"min":{"1min":0.68600000000000005,"5min":0.36199999999999999,"15min":0.33200000000000002},"max":{"1min":1.389,"5min":5.1520000000000001,"15min":13.911},"last":1.04}]}]},{"osd":5,"up_from":0,"seq":154618823098,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:06:31.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T09:06:31.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:31.246 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:31.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:31.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:31.349 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:31.382 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:31.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:31.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:31.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:31.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:31.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:31.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:31.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:31.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:31.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:31.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:31.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:31.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:31.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:31.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:31.918 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:31.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:31.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:32.022 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:32.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:32.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:32.155 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:32.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:32.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:32.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:32.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:32.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:32.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:32.533 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:32.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:32.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:32.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:32.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:32.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:32.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:32.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:32.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:32.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:32.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:32.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:32.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:32.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:33.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:33.055 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:32 smithi094 bash[59677]: cluster 2023-10-09T09:06:31.909523+0000 mgr.y (mgr.25421) 873 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:33.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:33.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:33.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:33.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:06:33.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:06:33.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:33.188 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:32 smithi016 bash[75179]: cluster 2023-10-09T09:06:31.909523+0000 mgr.y (mgr.25421) 873 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:33.188 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:32 smithi016 bash[77892]: cluster 2023-10-09T09:06:31.909523+0000 mgr.y (mgr.25421) 873 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:33.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:33.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:33.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:33.337 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:33.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:33.441 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:33.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:33.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:33.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:33.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:33.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:33.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:33.716 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:33.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:33.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:33.811 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:33.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:33.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:33.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:33.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:34.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:34.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:34.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:34.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:34.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:34.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:34.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:34.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:34.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:34.516 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:34.549 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:34.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:34.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:34.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:34.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:34.784 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:34.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:34.827 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:34.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:34.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:34.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:34.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:35.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:35.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:35.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:35.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:35.189 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:34 smithi094 bash[59677]: cluster 2023-10-09T09:06:33.910518+0000 mgr.y (mgr.25421) 874 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:35.189 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:34 smithi094 bash[59677]: audit 2023-10-09T09:06:34.236187+0000 mon.a (mon.0) 1182 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:35.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:35.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:35.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:35 smithi016 bash[75179]: cluster 2023-10-09T09:06:33.910518+0000 mgr.y (mgr.25421) 874 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:35.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:35 smithi016 bash[75179]: audit 2023-10-09T09:06:34.236187+0000 mon.a (mon.0) 1182 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:35.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:35 smithi016 bash[77892]: cluster 2023-10-09T09:06:33.910518+0000 mgr.y (mgr.25421) 874 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:35.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:35 smithi016 bash[77892]: audit 2023-10-09T09:06:34.236187+0000 mon.a (mon.0) 1182 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:35.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:35.347 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:35.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:35.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:35.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:35.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:35.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:35.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:35.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:35.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:35.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:35.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:35.845 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:35.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:35.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:35.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:35.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:35.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:35.989 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:06:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:06:35] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-10-09T09:06:36.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:36.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:36.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:36.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:36.279 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:36.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:36.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:36.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:36.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:36.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:36.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:36.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:36.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:36.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:36.803 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:36.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:36.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:36.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:36.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:37.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:37.045 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:36 smithi094 bash[59677]: cluster 2023-10-09T09:06:35.911553+0000 mgr.y (mgr.25421) 875 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:37.063 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:37.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:37.089 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:36 smithi016 bash[75179]: cluster 2023-10-09T09:06:35.911553+0000 mgr.y (mgr.25421) 875 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:37.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:37.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:37.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:37.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:37.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:37.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:37.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:37.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:37.451 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:37 smithi016 bash[77892]: cluster 2023-10-09T09:06:35.911553+0000 mgr.y (mgr.25421) 875 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:37.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:37.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:37.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:37.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:37.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:37.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:37.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:37.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:37.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:37.841 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:37.871 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:37.872 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:06:37.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:37.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:37.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:38.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:38.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:38.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:38.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:38.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:38.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:38.378 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:38.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:38.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:38.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:38.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:38.675 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:38.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:38.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:38.812 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:38.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:38.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:38.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:38.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:38.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:39.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:39.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:39.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:39.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:39.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:38 smithi016 bash[75179]: cluster 2023-10-09T09:06:37.912224+0000 mgr.y (mgr.25421) 876 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:39.180 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:38 smithi016 bash[77892]: cluster 2023-10-09T09:06:37.912224+0000 mgr.y (mgr.25421) 876 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:39.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:38 smithi094 bash[59677]: cluster 2023-10-09T09:06:37.912224+0000 mgr.y (mgr.25421) 876 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:39.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:39.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:39.427 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:39.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:39.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:39.597 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:39.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:39.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:39.722 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:39.826 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:39.855 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:39.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:39.992 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:39.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:39.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:40.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:40.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:40.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:40.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:40.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:40.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:40.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:40.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:40.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:40.482 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:40.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:40.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:40.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:40.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:40.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:40.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:40.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:40.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:40.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:40.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:40.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:40.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:40.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:41.011 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:41.045 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:41.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:41.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:41.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:40 smithi094 bash[59677]: cluster 2023-10-09T09:06:39.912876+0000 mgr.y (mgr.25421) 877 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:41.184 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:40 smithi094 bash[59677]: audit 2023-10-09T09:06:39.919417+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:41.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:41.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:40 smithi016 bash[77892]: cluster 2023-10-09T09:06:39.912876+0000 mgr.y (mgr.25421) 877 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:41.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:40 smithi016 bash[77892]: audit 2023-10-09T09:06:39.919417+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:41.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:40 smithi016 bash[75179]: cluster 2023-10-09T09:06:39.912876+0000 mgr.y (mgr.25421) 877 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:41.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:40 smithi016 bash[75179]: audit 2023-10-09T09:06:39.919417+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:06:41.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:41.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:41.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:41.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:41.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:41.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:41.573 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:41.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:41.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:41.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:41.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:41.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:41.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:41.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:42.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:42.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:42.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:42.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:42.342 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:42.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:42.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:42.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:42.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:42.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:42.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:42.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:42.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:42.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.773 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:42.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:42.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:42.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:42.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:42.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:43.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:43.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:43.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:43.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:43.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:43.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:43.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:42 smithi094 bash[59677]: cluster 2023-10-09T09:06:41.914227+0000 mgr.y (mgr.25421) 878 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:43.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:42 smithi016 bash[75179]: cluster 2023-10-09T09:06:41.914227+0000 mgr.y (mgr.25421) 878 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:43.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:42 smithi016 bash[77892]: cluster 2023-10-09T09:06:41.914227+0000 mgr.y (mgr.25421) 878 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:43.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:43.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:43.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:43.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:43.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:43.529 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:43.558 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:43.662 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:43.693 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:43.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:43.828 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:43.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:43.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:43.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:43.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:44.069 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:44.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:44.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:44.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:44.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:44.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:44.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:44.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:44.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:44.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:44.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:44.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:44.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:44.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:44.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:44.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:44.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:44.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:44.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:44.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:44.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:44.891 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:44.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:44.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:45.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:45.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:45.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:45.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:45.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:45.260 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:45.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:45.394 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:45.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:45.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:45.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:45.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:45.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:45.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:45.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:45.799 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:45 smithi016 bash[75179]: cluster 2023-10-09T09:06:43.915133+0000 mgr.y (mgr.25421) 879 : cluster 0 pgmap v920: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:45.799 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:06:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:06:45] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-10-09T09:06:45.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:45 smithi016 bash[77892]: cluster 2023-10-09T09:06:43.915133+0000 mgr.y (mgr.25421) 879 : cluster 0 pgmap v920: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:45.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:45.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:45.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:45.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:45.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:45.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:45 smithi094 bash[59677]: cluster 2023-10-09T09:06:43.915133+0000 mgr.y (mgr.25421) 879 : cluster 0 pgmap v920: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:46.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:46.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:46.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:46.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:46.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:46.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:46.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:46.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:46.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:46.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:46.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:46.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:46.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:46.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:46.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:46.601 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:46.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:46.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:46.705 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:46.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:46.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:46.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:46.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:46.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:46.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:47.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:47.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:47.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:47.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:47.241 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:47.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:47.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:47.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:47.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:47.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:47.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:47.582 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:06:47.583 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:06:47.583 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-10-09T09:06:47.583 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:06:47.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:47.666 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:47.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:47.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:47 smithi016 bash[77892]: cluster 2023-10-09T09:06:45.915912+0000 mgr.y (mgr.25421) 880 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:47 smithi016 bash[75179]: cluster 2023-10-09T09:06:45.915912+0000 mgr.y (mgr.25421) 880 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:47.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:47.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:47 smithi094 bash[59677]: cluster 2023-10-09T09:06:45.915912+0000 mgr.y (mgr.25421) 880 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:47.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:47.905 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:47.938 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:47.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:47.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:48.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.076 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:48.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:48.180 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.214 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:48.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:48.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:48.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:48.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:48.443 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:48.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:48.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:48.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:48.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:48.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:48.710 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:48.723 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:48 smithi094 bash[59677]: cluster 2023-10-09T09:06:47.916547+0000 mgr.y (mgr.25421) 881 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:48.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:48.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:48.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:48.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:48.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:49.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:49.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:48 smithi016 bash[77892]: cluster 2023-10-09T09:06:47.916547+0000 mgr.y (mgr.25421) 881 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:49.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:48 smithi016 bash[75179]: cluster 2023-10-09T09:06:47.916547+0000 mgr.y (mgr.25421) 881 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:49.113 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:49.145 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:49.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:49.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:49.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:49.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:49.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:49.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:49.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:49.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:49.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:49.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[77892]: audit 2023-10-09T09:06:49.237338+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:49.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[77892]: audit 2023-10-09T09:06:49.552725+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-10-09T09:06:49.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[77892]: audit 2023-10-09T09:06:49.553123+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-10-09T09:06:49.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[77892]: audit 2023-10-09T09:06:49.553445+0000 mon.a (mon.0) 1187 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-09T09:06:49.797 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[75179]: audit 2023-10-09T09:06:49.237338+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:49.797 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[75179]: audit 2023-10-09T09:06:49.552725+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-10-09T09:06:49.798 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[75179]: audit 2023-10-09T09:06:49.553123+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-10-09T09:06:49.798 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:49 smithi016 bash[75179]: audit 2023-10-09T09:06:49.553445+0000 mon.a (mon.0) 1187 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-09T09:06:49.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:49.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:49.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:49.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:49.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:49.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:49.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:49 smithi094 bash[59677]: audit 2023-10-09T09:06:49.237338+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:49.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:49 smithi094 bash[59677]: audit 2023-10-09T09:06:49.552725+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-10-09T09:06:49.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:49 smithi094 bash[59677]: audit 2023-10-09T09:06:49.553123+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-10-09T09:06:49.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:49 smithi094 bash[59677]: audit 2023-10-09T09:06:49.553445+0000 mon.a (mon.0) 1187 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-09T09:06:50.041 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:50.069 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:50.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:50.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:50.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:50.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:50.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:50.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:50.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:50.550 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:50.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:50.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:50.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:50.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:50.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:50.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[75179]: audit 2023-10-09T09:06:49.659978+0000 mon.a (mon.0) 1188 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-10-09T09:06:50.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[75179]: audit 2023-10-09T09:06:49.660069+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-10-09T09:06:50.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[75179]: audit 2023-10-09T09:06:49.660185+0000 mon.a (mon.0) 1190 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-09T09:06:50.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[75179]: cluster 2023-10-09T09:06:49.660265+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e215: 8 total, 8 up, 5 in 2023-10-09T09:06:50.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[75179]: cluster 2023-10-09T09:06:49.661477+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:50.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[75179]: cluster 2023-10-09T09:06:49.663714+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:50.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[75179]: cluster 2023-10-09T09:06:49.916964+0000 mgr.y (mgr.25421) 882 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:50.808 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:50.822 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:50 smithi094 bash[59677]: audit 2023-10-09T09:06:49.659978+0000 mon.a (mon.0) 1188 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-10-09T09:06:50.822 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:50 smithi094 bash[59677]: audit 2023-10-09T09:06:49.660069+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-10-09T09:06:50.822 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:50 smithi094 bash[59677]: audit 2023-10-09T09:06:49.660185+0000 mon.a (mon.0) 1190 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-09T09:06:50.823 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:50 smithi094 bash[59677]: cluster 2023-10-09T09:06:49.660265+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e215: 8 total, 8 up, 5 in 2023-10-09T09:06:50.823 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:50 smithi094 bash[59677]: cluster 2023-10-09T09:06:49.661477+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:50.823 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:50 smithi094 bash[59677]: cluster 2023-10-09T09:06:49.663714+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:50.823 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:50 smithi094 bash[59677]: cluster 2023-10-09T09:06:49.916964+0000 mgr.y (mgr.25421) 882 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:50.841 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:50.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:50.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:50.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:50.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[77892]: audit 2023-10-09T09:06:49.659978+0000 mon.a (mon.0) 1188 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-10-09T09:06:50.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[77892]: audit 2023-10-09T09:06:49.660069+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-10-09T09:06:51.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[77892]: audit 2023-10-09T09:06:49.660185+0000 mon.a (mon.0) 1190 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-09T09:06:51.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[77892]: cluster 2023-10-09T09:06:49.660265+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e215: 8 total, 8 up, 5 in 2023-10-09T09:06:51.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[77892]: cluster 2023-10-09T09:06:49.661477+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[77892]: cluster 2023-10-09T09:06:49.663714+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:50 smithi016 bash[77892]: cluster 2023-10-09T09:06:49.916964+0000 mgr.y (mgr.25421) 882 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:51.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:51.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:51.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:51.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:51.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:51.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:51.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:51.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:51.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:51.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:51.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:51.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:51.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:51.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:51.600 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:51.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:51.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:51.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:51.734 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:51.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[75179]: cluster 2023-10-09T09:06:50.002931+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[75179]: cluster 2023-10-09T09:06:50.003031+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[75179]: cluster 2023-10-09T09:06:50.664155+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[75179]: cluster 2023-10-09T09:06:50.668185+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e216: 8 total, 8 up, 5 in 2023-10-09T09:06:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[75179]: cluster 2023-10-09T09:06:50.669220+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[75179]: cluster 2023-10-09T09:06:50.671131+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[75179]: cluster 2023-10-09T09:06:50.672088+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[77892]: cluster 2023-10-09T09:06:50.002931+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[77892]: cluster 2023-10-09T09:06:50.003031+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[77892]: cluster 2023-10-09T09:06:50.664155+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[77892]: cluster 2023-10-09T09:06:50.668185+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e216: 8 total, 8 up, 5 in 2023-10-09T09:06:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[77892]: cluster 2023-10-09T09:06:50.669220+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[77892]: cluster 2023-10-09T09:06:50.671131+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:51 smithi016 bash[77892]: cluster 2023-10-09T09:06:50.672088+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:51.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:51.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:51.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:51 smithi094 bash[59677]: cluster 2023-10-09T09:06:50.002931+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:51 smithi094 bash[59677]: cluster 2023-10-09T09:06:50.003031+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T09:06:51.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:51 smithi094 bash[59677]: cluster 2023-10-09T09:06:50.664155+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:51 smithi094 bash[59677]: cluster 2023-10-09T09:06:50.668185+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e216: 8 total, 8 up, 5 in 2023-10-09T09:06:51.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:51 smithi094 bash[59677]: cluster 2023-10-09T09:06:50.669220+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:51 smithi094 bash[59677]: cluster 2023-10-09T09:06:50.671131+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:51 smithi094 bash[59677]: cluster 2023-10-09T09:06:50.672088+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:51.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:51.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:52.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:52.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:52.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:52.160 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:52.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:52.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:52.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:52.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:52.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:52.531 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:52.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:52.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:52.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:52.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:52 smithi016 bash[77892]: cluster 2023-10-09T09:06:51.103764+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:52.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:52 smithi016 bash[77892]: audit 2023-10-09T09:06:51.874780+0000 mon.a (mon.0) 1193 : audit 1 from='client.? 172.21.15.16:0/1038611236' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:52 smithi016 bash[77892]: cluster 2023-10-09T09:06:51.917512+0000 mgr.y (mgr.25421) 883 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:52.688 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T09:06:52.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:52.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:52.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:52.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:52.945 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:52 smithi016 bash[75179]: cluster 2023-10-09T09:06:51.103764+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:52.945 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:52 smithi016 bash[75179]: audit 2023-10-09T09:06:51.874780+0000 mon.a (mon.0) 1193 : audit 1 from='client.? 172.21.15.16:0/1038611236' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.945 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:52 smithi016 bash[75179]: cluster 2023-10-09T09:06:51.917512+0000 mgr.y (mgr.25421) 883 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:52.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:52.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:52.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:52.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:52 smithi094 bash[59677]: cluster 2023-10-09T09:06:51.103764+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T09:06:52.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:52 smithi094 bash[59677]: audit 2023-10-09T09:06:51.874780+0000 mon.a (mon.0) 1193 : audit 1 from='client.? 172.21.15.16:0/1038611236' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:52 smithi094 bash[59677]: cluster 2023-10-09T09:06:51.917512+0000 mgr.y (mgr.25421) 883 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:53.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:53.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:53.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:53.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:53.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:53.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:53.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:53.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:53.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:53.350 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:53.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:53.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:53.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:53.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:53.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:53.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:53.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:06:53.579 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:53.613 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:53.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:53.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:53.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:53.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[75179]: cluster 2023-10-09T09:06:52.679943+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:53.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[75179]: cluster 2023-10-09T09:06:52.680062+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:53.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[75179]: audit 2023-10-09T09:06:52.682565+0000 mon.a (mon.0) 1194 : audit 1 from='client.? 172.21.15.16:0/1038611236' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:53.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[75179]: cluster 2023-10-09T09:06:52.682659+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e217: 8 total, 8 up, 5 in 2023-10-09T09:06:53.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[77892]: cluster 2023-10-09T09:06:52.679943+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:53.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[77892]: cluster 2023-10-09T09:06:52.680062+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:53.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[77892]: audit 2023-10-09T09:06:52.682565+0000 mon.a (mon.0) 1194 : audit 1 from='client.? 172.21.15.16:0/1038611236' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:53.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:53 smithi016 bash[77892]: cluster 2023-10-09T09:06:52.682659+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e217: 8 total, 8 up, 5 in 2023-10-09T09:06:53.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:53.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:53.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:53.841 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:53.854 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:53 smithi094 bash[59677]: cluster 2023-10-09T09:06:52.679943+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:53.854 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:53 smithi094 bash[59677]: cluster 2023-10-09T09:06:52.680062+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:53.854 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:53 smithi094 bash[59677]: audit 2023-10-09T09:06:52.682565+0000 mon.a (mon.0) 1194 : audit 1 from='client.? 172.21.15.16:0/1038611236' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:53.855 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:53 smithi094 bash[59677]: cluster 2023-10-09T09:06:52.682659+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e217: 8 total, 8 up, 5 in 2023-10-09T09:06:53.877 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:53.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:53.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:53.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:54.101 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:54.125 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:06:54.125 INFO:teuthology.orchestra.run.smithi016.stdout:{"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1369,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":217,"num_osds":8,"num_up_osds":8,"osd_up_since":1696841235,"num_in_osds":5,"osd_in_since":1696842121,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":24,"data_bytes":595872,"bytes_used":3423547392,"bytes_avail":476551630848,"bytes_total":479975178240},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi016.ccdktz","status":"up:active","gid":14634}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"}},"servicemap":{"epoch":30,"modified":"2023-10-09T09:03:15.825457+0000","services":{}},"progress_events":{"8402cf81-b0a2-4ab7-aed5-036e29e682ad":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 7m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T09:06:54.135 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:54.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:54.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:54.373 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:54.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:54.508 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:54.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:54.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:54.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:54.675 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:54.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:54.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:54.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:54.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:54.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:54.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:54.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:54 smithi094 bash[59677]: cluster 2023-10-09T09:06:53.918288+0000 mgr.y (mgr.25421) 884 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:54.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:54 smithi094 bash[59677]: audit 2023-10-09T09:06:54.124499+0000 mon.a (mon.0) 1196 : audit 0 from='client.? 172.21.15.16:0/2602504868' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:06:54.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:54 smithi016 bash[75179]: cluster 2023-10-09T09:06:53.918288+0000 mgr.y (mgr.25421) 884 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:55.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:54 smithi016 bash[75179]: audit 2023-10-09T09:06:54.124499+0000 mon.a (mon.0) 1196 : audit 0 from='client.? 172.21.15.16:0/2602504868' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:06:55.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:54 smithi016 bash[77892]: cluster 2023-10-09T09:06:53.918288+0000 mgr.y (mgr.25421) 884 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:55.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:54 smithi016 bash[77892]: audit 2023-10-09T09:06:54.124499+0000 mon.a (mon.0) 1196 : audit 0 from='client.? 172.21.15.16:0/2602504868' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:06:55.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:55.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:55.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:55.170 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:55.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:55.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:55.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:55.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:55.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:55.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:55.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:55.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:55.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:55.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:55.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:55.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:55.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:55.824 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:55 smithi016 bash[75179]: cluster 2023-10-09T09:06:54.430533+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:55.824 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:06:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:06:55] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.33.4" 2023-10-09T09:06:55.824 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:55 smithi016 bash[77892]: cluster 2023-10-09T09:06:54.430533+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:55.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:55.941 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:55.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:55 smithi094 bash[59677]: cluster 2023-10-09T09:06:54.430533+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T09:06:55.970 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:55.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:56.073 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:56.100 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:56.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:56.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:56.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:56.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:56.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:56.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:56.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:56.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:56.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:56.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:56.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:56.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:56.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:56.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:56.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:56.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:56.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:56.732 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:56.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:56 smithi016 bash[75179]: cluster 2023-10-09T09:06:55.919096+0000 mgr.y (mgr.25421) 885 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:56 smithi016 bash[77892]: cluster 2023-10-09T09:06:55.919096+0000 mgr.y (mgr.25421) 885 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:56.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:56.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:56.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:56.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:56.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:56.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:56.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:56 smithi094 bash[59677]: cluster 2023-10-09T09:06:55.919096+0000 mgr.y (mgr.25421) 885 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 444 GiB / 447 GiB avail 2023-10-09T09:06:56.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:57.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:57.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:57.112 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T09:06:57.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:06:57.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:57.160 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:57.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:57.295 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T09:06:57.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:06:57.298 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:06:57.298 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:06:57.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:06:57.390 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-10-09T09:06:57.390 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:06:57.390 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_172771 2023-10-09T09:06:57.390 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:06:57.390 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:06:57.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:57.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:57.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:57.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:57.582 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:06:57.620 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:06:57.659 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:06:57.668 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:57.696 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:57.716 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:06:57.750 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:06:57.799 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:06:57.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:57.808 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:57 smithi016 bash[75179]: audit 2023-10-09T09:06:57.538573+0000 mon.a (mon.0) 1197 : audit 0 from='client.? 172.21.15.16:0/1146180719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:06:57.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:57 smithi016 bash[77892]: audit 2023-10-09T09:06:57.538573+0000 mon.a (mon.0) 1197 : audit 0 from='client.? 172.21.15.16:0/1146180719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:06:57.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:06:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:06:57.830 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:06:57.871 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:06:57.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:57.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:57.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:57.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:57 smithi094 bash[59677]: audit 2023-10-09T09:06:57.538573+0000 mon.a (mon.0) 1197 : audit 0 from='client.? 172.21.15.16:0/1146180719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:06:58.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:58.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:58.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:58.194 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:58.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:06:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:58.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:06:58.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:58.358 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:58.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:58.390 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2048 2032 126.993 127 0.00721321 0.00782987 2023-10-09T09:06:58.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:58.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:58.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:58.591 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:58.617 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:58.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:58.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:58.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:58.735 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:06:58.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:58.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:58 smithi016 bash[75179]: cluster 2023-10-09T09:06:57.919752+0000 mgr.y (mgr.25421) 886 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 6.3 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 728 KiB/s wr, 11 op/s 2023-10-09T09:06:58.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:58 smithi016 bash[75179]: audit 2023-10-09T09:06:58.033563+0000 mon.c (mon.2) 86 : audit 1 from='client.? 172.21.15.16:0/737683703' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:58.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:58 smithi016 bash[75179]: audit 2023-10-09T09:06:58.034510+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:58 smithi016 bash[77892]: cluster 2023-10-09T09:06:57.919752+0000 mgr.y (mgr.25421) 886 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 6.3 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 728 KiB/s wr, 11 op/s 2023-10-09T09:06:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:58 smithi016 bash[77892]: audit 2023-10-09T09:06:58.033563+0000 mon.c (mon.2) 86 : audit 1 from='client.? 172.21.15.16:0/737683703' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:58 smithi016 bash[77892]: audit 2023-10-09T09:06:58.034510+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:58.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:58.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:58.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:06:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:58.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:06:58.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:58 smithi094 bash[59677]: cluster 2023-10-09T09:06:57.919752+0000 mgr.y (mgr.25421) 886 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 6.3 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 728 KiB/s wr, 11 op/s 2023-10-09T09:06:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:58 smithi094 bash[59677]: audit 2023-10-09T09:06:58.033563+0000 mon.c (mon.2) 86 : audit 1 from='client.? 172.21.15.16:0/737683703' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:58.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:58 smithi094 bash[59677]: audit 2023-10-09T09:06:58.034510+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:06:59.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:59.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:59.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:59.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:59.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:59.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:59.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:59.390 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 4541 4525 141.396 155.812 0.00570643 0.0070574 2023-10-09T09:06:59.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:06:59.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:59.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:06:59.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:06:59.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:59.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:06:59.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:06:59.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:59.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[75179]: cluster 2023-10-09T09:06:58.705385+0000 mon.a (mon.0) 1199 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:06:59.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[75179]: cluster 2023-10-09T09:06:58.705430+0000 mon.a (mon.0) 1200 : cluster 1 Cluster is now healthy 2023-10-09T09:06:59.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[75179]: audit 2023-10-09T09:06:58.722636+0000 mon.a (mon.0) 1201 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:06:59.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[75179]: cluster 2023-10-09T09:06:58.723046+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T09:06:59.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[75179]: cluster 2023-10-09T09:06:58.723085+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e218: 8 total, 8 up, 5 in 2023-10-09T09:06:59.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[75179]: cluster 2023-10-09T09:06:58.724366+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T09:06:59.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[75179]: audit 2023-10-09T09:06:59.588137+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:59.778 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[77892]: cluster 2023-10-09T09:06:58.705385+0000 mon.a (mon.0) 1199 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:06:59.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[77892]: cluster 2023-10-09T09:06:58.705430+0000 mon.a (mon.0) 1200 : cluster 1 Cluster is now healthy 2023-10-09T09:06:59.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[77892]: audit 2023-10-09T09:06:58.722636+0000 mon.a (mon.0) 1201 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:06:59.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[77892]: cluster 2023-10-09T09:06:58.723046+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T09:06:59.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[77892]: cluster 2023-10-09T09:06:58.723085+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e218: 8 total, 8 up, 5 in 2023-10-09T09:06:59.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[77892]: cluster 2023-10-09T09:06:58.724366+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T09:06:59.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:06:59 smithi016 bash[77892]: audit 2023-10-09T09:06:59.588137+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:59.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:06:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:06:59.901 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:59.909 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:59 smithi094 bash[59677]: cluster 2023-10-09T09:06:58.705385+0000 mon.a (mon.0) 1199 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:06:59.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:59 smithi094 bash[59677]: cluster 2023-10-09T09:06:58.705430+0000 mon.a (mon.0) 1200 : cluster 1 Cluster is now healthy 2023-10-09T09:06:59.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:59 smithi094 bash[59677]: audit 2023-10-09T09:06:58.722636+0000 mon.a (mon.0) 1201 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:06:59.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:59 smithi094 bash[59677]: cluster 2023-10-09T09:06:58.723046+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T09:06:59.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:59 smithi094 bash[59677]: cluster 2023-10-09T09:06:58.723085+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e218: 8 total, 8 up, 5 in 2023-10-09T09:06:59.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:59 smithi094 bash[59677]: cluster 2023-10-09T09:06:58.724366+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T09:06:59.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:06:59 smithi094 bash[59677]: audit 2023-10-09T09:06:59.588137+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:59.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:06:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:06:59.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:00.029 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:00.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:00.158 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:00.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:00.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:00.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:00.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:00.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 6319 6303 131.302 111.125 0.00382458 0.00760032 2023-10-09T09:07:00.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:00.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:00.553 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:00.580 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:00.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:00.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:00.809 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:00.836 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:00.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:00.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:00.967 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:00.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:00.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:01.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:01.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:01.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:01.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:01.211 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:01 smithi094 bash[59677]: cluster 2023-10-09T09:06:59.920302+0000 mgr.y (mgr.25421) 887 : cluster 0 pgmap v932: 89 pgs: 89 active+clean; 13 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T09:07:01.211 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:01 smithi094 bash[59677]: audit 2023-10-09T09:07:00.063116+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:01.212 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:01 smithi094 bash[59677]: audit 2023-10-09T09:07:00.070827+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:01.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:01.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:01.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:01.341 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:01 smithi016 bash[75179]: cluster 2023-10-09T09:06:59.920302+0000 mgr.y (mgr.25421) 887 : cluster 0 pgmap v932: 89 pgs: 89 active+clean; 13 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T09:07:01.341 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:01 smithi016 bash[75179]: audit 2023-10-09T09:07:00.063116+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:01.342 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:01 smithi016 bash[75179]: audit 2023-10-09T09:07:00.070827+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:01.342 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:01 smithi016 bash[77892]: cluster 2023-10-09T09:06:59.920302+0000 mgr.y (mgr.25421) 887 : cluster 0 pgmap v932: 89 pgs: 89 active+clean; 13 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T09:07:01.342 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:01 smithi016 bash[77892]: audit 2023-10-09T09:07:00.063116+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:01.342 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:01 smithi016 bash[77892]: audit 2023-10-09T09:07:00.070827+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:01.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:01.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:01.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:01.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 8499 8483 132.536 136.25 0.00587771 0.007533 2023-10-09T09:07:01.462 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:01.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:01.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:01.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:01.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:01.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:01.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:01.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:01.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:01.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:01.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:01.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:01.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:01.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:01.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:02.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:02.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:02.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:02.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:02.154 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:02.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:02.282 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:02.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:07:02.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:02.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 10542 10526 131.564 127.688 0.00717647 0.00759027 2023-10-09T09:07:02.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:02.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:02.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:02.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:02.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:02.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:02.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:02.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:02.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:02.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:02.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:02.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:02.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:02.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:03.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:03.091 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:03 smithi016 bash[77892]: cluster 2023-10-09T09:07:01.921094+0000 mgr.y (mgr.25421) 888 : cluster 0 pgmap v933: 89 pgs: 89 active+clean; 348 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 38 MiB/s wr, 601 op/s 2023-10-09T09:07:03.092 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:03 smithi016 bash[75179]: cluster 2023-10-09T09:07:01.921094+0000 mgr.y (mgr.25421) 888 : cluster 0 pgmap v933: 89 pgs: 89 active+clean; 348 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 38 MiB/s wr, 601 op/s 2023-10-09T09:07:03.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:03.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:03.216 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:03.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:03.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:03.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:03.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:03.366 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:03 smithi094 bash[59677]: cluster 2023-10-09T09:07:01.921094+0000 mgr.y (mgr.25421) 888 : cluster 0 pgmap v933: 89 pgs: 89 active+clean; 348 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 38 MiB/s wr, 601 op/s 2023-10-09T09:07:03.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:03.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:03.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:03.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 12310 12294 128.051 110.5 0.0701755 0.0078006 2023-10-09T09:07:03.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:03.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:03.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:03.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:03.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:03.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:03.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:03.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:03.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:07:03.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:03.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:03.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:03.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:03.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:03.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:03.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:03.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:04.030 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:04.060 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:04.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:04.190 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:04.200 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:04.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:04.323 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:04.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 13844 13828 123.453 95.875 0.00543446 0.00791196 2023-10-09T09:07:04.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:04.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:04.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:04.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:04.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:04.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:04.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:04.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:04.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:04.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:04.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:04.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:04.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:05.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:05.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:05.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:05.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:05.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:05 smithi016 bash[75179]: cluster 2023-10-09T09:07:03.921736+0000 mgr.y (mgr.25421) 889 : cluster 0 pgmap v934: 89 pgs: 89 active+clean; 510 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:07:05.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:05 smithi016 bash[75179]: audit 2023-10-09T09:07:04.244426+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:05.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:05 smithi016 bash[75179]: audit 2023-10-09T09:07:04.245574+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:05.258 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:05.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:05.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:05.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:05.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 14861 14845 115.965 63.5625 0.00819267 0.00861191 2023-10-09T09:07:05.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:05.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:05.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:05.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:05 smithi094 bash[59677]: cluster 2023-10-09T09:07:03.921736+0000 mgr.y (mgr.25421) 889 : cluster 0 pgmap v934: 89 pgs: 89 active+clean; 510 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:07:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:05 smithi094 bash[59677]: audit 2023-10-09T09:07:04.244426+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:05.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:05 smithi094 bash[59677]: audit 2023-10-09T09:07:04.245574+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:05.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:05.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:05 smithi016 bash[77892]: cluster 2023-10-09T09:07:03.921736+0000 mgr.y (mgr.25421) 889 : cluster 0 pgmap v934: 89 pgs: 89 active+clean; 510 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:07:05.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:05 smithi016 bash[77892]: audit 2023-10-09T09:07:04.244426+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:05.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:05 smithi016 bash[77892]: audit 2023-10-09T09:07:04.245574+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:05.526 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:05.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:05.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:05.656 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:05.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:05.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:05.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:05.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:05.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:05.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:05.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:05.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:07:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:07:05] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.33.4" 2023-10-09T09:07:06.028 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:06.070 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:06.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:06.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:06.230 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:06.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:06.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:06.238 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:06.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:06.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:06.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:06.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 16344 16328 113.378 92.6875 0.00222184 0.00873699 2023-10-09T09:07:06.472 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:06.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:06.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:06.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:06.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:06.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:06.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:06.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:06.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:06.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:06.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:06.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:06.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:06.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:06.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:07.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:07.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:07.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:07.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:07.163 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:07.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:07.293 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:07.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 17630 17614 110.077 80.375 0.011454 0.00906919 2023-10-09T09:07:07.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:07.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:07.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.664 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[75179]: cluster 2023-10-09T09:07:05.922368+0000 mgr.y (mgr.25421) 890 : cluster 0 pgmap v935: 89 pgs: 89 active+clean; 875 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:07:07.664 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[75179]: audit 2023-10-09T09:07:06.420752+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[75179]: audit 2023-10-09T09:07:06.426256+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[75179]: audit 2023-10-09T09:07:06.841736+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[75179]: audit 2023-10-09T09:07:06.850970+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[77892]: cluster 2023-10-09T09:07:05.922368+0000 mgr.y (mgr.25421) 890 : cluster 0 pgmap v935: 89 pgs: 89 active+clean; 875 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[77892]: audit 2023-10-09T09:07:06.420752+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[77892]: audit 2023-10-09T09:07:06.426256+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[77892]: audit 2023-10-09T09:07:06.841736+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:07 smithi016 bash[77892]: audit 2023-10-09T09:07:06.850970+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:07 smithi094 bash[59677]: cluster 2023-10-09T09:07:05.922368+0000 mgr.y (mgr.25421) 890 : cluster 0 pgmap v935: 89 pgs: 89 active+clean; 875 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:07:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:07 smithi094 bash[59677]: audit 2023-10-09T09:07:06.420752+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:07 smithi094 bash[59677]: audit 2023-10-09T09:07:06.426256+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:07 smithi094 bash[59677]: audit 2023-10-09T09:07:06.841736+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:07 smithi094 bash[59677]: audit 2023-10-09T09:07:06.850970+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:07.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:07.910 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:07.935 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:07.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:08.039 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:08.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:08.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:08.170 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:08.193 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:08.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:08.273 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:08.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:08.324 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:08.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:08.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 18929 18913 107.45 81.1875 0.00621087 0.00929981 2023-10-09T09:07:08.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:08.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:08 smithi094 bash[59677]: audit 2023-10-09T09:07:07.485216+0000 mon.a (mon.0) 1212 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:07:08.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:08 smithi094 bash[59677]: audit 2023-10-09T09:07:07.486311+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:07:08.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:08 smithi094 bash[59677]: audit 2023-10-09T09:07:07.495610+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:08.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:08.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:08.557 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:08.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:08.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:08.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:08.697 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:08 smithi016 bash[75179]: audit 2023-10-09T09:07:07.485216+0000 mon.a (mon.0) 1212 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:07:08.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:08 smithi016 bash[75179]: audit 2023-10-09T09:07:07.486311+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:07:08.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:08 smithi016 bash[75179]: audit 2023-10-09T09:07:07.495610+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:08.698 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:08 smithi016 bash[77892]: audit 2023-10-09T09:07:07.485216+0000 mon.a (mon.0) 1212 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:07:08.698 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:08 smithi016 bash[77892]: audit 2023-10-09T09:07:07.486311+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:07:08.698 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:08 smithi016 bash[77892]: audit 2023-10-09T09:07:07.495610+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:08.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:08.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:08.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:08.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:08.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:08.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:08.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:08.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:08.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:09.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:09.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:09.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:09.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:09.239 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:09.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:09.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:09.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:09.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:09.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:09.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 20789 20773 108.182 116.25 0.00874535 0.00923528 2023-10-09T09:07:09.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:09.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:09.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:09.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:09.613 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:09 smithi016 bash[75179]: cluster 2023-10-09T09:07:07.922957+0000 mgr.y (mgr.25421) 891 : cluster 0 pgmap v936: 89 pgs: 89 active+clean; 985 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T09:07:09.614 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:09 smithi016 bash[77892]: cluster 2023-10-09T09:07:07.922957+0000 mgr.y (mgr.25421) 891 : cluster 0 pgmap v936: 89 pgs: 89 active+clean; 985 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T09:07:09.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:09.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:09.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:09.738 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:09 smithi094 bash[59677]: cluster 2023-10-09T09:07:07.922957+0000 mgr.y (mgr.25421) 891 : cluster 0 pgmap v936: 89 pgs: 89 active+clean; 985 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T09:07:09.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:09.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:09.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:09.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:09.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:09.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:10.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:10.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:10.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:10.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:10.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:10.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:10.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:10.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:10.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 22435 22419 107.773 102.875 0.0105749 0.00927037 2023-10-09T09:07:10.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:10.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:10.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:10.546 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:10.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:10.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:10.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:10 smithi016 bash[75179]: cluster 2023-10-09T09:07:09.923476+0000 mgr.y (mgr.25421) 892 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 1011 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T09:07:10.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:10 smithi016 bash[77892]: cluster 2023-10-09T09:07:09.923476+0000 mgr.y (mgr.25421) 892 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 1011 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T09:07:10.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:10.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:10.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:10.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:10.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:10.944 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:10 smithi094 bash[59677]: cluster 2023-10-09T09:07:09.923476+0000 mgr.y (mgr.25421) 892 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 1011 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T09:07:10.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:10.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:10.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:11.071 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:11.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:11.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:11.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:11.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:11.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:11.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:11.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:11.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:11.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:11.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:11.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 24010 23994 107.106 98.4375 0.00387152 0.00933014 2023-10-09T09:07:11.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:11.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:11.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:11.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:11.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:11.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:11.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:11.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:11.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:11.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:11.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:11.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:11.860 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:11.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:11.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:11.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:12.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:12.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:12.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:12.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:12.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:12.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:12.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:12.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:12.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:12.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 25575 25559 106.486 97.8125 0.0081297 0.00938209 2023-10-09T09:07:12.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:12.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:12.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:12.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:12.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:12.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:12.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:12.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:12.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:12.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:12.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:12.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:13.064 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:13.074 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:12 smithi094 bash[59677]: cluster 2023-10-09T09:07:11.924217+0000 mgr.y (mgr.25421) 893 : cluster 0 pgmap v938: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T09:07:13.093 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:13.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:13.196 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:13.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:13.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:13.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:12 smithi016 bash[75179]: cluster 2023-10-09T09:07:11.924217+0000 mgr.y (mgr.25421) 893 : cluster 0 pgmap v938: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T09:07:13.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:12 smithi016 bash[77892]: cluster 2023-10-09T09:07:11.924217+0000 mgr.y (mgr.25421) 893 : cluster 0 pgmap v938: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T09:07:13.326 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:13.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:13.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:13.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:13.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 27098 27082 105.778 95.1875 0.00813857 0.00939899 2023-10-09T09:07:13.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:13.489 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:13.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:13.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:13.592 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:13.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:13.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:13.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:13.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:13.752 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:13.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:13.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:13.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:13.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:13.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:14.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:14.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:14.117 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:14.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:14.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:14.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:14.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:14.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:14.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 28727 28711 105.545 101.812 0.00557981 0.00946626 2023-10-09T09:07:14.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:14.516 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:14.541 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:14.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:14.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:14.777 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:14.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:14.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:14.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:14.933 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:14.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:14.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:15.037 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:15.050 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:14 smithi094 bash[59677]: cluster 2023-10-09T09:07:13.924905+0000 mgr.y (mgr.25421) 894 : cluster 0 pgmap v939: 89 pgs: 89 active+clean; 1.5 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:07:15.068 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:15.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:15.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:15.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:15.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:15.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:15.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:15.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:15.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:15.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 30040 30024 104.24 82.0625 0.00585339 0.00958667 2023-10-09T09:07:15.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:15.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:15.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:15.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:15 smithi016 bash[75179]: cluster 2023-10-09T09:07:13.924905+0000 mgr.y (mgr.25421) 894 : cluster 0 pgmap v939: 89 pgs: 89 active+clean; 1.5 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:07:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:15 smithi016 bash[77892]: cluster 2023-10-09T09:07:13.924905+0000 mgr.y (mgr.25421) 894 : cluster 0 pgmap v939: 89 pgs: 89 active+clean; 1.5 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:07:15.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:15.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:15.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:15.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:15.714 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:15.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:15.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:15.845 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:15.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:15.870 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:07:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:07:15] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.33.4" 2023-10-09T09:07:15.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:15.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:15.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:16.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:16.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:16.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:16.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:16.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:16.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:16.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:16.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 31375 31359 103.144 83.4375 0.0144472 0.00968847 2023-10-09T09:07:16.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:16.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:16.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:16.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:16.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:16.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:16.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:16.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:16.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:16.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:16.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:17.018 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:17.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:17.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:17.156 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:16 smithi016 bash[77892]: cluster 2023-10-09T09:07:15.925535+0000 mgr.y (mgr.25421) 895 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T09:07:17.156 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:16 smithi016 bash[75179]: cluster 2023-10-09T09:07:15.925535+0000 mgr.y (mgr.25421) 895 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T09:07:17.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:17.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:16 smithi094 bash[59677]: cluster 2023-10-09T09:07:15.925535+0000 mgr.y (mgr.25421) 895 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T09:07:17.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:17.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:17.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:07:17.392068+0000 min lat: 0.00143057 max lat: 0.310598 avg lat: 0.00976639 2023-10-09T09:07:17.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:07:17.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 32759 32743 102.312 86.5 0.0129724 0.00976639 2023-10-09T09:07:17.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:17.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:17.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:17.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:17.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:17.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:17.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:17.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:17.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:17.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:17.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:17.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:17.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:17.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:17.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:18.059 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:18.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:18.187 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:18.214 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:18.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:18.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:18.392 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 34187 34171 101.689 89.25 0.0139505 0.00982692 2023-10-09T09:07:18.448 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:18.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:18.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:18.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:18.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:18.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:18.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:18.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:18.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:18.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:18.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:18.987 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:18 smithi016 bash[75179]: cluster 2023-10-09T09:07:17.926169+0000 mgr.y (mgr.25421) 896 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 1.9 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:07:18.995 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:18.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:18.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:19.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:19.136 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:19.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:19.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:19.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:18 smithi094 bash[59677]: cluster 2023-10-09T09:07:17.926169+0000 mgr.y (mgr.25421) 896 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 1.9 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:07:19.242 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:19.267 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:19.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:19.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:19.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:19.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 35512 35496 100.831 82.8125 0.0027684 0.00989997 2023-10-09T09:07:19.393 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:19.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:19.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:19.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:19.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:19.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:19.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:19.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:19.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:19.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:19.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:19.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:19.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:19.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:19.914 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:19.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:19.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:19.977 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:18 smithi016 bash[77892]: cluster 2023-10-09T09:07:17.926169+0000 mgr.y (mgr.25421) 896 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 1.9 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:07:20.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:20.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:20.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:20.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:20.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:20.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:20.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:19 smithi094 bash[59677]: audit 2023-10-09T09:07:19.237633+0000 mon.a (mon.0) 1215 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:20.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:19 smithi016 bash[75179]: audit 2023-10-09T09:07:19.237633+0000 mon.a (mon.0) 1215 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:20.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:19 smithi016 bash[77892]: audit 2023-10-09T09:07:19.237633+0000 mon.a (mon.0) 1215 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:20.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:20.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:20.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:20.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 36846 36830 100.072 83.375 0.00369297 0.00998561 2023-10-09T09:07:20.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:20.434 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:20.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:20.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:20.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:20.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:20.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:20.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:20.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:20.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:20.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:20.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:20.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:20.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:20.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:21.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:21.110 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:21.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:21.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:21.213 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:21.224 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:20 smithi094 bash[59677]: cluster 2023-10-09T09:07:19.926685+0000 mgr.y (mgr.25421) 897 : cluster 0 pgmap v942: 89 pgs: 89 active+clean; 1.9 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:07:21.240 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:21.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:21.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:20 smithi016 bash[75179]: cluster 2023-10-09T09:07:19.926685+0000 mgr.y (mgr.25421) 897 : cluster 0 pgmap v942: 89 pgs: 89 active+clean; 1.9 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:07:21.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:20 smithi016 bash[77892]: cluster 2023-10-09T09:07:19.926685+0000 mgr.y (mgr.25421) 897 : cluster 0 pgmap v942: 89 pgs: 89 active+clean; 1.9 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:07:21.346 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:21.372 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:21.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:21.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:21.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 38201 38185 99.4302 84.6875 0.00330122 0.0100515 2023-10-09T09:07:21.474 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:21.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:21.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:21.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:21.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:21.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:21.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:21.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:21.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:21.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:21.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:21.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:21.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:21.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:22.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:22.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:22.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:22.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:22.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:22.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 39505 39489 98.7127 81.5 0.0193877 0.0101243 2023-10-09T09:07:22.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:22.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:22.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:22.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:22.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:22.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:22.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:22.810 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.836 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:22.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:22.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:22.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:22.993 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:22 smithi016 bash[75179]: cluster 2023-10-09T09:07:21.927367+0000 mgr.y (mgr.25421) 898 : cluster 0 pgmap v943: 89 pgs: 89 active+clean; 2.2 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T09:07:23.069 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:23.080 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:22 smithi094 bash[59677]: cluster 2023-10-09T09:07:21.927367+0000 mgr.y (mgr.25421) 898 : cluster 0 pgmap v943: 89 pgs: 89 active+clean; 2.2 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T09:07:23.096 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:23.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:23.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:23.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:23.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:23.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:23.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:22 smithi016 bash[77892]: cluster 2023-10-09T09:07:21.927367+0000 mgr.y (mgr.25421) 898 : cluster 0 pgmap v943: 89 pgs: 89 active+clean; 2.2 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T09:07:23.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:23.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:23.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:23.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:23.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 40597 40581 97.5407 68.25 0.00702849 0.010231 2023-10-09T09:07:23.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:23.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:23.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:23.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:23.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:23.606 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:23.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:23.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:23.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:23.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:23.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:23.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:23.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:23.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:23.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:24.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:24.105 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:24.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:24.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:24.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:24.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:24.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:24.389 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:24.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 41604 41588 96.2589 62.9375 0.00403629 0.0103776 2023-10-09T09:07:24.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:24.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:24.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:24.613 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:24.638 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:24.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:24.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:24.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:24.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:24.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:24.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:25.008 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:25.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:25.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:25.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:25.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:24 smithi094 bash[59677]: cluster 2023-10-09T09:07:23.928212+0000 mgr.y (mgr.25421) 899 : cluster 0 pgmap v944: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:07:25.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:25.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:24 smithi016 bash[75179]: cluster 2023-10-09T09:07:23.928212+0000 mgr.y (mgr.25421) 899 : cluster 0 pgmap v944: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:07:25.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:24 smithi016 bash[77892]: cluster 2023-10-09T09:07:23.928212+0000 mgr.y (mgr.25421) 899 : cluster 0 pgmap v944: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:07:25.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:25.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:25.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 42671 42655 95.2025 66.6875 0.00891285 0.0104972 2023-10-09T09:07:25.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:25.434 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:25.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:25.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:25.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:25.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:25.806 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:25.837 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:25.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:25.965 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:07:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:07:25] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.33.4" 2023-10-09T09:07:25.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:25.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:26.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:26.115 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:26.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:26.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:26.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:26.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:26.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:26.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:26.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:26.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 43807 43791 94.3676 71 0.00730602 0.0105845 2023-10-09T09:07:26.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:26.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:26.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:26.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:26.656 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:26.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:26.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:26.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:26.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:26.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:26.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:26.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:26.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:27.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:27.063 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:27.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:27 smithi016 bash[75179]: cluster 2023-10-09T09:07:25.928730+0000 mgr.y (mgr.25421) 900 : cluster 0 pgmap v945: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T09:07:27.066 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:27 smithi016 bash[77892]: cluster 2023-10-09T09:07:25.928730+0000 mgr.y (mgr.25421) 900 : cluster 0 pgmap v945: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T09:07:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:27.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:27.199 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:27.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:27.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:27.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:27.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:27.383 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:07:27.383 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:07:27.393 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 44704 44688 93.0908 56.0625 0.0380038 0.0107344 2023-10-09T09:07:27.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:27.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:27.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:26 smithi094 bash[59677]: cluster 2023-10-09T09:07:25.928730+0000 mgr.y (mgr.25421) 900 : cluster 0 pgmap v945: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T09:07:27.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:27.591 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:07:27.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:27.600 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:07:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:27.644 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:07:27.687 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:07:27.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:27.724 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:07:27.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:27.793 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:07:27.818 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:07:27.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:27.855 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:07:27.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:27.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:27.896 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:07:27.952 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:07:27.956 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.956 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.956 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.956 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.956 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.956 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.957 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.957 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:07:27.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:27.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:27.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:27.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:28.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:28.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:28.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:28.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:28.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:28 smithi094 bash[59677]: audit 2023-10-09T09:07:27.570189+0000 mon.a (mon.0) 1216 : audit 0 from='client.? 172.21.15.16:0/2516636170' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:07:28.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:28 smithi094 bash[59677]: audit 2023-10-09T09:07:27.570979+0000 mgr.y (mgr.25421) 901 : audit 0 from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:07:28.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:28 smithi016 bash[75179]: audit 2023-10-09T09:07:27.570189+0000 mon.a (mon.0) 1216 : audit 0 from='client.? 172.21.15.16:0/2516636170' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:07:28.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:28 smithi016 bash[75179]: audit 2023-10-09T09:07:27.570979+0000 mgr.y (mgr.25421) 901 : audit 0 from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:07:28.245 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:28 smithi016 bash[77892]: audit 2023-10-09T09:07:27.570189+0000 mon.a (mon.0) 1216 : audit 0 from='client.? 172.21.15.16:0/2516636170' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:07:28.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:28 smithi016 bash[77892]: audit 2023-10-09T09:07:27.570979+0000 mgr.y (mgr.25421) 901 : audit 0 from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:07:28.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:28.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:28.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:28.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:28.394 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 45649 45633 91.9897 59.0625 0.0241044 0.0108629 2023-10-09T09:07:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:28.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:28.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:28.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:28.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:28.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:28.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:28.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:28.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:28.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:28.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:28.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:28.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:28.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:28.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:29.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:29.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:29.042 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:07:29.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:29.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:29.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:29.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:29.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:29 smithi016 bash[75179]: cluster 2023-10-09T09:07:27.929372+0000 mgr.y (mgr.25421) 902 : cluster 0 pgmap v946: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T09:07:29.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:29 smithi016 bash[75179]: audit 2023-10-09T09:07:28.276217+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.16:0/3778143442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:07:29.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:29 smithi016 bash[75179]: audit 2023-10-09T09:07:28.277780+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:07:29.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:29 smithi016 bash[77892]: cluster 2023-10-09T09:07:27.929372+0000 mgr.y (mgr.25421) 902 : cluster 0 pgmap v946: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T09:07:29.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:29 smithi016 bash[77892]: audit 2023-10-09T09:07:28.276217+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.16:0/3778143442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:07:29.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:29 smithi016 bash[77892]: audit 2023-10-09T09:07:28.277780+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:07:29.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:29.394 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 46894 46878 91.5465 77.8125 0.0133514 0.0109177 2023-10-09T09:07:29.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:29.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:29.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:29 smithi094 bash[59677]: cluster 2023-10-09T09:07:27.929372+0000 mgr.y (mgr.25421) 902 : cluster 0 pgmap v946: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T09:07:29.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:29 smithi094 bash[59677]: audit 2023-10-09T09:07:28.276217+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.16:0/3778143442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:07:29.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:29 smithi094 bash[59677]: audit 2023-10-09T09:07:28.277780+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:07:29.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:29.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:29.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:29.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:29.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:29.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:29.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:29.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:29.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:29.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:29.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:30.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:30.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:30.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:30.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:30.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:30.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:30.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[75179]: cluster 2023-10-09T09:07:29.007773+0000 mon.a (mon.0) 1218 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:07:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[75179]: cluster 2023-10-09T09:07:29.031540+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[75179]: audit 2023-10-09T09:07:29.035931+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:07:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[75179]: cluster 2023-10-09T09:07:29.036091+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e219: 8 total, 8 up, 5 in 2023-10-09T09:07:30.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[75179]: cluster 2023-10-09T09:07:29.083322+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[75179]: cluster 2023-10-09T09:07:29.087631+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[75179]: cluster 2023-10-09T09:07:29.089836+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[77892]: cluster 2023-10-09T09:07:29.007773+0000 mon.a (mon.0) 1218 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:07:30.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[77892]: cluster 2023-10-09T09:07:29.031540+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[77892]: audit 2023-10-09T09:07:29.035931+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:07:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[77892]: cluster 2023-10-09T09:07:29.036091+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e219: 8 total, 8 up, 5 in 2023-10-09T09:07:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[77892]: cluster 2023-10-09T09:07:29.083322+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[77892]: cluster 2023-10-09T09:07:29.087631+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:30 smithi016 bash[77892]: cluster 2023-10-09T09:07:29.089836+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:30.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:30.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:30.394 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 48014 47998 90.8933 70 0.0117316 0.0109945 2023-10-09T09:07:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:30 smithi094 bash[59677]: cluster 2023-10-09T09:07:29.007773+0000 mon.a (mon.0) 1218 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:07:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:30 smithi094 bash[59677]: cluster 2023-10-09T09:07:29.031540+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:30 smithi094 bash[59677]: audit 2023-10-09T09:07:29.035931+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:07:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:30 smithi094 bash[59677]: cluster 2023-10-09T09:07:29.036091+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e219: 8 total, 8 up, 5 in 2023-10-09T09:07:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:30 smithi094 bash[59677]: cluster 2023-10-09T09:07:29.083322+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:30 smithi094 bash[59677]: cluster 2023-10-09T09:07:29.087631+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:30 smithi094 bash[59677]: cluster 2023-10-09T09:07:29.089836+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T09:07:30.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:30.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:30.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:30.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:30.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:30.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:30.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:30.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:30.799 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:30.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:30.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:30.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:30.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:30.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:30.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:31.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:31.058 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:31 smithi094 bash[59677]: cluster 2023-10-09T09:07:29.929845+0000 mgr.y (mgr.25421) 903 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T09:07:31.079 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:31.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:31.184 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:31.216 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:31.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:31.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:31 smithi016 bash[75179]: cluster 2023-10-09T09:07:29.929845+0000 mgr.y (mgr.25421) 903 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T09:07:31.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:31 smithi016 bash[77892]: cluster 2023-10-09T09:07:29.929845+0000 mgr.y (mgr.25421) 903 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T09:07:31.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:31.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:31.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:31.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 48999 48983 90.0305 61.5625 0.0198424 0.0111008 2023-10-09T09:07:31.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:31.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:31.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:31.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:31.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:31.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:31.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:31.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:31.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:31.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:31.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:31.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:31.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:31.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:31.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:32.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:32.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:32.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:32.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:32.241 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:32.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:32.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:32.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 50094 50078 89.4135 68.4375 0.0381031 0.0111748 2023-10-09T09:07:32.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:32.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:32.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:32.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:32.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:32.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:32.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:07:32.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:32.832 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":945,"stamp":"2023-10-09T09:07:25.928456+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2566920096,"num_objects":39183,"num_object_clones":0,"num_object_copies":117549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39183,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":39419,"num_write_kb":2508621,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":2374946,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39353,"ondisk_log_size":39353,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":10684912,"kb_used_data":7716296,"kb_used_omap":117,"kb_used_meta":2968394,"kb_avail":458040848,"statfs":{"total":479975178240,"available":469033828352,"internally_reserved":0,"allocated":7901487104,"data_stored":7897927726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":3039636114},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1001783296,"num_objects":15286,"num_object_clones":0,"num_object_copies":45858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15286,"num_write_kb":978304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004023"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":80,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.685139+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:06:52.685139+0000","last_peered":"2023-10-09T09:06:52.685139+0000","last_clean":"2023-10-09T09:06:52.685139+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:06:52.685139+0000","last_undegraded":"2023-10-09T09:06:52.685139+0000","last_fullsized":"2023-10-09T09:06:52.685139+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":230,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114215+0000","last_change":"2023-10-09T09:02:02.625626+0000","last_active":"2023-10-09T09:06:59.114215+0000","last_peered":"2023-10-09T09:06:59.114215+0000","last_clean":"2023-10-09T09:06:59.114215+0000","last_became_active":"2023-10-09T09:02:02.625312+0000","last_became_peered":"2023-10-09T09:02:02.625312+0000","last_unstale":"2023-10-09T09:06:59.114215+0000","last_undegraded":"2023-10-09T09:06:59.114215+0000","last_fullsized":"2023-10-09T09:06:59.114215+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:01.640943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":183,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114020+0000","last_change":"2023-10-09T09:02:02.626274+0000","last_active":"2023-10-09T09:06:59.114020+0000","last_peered":"2023-10-09T09:06:59.114020+0000","last_clean":"2023-10-09T09:06:59.114020+0000","last_became_active":"2023-10-09T09:02:02.625973+0000","last_became_peered":"2023-10-09T09:02:02.625973+0000","last_unstale":"2023-10-09T09:06:59.114020+0000","last_undegraded":"2023-10-09T09:06:59.114020+0000","last_fullsized":"2023-10-09T09:06:59.114020+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:44.223794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":230,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.933252+0000","last_change":"2023-10-09T09:03:19.554441+0000","last_active":"2023-10-09T09:06:53.933252+0000","last_peered":"2023-10-09T09:06:53.933252+0000","last_clean":"2023-10-09T09:06:53.933252+0000","last_became_active":"2023-10-09T09:03:19.553967+0000","last_became_peered":"2023-10-09T09:03:19.553967+0000","last_unstale":"2023-10-09T09:06:53.933252+0000","last_undegraded":"2023-10-09T09:06:53.933252+0000","last_fullsized":"2023-10-09T09:06:53.933252+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:26.065993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":219,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333102+0000","last_change":"2023-10-09T09:02:02.656821+0000","last_active":"2023-10-09T09:06:53.333102+0000","last_peered":"2023-10-09T09:06:53.333102+0000","last_clean":"2023-10-09T09:06:53.333102+0000","last_became_active":"2023-10-09T09:02:02.656523+0000","last_became_peered":"2023-10-09T09:02:02.656523+0000","last_unstale":"2023-10-09T09:06:53.333102+0000","last_undegraded":"2023-10-09T09:06:53.333102+0000","last_fullsized":"2023-10-09T09:06:53.333102+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:21.635518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":94,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.450230+0000","last_change":"2023-10-09T09:03:19.582769+0000","last_active":"2023-10-09T09:06:53.450230+0000","last_peered":"2023-10-09T09:06:53.450230+0000","last_clean":"2023-10-09T09:06:53.450230+0000","last_became_active":"2023-10-09T09:03:19.577023+0000","last_became_peered":"2023-10-09T09:03:19.577023+0000","last_unstale":"2023-10-09T09:06:53.450230+0000","last_undegraded":"2023-10-09T09:06:53.450230+0000","last_fullsized":"2023-10-09T09:06:53.450230+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":192,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.933317+0000","last_change":"2023-10-09T09:01:36.280403+0000","last_active":"2023-10-09T09:06:53.933317+0000","last_peered":"2023-10-09T09:06:53.933317+0000","last_clean":"2023-10-09T09:06:53.933317+0000","last_became_active":"2023-10-09T09:01:36.280070+0000","last_became_peered":"2023-10-09T09:01:36.280070+0000","last_unstale":"2023-10-09T09:06:53.933317+0000","last_undegraded":"2023-10-09T09:06:53.933317+0000","last_fullsized":"2023-10-09T09:06:53.933317+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:33.796899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":230,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333330+0000","last_change":"2023-10-09T09:02:02.630162+0000","last_active":"2023-10-09T09:06:53.333330+0000","last_peered":"2023-10-09T09:06:53.333330+0000","last_clean":"2023-10-09T09:06:53.333330+0000","last_became_active":"2023-10-09T09:02:02.629768+0000","last_became_peered":"2023-10-09T09:02:02.629768+0000","last_unstale":"2023-10-09T09:06:53.333330+0000","last_undegraded":"2023-10-09T09:06:53.333330+0000","last_fullsized":"2023-10-09T09:06:53.333330+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:36.859890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":36,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333934+0000","last_change":"2023-10-09T09:03:19.562130+0000","last_active":"2023-10-09T09:06:53.333934+0000","last_peered":"2023-10-09T09:06:53.333934+0000","last_clean":"2023-10-09T09:06:53.333934+0000","last_became_active":"2023-10-09T09:03:19.557757+0000","last_became_peered":"2023-10-09T09:03:19.557757+0000","last_unstale":"2023-10-09T09:06:53.333934+0000","last_undegraded":"2023-10-09T09:06:53.333934+0000","last_fullsized":"2023-10-09T09:06:53.333934+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:46.546761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":191,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333051+0000","last_change":"2023-10-09T09:01:36.279247+0000","last_active":"2023-10-09T09:06:53.333051+0000","last_peered":"2023-10-09T09:06:53.333051+0000","last_clean":"2023-10-09T09:06:53.333051+0000","last_became_active":"2023-10-09T09:01:36.278376+0000","last_became_peered":"2023-10-09T09:01:36.278376+0000","last_unstale":"2023-10-09T09:06:53.333051+0000","last_undegraded":"2023-10-09T09:06:53.333051+0000","last_fullsized":"2023-10-09T09:06:53.333051+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:19.166567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":176,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333641+0000","last_change":"2023-10-09T08:59:53.544302+0000","last_active":"2023-10-09T09:06:53.333641+0000","last_peered":"2023-10-09T09:06:53.333641+0000","last_clean":"2023-10-09T09:06:53.333641+0000","last_became_active":"2023-10-09T08:59:53.531404+0000","last_became_peered":"2023-10-09T08:59:53.531404+0000","last_unstale":"2023-10-09T09:06:53.333641+0000","last_undegraded":"2023-10-09T09:06:53.333641+0000","last_fullsized":"2023-10-09T09:06:53.333641+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:48.670745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":177,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.448855+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:06:53.448855+0000","last_peered":"2023-10-09T09:06:53.448855+0000","last_clean":"2023-10-09T09:06:53.448855+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:06:53.448855+0000","last_undegraded":"2023-10-09T09:06:53.448855+0000","last_fullsized":"2023-10-09T09:06:53.448855+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":100,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684882+0000","last_change":"2023-10-09T09:00:31.867387+0000","last_active":"2023-10-09T09:06:52.684882+0000","last_peered":"2023-10-09T09:06:52.684882+0000","last_clean":"2023-10-09T09:06:52.684882+0000","last_became_active":"2023-10-09T09:00:31.859752+0000","last_became_peered":"2023-10-09T09:00:31.859752+0000","last_unstale":"2023-10-09T09:06:52.684882+0000","last_undegraded":"2023-10-09T09:06:52.684882+0000","last_fullsized":"2023-10-09T09:06:52.684882+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":89,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684489+0000","last_change":"2023-10-09T09:02:02.630025+0000","last_active":"2023-10-09T09:06:52.684489+0000","last_peered":"2023-10-09T09:06:52.684489+0000","last_clean":"2023-10-09T09:06:52.684489+0000","last_became_active":"2023-10-09T09:02:02.629474+0000","last_became_peered":"2023-10-09T09:02:02.629474+0000","last_unstale":"2023-10-09T09:06:52.684489+0000","last_undegraded":"2023-10-09T09:06:52.684489+0000","last_fullsized":"2023-10-09T09:06:52.684489+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":92,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684811+0000","last_change":"2023-10-09T09:00:31.959283+0000","last_active":"2023-10-09T09:06:52.684811+0000","last_peered":"2023-10-09T09:06:52.684811+0000","last_clean":"2023-10-09T09:06:52.684811+0000","last_became_active":"2023-10-09T09:00:31.861638+0000","last_became_peered":"2023-10-09T09:00:31.861638+0000","last_unstale":"2023-10-09T09:06:52.684811+0000","last_undegraded":"2023-10-09T09:06:52.684811+0000","last_fullsized":"2023-10-09T09:06:52.684811+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":212,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333610+0000","last_change":"2023-10-09T09:02:02.636035+0000","last_active":"2023-10-09T09:06:53.333610+0000","last_peered":"2023-10-09T09:06:53.333610+0000","last_clean":"2023-10-09T09:06:53.333610+0000","last_became_active":"2023-10-09T09:02:02.635621+0000","last_became_peered":"2023-10-09T09:02:02.635621+0000","last_unstale":"2023-10-09T09:06:53.333610+0000","last_undegraded":"2023-10-09T09:06:53.333610+0000","last_fullsized":"2023-10-09T09:06:53.333610+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:46.294650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":220,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932703+0000","last_change":"2023-10-09T09:03:19.564077+0000","last_active":"2023-10-09T09:06:53.932703+0000","last_peered":"2023-10-09T09:06:53.932703+0000","last_clean":"2023-10-09T09:06:53.932703+0000","last_became_active":"2023-10-09T09:03:19.563764+0000","last_became_peered":"2023-10-09T09:03:19.563764+0000","last_unstale":"2023-10-09T09:06:53.932703+0000","last_undegraded":"2023-10-09T09:06:53.932703+0000","last_fullsized":"2023-10-09T09:06:53.932703+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:38.043450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":137,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.450179+0000","last_change":"2023-10-09T09:06:50.678855+0000","last_active":"2023-10-09T09:06:53.450179+0000","last_peered":"2023-10-09T09:06:53.450179+0000","last_clean":"2023-10-09T09:06:53.450179+0000","last_became_active":"2023-10-09T09:06:50.678286+0000","last_became_peered":"2023-10-09T09:06:50.678286+0000","last_unstale":"2023-10-09T09:06:53.450179+0000","last_undegraded":"2023-10-09T09:06:53.450179+0000","last_fullsized":"2023-10-09T09:06:53.450179+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684898+0000","last_change":"2023-10-09T09:00:31.851267+0000","last_active":"2023-10-09T09:06:52.684898+0000","last_peered":"2023-10-09T09:06:52.684898+0000","last_clean":"2023-10-09T09:06:52.684898+0000","last_became_active":"2023-10-09T09:00:31.850918+0000","last_became_peered":"2023-10-09T09:00:31.850918+0000","last_unstale":"2023-10-09T09:06:52.684898+0000","last_undegraded":"2023-10-09T09:06:52.684898+0000","last_fullsized":"2023-10-09T09:06:52.684898+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":192,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.933290+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T09:06:53.933290+0000","last_peered":"2023-10-09T09:06:53.933290+0000","last_clean":"2023-10-09T09:06:53.933290+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T09:06:53.933290+0000","last_undegraded":"2023-10-09T09:06:53.933290+0000","last_fullsized":"2023-10-09T09:06:53.933290+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":96,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.448638+0000","last_change":"2023-10-09T09:01:36.301852+0000","last_active":"2023-10-09T09:06:53.448638+0000","last_peered":"2023-10-09T09:06:53.448638+0000","last_clean":"2023-10-09T09:06:53.448638+0000","last_became_active":"2023-10-09T09:01:36.284416+0000","last_became_peered":"2023-10-09T09:01:36.284416+0000","last_unstale":"2023-10-09T09:06:53.448638+0000","last_undegraded":"2023-10-09T09:06:53.448638+0000","last_fullsized":"2023-10-09T09:06:53.448638+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":36,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333930+0000","last_change":"2023-10-09T09:03:19.575211+0000","last_active":"2023-10-09T09:06:53.333930+0000","last_peered":"2023-10-09T09:06:53.333930+0000","last_clean":"2023-10-09T09:06:53.333930+0000","last_became_active":"2023-10-09T09:03:19.574829+0000","last_became_peered":"2023-10-09T09:03:19.574829+0000","last_unstale":"2023-10-09T09:06:53.333930+0000","last_undegraded":"2023-10-09T09:06:53.333930+0000","last_fullsized":"2023-10-09T09:06:53.333930+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:14.763790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"218'2458","reported_seq":2473,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:22.619000+0000","last_change":"2023-10-09T09:06:27.619755+0000","last_active":"2023-10-09T09:07:22.619000+0000","last_peered":"2023-10-09T09:07:22.619000+0000","last_clean":"2023-10-09T09:07:22.619000+0000","last_became_active":"2023-10-09T09:06:27.619440+0000","last_became_peered":"2023-10-09T09:06:27.619440+0000","last_unstale":"2023-10-09T09:07:22.619000+0000","last_undegraded":"2023-10-09T09:07:22.619000+0000","last_fullsized":"2023-10-09T09:07:22.619000+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2458,"log_dups_size":0,"ondisk_log_size":2458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:12.821748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161087488,"num_objects":2458,"num_object_clones":0,"num_object_copies":7374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2458,"num_write_kb":157312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":17,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:54.507037+0000","last_change":"2023-10-09T09:06:50.677770+0000","last_active":"2023-10-09T09:06:54.507037+0000","last_peered":"2023-10-09T09:06:54.507037+0000","last_clean":"2023-10-09T09:06:54.507037+0000","last_became_active":"2023-10-09T09:06:50.674696+0000","last_became_peered":"2023-10-09T09:06:50.674696+0000","last_unstale":"2023-10-09T09:06:54.507037+0000","last_undegraded":"2023-10-09T09:06:54.507037+0000","last_fullsized":"2023-10-09T09:06:54.507037+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:24:52.193110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":195,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.448140+0000","last_change":"2023-10-09T09:03:19.582540+0000","last_active":"2023-10-09T09:06:53.448140+0000","last_peered":"2023-10-09T09:06:53.448140+0000","last_clean":"2023-10-09T09:06:53.448140+0000","last_became_active":"2023-10-09T09:03:19.577044+0000","last_became_peered":"2023-10-09T09:03:19.577044+0000","last_unstale":"2023-10-09T09:06:53.448140+0000","last_undegraded":"2023-10-09T09:06:53.448140+0000","last_fullsized":"2023-10-09T09:06:53.448140+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"218'2735","reported_seq":2749,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:25.392826+0000","last_change":"2023-10-09T09:06:27.616114+0000","last_active":"2023-10-09T09:07:25.392826+0000","last_peered":"2023-10-09T09:07:25.392826+0000","last_clean":"2023-10-09T09:07:25.392826+0000","last_became_active":"2023-10-09T09:06:27.615815+0000","last_became_peered":"2023-10-09T09:06:27.615815+0000","last_unstale":"2023-10-09T09:07:25.392826+0000","last_undegraded":"2023-10-09T09:07:25.392826+0000","last_fullsized":"2023-10-09T09:07:25.392826+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2735,"log_dups_size":0,"ondisk_log_size":2735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179240960,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2735,"num_write_kb":175040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":88,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.685560+0000","last_change":"2023-10-09T09:02:02.724809+0000","last_active":"2023-10-09T09:06:52.685560+0000","last_peered":"2023-10-09T09:06:52.685560+0000","last_clean":"2023-10-09T09:06:52.685560+0000","last_became_active":"2023-10-09T09:02:02.666202+0000","last_became_peered":"2023-10-09T09:02:02.666202+0000","last_unstale":"2023-10-09T09:06:52.685560+0000","last_undegraded":"2023-10-09T09:06:52.685560+0000","last_fullsized":"2023-10-09T09:06:52.685560+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":98,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684642+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:06:52.684642+0000","last_peered":"2023-10-09T09:06:52.684642+0000","last_clean":"2023-10-09T09:06:52.684642+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:06:52.684642+0000","last_undegraded":"2023-10-09T09:06:52.684642+0000","last_fullsized":"2023-10-09T09:06:52.684642+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"218'2281","reported_seq":2296,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.864797+0000","last_change":"2023-10-09T09:06:27.614853+0000","last_active":"2023-10-09T09:07:20.864797+0000","last_peered":"2023-10-09T09:07:20.864797+0000","last_clean":"2023-10-09T09:07:20.864797+0000","last_became_active":"2023-10-09T09:06:27.614466+0000","last_became_peered":"2023-10-09T09:06:27.614466+0000","last_unstale":"2023-10-09T09:07:20.864797+0000","last_undegraded":"2023-10-09T09:07:20.864797+0000","last_fullsized":"2023-10-09T09:07:20.864797+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2281,"log_dups_size":0,"ondisk_log_size":2281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:20.558954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149487616,"num_objects":2281,"num_object_clones":0,"num_object_copies":6843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2281,"num_write_kb":145984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":127,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.450064+0000","last_change":"2023-10-09T09:01:36.305615+0000","last_active":"2023-10-09T09:06:53.450064+0000","last_peered":"2023-10-09T09:06:53.450064+0000","last_clean":"2023-10-09T09:06:53.450064+0000","last_became_active":"2023-10-09T09:01:36.292828+0000","last_became_peered":"2023-10-09T09:01:36.292828+0000","last_unstale":"2023-10-09T09:06:53.450064+0000","last_undegraded":"2023-10-09T09:06:53.450064+0000","last_fullsized":"2023-10-09T09:06:53.450064+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":94,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.449053+0000","last_change":"2023-10-09T09:03:19.564369+0000","last_active":"2023-10-09T09:06:53.449053+0000","last_peered":"2023-10-09T09:06:53.449053+0000","last_clean":"2023-10-09T09:06:53.449053+0000","last_became_active":"2023-10-09T09:03:19.563989+0000","last_became_peered":"2023-10-09T09:03:19.563989+0000","last_unstale":"2023-10-09T09:06:53.449053+0000","last_undegraded":"2023-10-09T09:06:53.449053+0000","last_fullsized":"2023-10-09T09:06:53.449053+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"218'2478","reported_seq":2493,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:23.243723+0000","last_change":"2023-10-09T09:06:27.620026+0000","last_active":"2023-10-09T09:07:23.243723+0000","last_peered":"2023-10-09T09:07:23.243723+0000","last_clean":"2023-10-09T09:07:23.243723+0000","last_became_active":"2023-10-09T09:06:27.619665+0000","last_became_peered":"2023-10-09T09:06:27.619665+0000","last_unstale":"2023-10-09T09:07:23.243723+0000","last_undegraded":"2023-10-09T09:07:23.243723+0000","last_fullsized":"2023-10-09T09:07:23.243723+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2478,"log_dups_size":0,"ondisk_log_size":2478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:13.420364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162398208,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2478,"num_write_kb":158592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333725+0000","last_change":"2023-10-09T09:02:02.664617+0000","last_active":"2023-10-09T09:06:53.333725+0000","last_peered":"2023-10-09T09:06:53.333725+0000","last_clean":"2023-10-09T09:06:53.333725+0000","last_became_active":"2023-10-09T09:02:02.664305+0000","last_became_peered":"2023-10-09T09:02:02.664305+0000","last_unstale":"2023-10-09T09:06:53.333725+0000","last_undegraded":"2023-10-09T09:06:53.333725+0000","last_fullsized":"2023-10-09T09:06:53.333725+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:14.520177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":211,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932672+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:06:53.932672+0000","last_peered":"2023-10-09T09:06:53.932672+0000","last_clean":"2023-10-09T09:06:53.932672+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:06:53.932672+0000","last_undegraded":"2023-10-09T09:06:53.932672+0000","last_fullsized":"2023-10-09T09:06:53.932672+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"218'2373","reported_seq":2388,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.820334+0000","last_change":"2023-10-09T09:06:27.616764+0000","last_active":"2023-10-09T09:07:20.820334+0000","last_peered":"2023-10-09T09:07:20.820334+0000","last_clean":"2023-10-09T09:07:20.820334+0000","last_became_active":"2023-10-09T09:06:27.615844+0000","last_became_peered":"2023-10-09T09:06:27.615844+0000","last_unstale":"2023-10-09T09:07:20.820334+0000","last_undegraded":"2023-10-09T09:07:20.820334+0000","last_fullsized":"2023-10-09T09:07:20.820334+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2373,"log_dups_size":0,"ondisk_log_size":2373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:11.236973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155516928,"num_objects":2373,"num_object_clones":0,"num_object_copies":7119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2373,"num_write_kb":151872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":219,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333180+0000","last_change":"2023-10-09T08:59:53.642095+0000","last_active":"2023-10-09T09:06:53.333180+0000","last_peered":"2023-10-09T09:06:53.333180+0000","last_clean":"2023-10-09T09:06:53.333180+0000","last_became_active":"2023-10-09T08:59:53.531588+0000","last_became_peered":"2023-10-09T08:59:53.531588+0000","last_unstale":"2023-10-09T09:06:53.333180+0000","last_undegraded":"2023-10-09T09:06:53.333180+0000","last_fullsized":"2023-10-09T09:06:53.333180+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.878230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":80,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684682+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:06:52.684682+0000","last_peered":"2023-10-09T09:06:52.684682+0000","last_clean":"2023-10-09T09:06:52.684682+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:06:52.684682+0000","last_undegraded":"2023-10-09T09:06:52.684682+0000","last_fullsized":"2023-10-09T09:06:52.684682+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"218'2374","reported_seq":2388,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:21.120807+0000","last_change":"2023-10-09T09:06:27.621010+0000","last_active":"2023-10-09T09:07:21.120807+0000","last_peered":"2023-10-09T09:07:21.120807+0000","last_clean":"2023-10-09T09:07:21.120807+0000","last_became_active":"2023-10-09T09:06:27.620675+0000","last_became_peered":"2023-10-09T09:06:27.620675+0000","last_unstale":"2023-10-09T09:07:21.120807+0000","last_undegraded":"2023-10-09T09:07:21.120807+0000","last_fullsized":"2023-10-09T09:07:21.120807+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2374,"log_dups_size":0,"ondisk_log_size":2374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155582464,"num_objects":2374,"num_object_clones":0,"num_object_copies":7122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2374,"num_write_kb":151936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.448996+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:06:53.448996+0000","last_peered":"2023-10-09T09:06:53.448996+0000","last_clean":"2023-10-09T09:06:53.448996+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:06:53.448996+0000","last_undegraded":"2023-10-09T09:06:53.448996+0000","last_fullsized":"2023-10-09T09:06:53.448996+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.685306+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:06:52.685306+0000","last_peered":"2023-10-09T09:06:52.685306+0000","last_clean":"2023-10-09T09:06:52.685306+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:06:52.685306+0000","last_undegraded":"2023-10-09T09:06:52.685306+0000","last_fullsized":"2023-10-09T09:06:52.685306+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"218'2390","reported_seq":2405,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.831489+0000","last_change":"2023-10-09T09:06:27.621711+0000","last_active":"2023-10-09T09:07:20.831489+0000","last_peered":"2023-10-09T09:07:20.831489+0000","last_clean":"2023-10-09T09:07:20.831489+0000","last_became_active":"2023-10-09T09:06:27.620532+0000","last_became_peered":"2023-10-09T09:06:27.620532+0000","last_unstale":"2023-10-09T09:07:20.831489+0000","last_undegraded":"2023-10-09T09:07:20.831489+0000","last_fullsized":"2023-10-09T09:07:20.831489+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2390,"log_dups_size":0,"ondisk_log_size":2390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:42.373323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156631040,"num_objects":2390,"num_object_clones":0,"num_object_copies":7170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2390,"num_write_kb":152960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684377+0000","last_change":"2023-10-09T09:03:19.563963+0000","last_active":"2023-10-09T09:06:52.684377+0000","last_peered":"2023-10-09T09:06:52.684377+0000","last_clean":"2023-10-09T09:06:52.684377+0000","last_became_active":"2023-10-09T09:03:19.563629+0000","last_became_peered":"2023-10-09T09:03:19.563629+0000","last_unstale":"2023-10-09T09:06:52.684377+0000","last_undegraded":"2023-10-09T09:06:52.684377+0000","last_fullsized":"2023-10-09T09:06:52.684377+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":89,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.685452+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:06:52.685452+0000","last_peered":"2023-10-09T09:06:52.685452+0000","last_clean":"2023-10-09T09:06:52.685452+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:06:52.685452+0000","last_undegraded":"2023-10-09T09:06:52.685452+0000","last_fullsized":"2023-10-09T09:06:52.685452+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"218'2362","reported_seq":2377,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.859652+0000","last_change":"2023-10-09T09:06:27.617347+0000","last_active":"2023-10-09T09:07:20.859652+0000","last_peered":"2023-10-09T09:07:20.859652+0000","last_clean":"2023-10-09T09:07:20.859652+0000","last_became_active":"2023-10-09T09:06:27.615423+0000","last_became_peered":"2023-10-09T09:06:27.615423+0000","last_unstale":"2023-10-09T09:07:20.859652+0000","last_undegraded":"2023-10-09T09:07:20.859652+0000","last_fullsized":"2023-10-09T09:07:20.859652+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2362,"log_dups_size":0,"ondisk_log_size":2362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:53.926159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154796032,"num_objects":2362,"num_object_clones":0,"num_object_copies":7086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2362,"num_write_kb":151168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":36,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.332816+0000","last_change":"2023-10-09T09:03:19.572389+0000","last_active":"2023-10-09T09:06:53.332816+0000","last_peered":"2023-10-09T09:06:53.332816+0000","last_clean":"2023-10-09T09:06:53.332816+0000","last_became_active":"2023-10-09T09:03:19.572091+0000","last_became_peered":"2023-10-09T09:03:19.572091+0000","last_unstale":"2023-10-09T09:06:53.332816+0000","last_undegraded":"2023-10-09T09:06:53.332816+0000","last_fullsized":"2023-10-09T09:06:53.332816+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:27.702718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.334077+0000","last_change":"2023-10-09T09:02:02.636996+0000","last_active":"2023-10-09T09:06:53.334077+0000","last_peered":"2023-10-09T09:06:53.334077+0000","last_clean":"2023-10-09T09:06:53.334077+0000","last_became_active":"2023-10-09T09:02:02.636702+0000","last_became_peered":"2023-10-09T09:02:02.636702+0000","last_unstale":"2023-10-09T09:06:53.334077+0000","last_undegraded":"2023-10-09T09:06:53.334077+0000","last_fullsized":"2023-10-09T09:06:53.334077+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:45.392645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"218'2360","reported_seq":2375,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.864305+0000","last_change":"2023-10-09T09:06:27.616361+0000","last_active":"2023-10-09T09:07:20.864305+0000","last_peered":"2023-10-09T09:07:20.864305+0000","last_clean":"2023-10-09T09:07:20.864305+0000","last_became_active":"2023-10-09T09:06:27.616004+0000","last_became_peered":"2023-10-09T09:06:27.616004+0000","last_unstale":"2023-10-09T09:07:20.864305+0000","last_undegraded":"2023-10-09T09:07:20.864305+0000","last_fullsized":"2023-10-09T09:07:20.864305+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2360,"log_dups_size":0,"ondisk_log_size":2360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:10.699277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154664960,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2360,"num_write_kb":151040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":161,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932808+0000","last_change":"2023-10-09T09:03:19.556450+0000","last_active":"2023-10-09T09:06:53.932808+0000","last_peered":"2023-10-09T09:06:53.932808+0000","last_clean":"2023-10-09T09:06:53.932808+0000","last_became_active":"2023-10-09T09:03:19.556190+0000","last_became_peered":"2023-10-09T09:03:19.556190+0000","last_unstale":"2023-10-09T09:06:53.932808+0000","last_undegraded":"2023-10-09T09:06:53.932808+0000","last_fullsized":"2023-10-09T09:06:53.932808+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:23.704455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":156,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333757+0000","last_change":"2023-10-09T09:01:36.285844+0000","last_active":"2023-10-09T09:06:53.333757+0000","last_peered":"2023-10-09T09:06:53.333757+0000","last_clean":"2023-10-09T09:06:53.333757+0000","last_became_active":"2023-10-09T09:01:36.285525+0000","last_became_peered":"2023-10-09T09:01:36.285525+0000","last_unstale":"2023-10-09T09:06:53.333757+0000","last_undegraded":"2023-10-09T09:06:53.333757+0000","last_fullsized":"2023-10-09T09:06:53.333757+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:05.624756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":197,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.685304+0000","last_change":"2023-10-09T09:02:52.021749+0000","last_active":"2023-10-09T09:06:52.685304+0000","last_peered":"2023-10-09T09:06:52.685304+0000","last_clean":"2023-10-09T09:06:52.685304+0000","last_became_active":"2023-10-09T09:02:52.021416+0000","last_became_peered":"2023-10-09T09:02:52.021416+0000","last_unstale":"2023-10-09T09:06:52.685304+0000","last_undegraded":"2023-10-09T09:06:52.685304+0000","last_fullsized":"2023-10-09T09:06:52.685304+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"218'2448","reported_seq":2463,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:22.588286+0000","last_change":"2023-10-09T09:06:27.620398+0000","last_active":"2023-10-09T09:07:22.588286+0000","last_peered":"2023-10-09T09:07:22.588286+0000","last_clean":"2023-10-09T09:07:22.588286+0000","last_became_active":"2023-10-09T09:06:27.619485+0000","last_became_peered":"2023-10-09T09:06:27.619485+0000","last_unstale":"2023-10-09T09:07:22.588286+0000","last_undegraded":"2023-10-09T09:07:22.588286+0000","last_fullsized":"2023-10-09T09:07:22.588286+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2448,"log_dups_size":0,"ondisk_log_size":2448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:19.177497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160432128,"num_objects":2448,"num_object_clones":0,"num_object_copies":7344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2448,"num_write_kb":156672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":185,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.448759+0000","last_change":"2023-10-09T09:03:19.564803+0000","last_active":"2023-10-09T09:06:53.448759+0000","last_peered":"2023-10-09T09:06:53.448759+0000","last_clean":"2023-10-09T09:06:53.448759+0000","last_became_active":"2023-10-09T09:03:19.564520+0000","last_became_peered":"2023-10-09T09:03:19.564520+0000","last_unstale":"2023-10-09T09:06:53.448759+0000","last_undegraded":"2023-10-09T09:06:53.448759+0000","last_fullsized":"2023-10-09T09:06:53.448759+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":80,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.685061+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:06:52.685061+0000","last_peered":"2023-10-09T09:06:52.685061+0000","last_clean":"2023-10-09T09:06:52.685061+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:06:52.685061+0000","last_undegraded":"2023-10-09T09:06:52.685061+0000","last_fullsized":"2023-10-09T09:06:52.685061+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":185,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:54.506644+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:06:54.506644+0000","last_peered":"2023-10-09T09:06:54.506644+0000","last_clean":"2023-10-09T09:06:54.506644+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:06:54.506644+0000","last_undegraded":"2023-10-09T09:06:54.506644+0000","last_fullsized":"2023-10-09T09:06:54.506644+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"218'2317","reported_seq":2332,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.867129+0000","last_change":"2023-10-09T09:06:27.617113+0000","last_active":"2023-10-09T09:07:20.867129+0000","last_peered":"2023-10-09T09:07:20.867129+0000","last_clean":"2023-10-09T09:07:20.867129+0000","last_became_active":"2023-10-09T09:06:27.615429+0000","last_became_peered":"2023-10-09T09:06:27.615429+0000","last_unstale":"2023-10-09T09:07:20.867129+0000","last_undegraded":"2023-10-09T09:07:20.867129+0000","last_fullsized":"2023-10-09T09:07:20.867129+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2317,"log_dups_size":0,"ondisk_log_size":2317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:07.620043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2317,"num_write_kb":148288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":243,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:54.506083+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:06:54.506083+0000","last_peered":"2023-10-09T09:06:54.506083+0000","last_clean":"2023-10-09T09:06:54.506083+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:06:54.506083+0000","last_undegraded":"2023-10-09T09:06:54.506083+0000","last_fullsized":"2023-10-09T09:06:54.506083+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":178,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.448204+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:06:53.448204+0000","last_peered":"2023-10-09T09:06:53.448204+0000","last_clean":"2023-10-09T09:06:53.448204+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:06:53.448204+0000","last_undegraded":"2023-10-09T09:06:53.448204+0000","last_fullsized":"2023-10-09T09:06:53.448204+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":225,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.933280+0000","last_change":"2023-10-09T09:03:19.575546+0000","last_active":"2023-10-09T09:06:53.933280+0000","last_peered":"2023-10-09T09:06:53.933280+0000","last_clean":"2023-10-09T09:06:53.933280+0000","last_became_active":"2023-10-09T09:03:19.575075+0000","last_became_peered":"2023-10-09T09:03:19.575075+0000","last_unstale":"2023-10-09T09:06:53.933280+0000","last_undegraded":"2023-10-09T09:06:53.933280+0000","last_fullsized":"2023-10-09T09:06:53.933280+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:14.350886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"218'2397","reported_seq":2412,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.831174+0000","last_change":"2023-10-09T09:06:27.620038+0000","last_active":"2023-10-09T09:07:20.831174+0000","last_peered":"2023-10-09T09:07:20.831174+0000","last_clean":"2023-10-09T09:07:20.831174+0000","last_became_active":"2023-10-09T09:06:27.619703+0000","last_became_peered":"2023-10-09T09:06:27.619703+0000","last_unstale":"2023-10-09T09:07:20.831174+0000","last_undegraded":"2023-10-09T09:07:20.831174+0000","last_fullsized":"2023-10-09T09:07:20.831174+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2397,"log_dups_size":0,"ondisk_log_size":2397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:51.146805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157089792,"num_objects":2397,"num_object_clones":0,"num_object_copies":7191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2397,"num_write_kb":153408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":149,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:54.506541+0000","last_change":"2023-10-09T09:01:36.274911+0000","last_active":"2023-10-09T09:06:54.506541+0000","last_peered":"2023-10-09T09:06:54.506541+0000","last_clean":"2023-10-09T09:06:54.506541+0000","last_became_active":"2023-10-09T09:01:36.274560+0000","last_became_peered":"2023-10-09T09:01:36.274560+0000","last_unstale":"2023-10-09T09:06:54.506541+0000","last_undegraded":"2023-10-09T09:06:54.506541+0000","last_fullsized":"2023-10-09T09:06:54.506541+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:55.305969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":458,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932115+0000","last_change":"2023-10-09T09:02:05.534070+0000","last_active":"2023-10-09T09:06:53.932115+0000","last_peered":"2023-10-09T09:06:53.932115+0000","last_clean":"2023-10-09T09:06:53.932115+0000","last_became_active":"2023-10-09T09:02:05.533567+0000","last_became_peered":"2023-10-09T09:02:05.533567+0000","last_unstale":"2023-10-09T09:06:53.932115+0000","last_undegraded":"2023-10-09T09:06:53.932115+0000","last_fullsized":"2023-10-09T09:06:53.932115+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:59.568077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333379+0000","last_change":"2023-10-09T09:03:19.562659+0000","last_active":"2023-10-09T09:06:53.333379+0000","last_peered":"2023-10-09T09:06:53.333379+0000","last_clean":"2023-10-09T09:06:53.333379+0000","last_became_active":"2023-10-09T09:03:19.562284+0000","last_became_peered":"2023-10-09T09:03:19.562284+0000","last_unstale":"2023-10-09T09:06:53.333379+0000","last_undegraded":"2023-10-09T09:06:53.333379+0000","last_fullsized":"2023-10-09T09:06:53.333379+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:26.537954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":67,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.450200+0000","last_change":"2023-10-09T09:00:31.859121+0000","last_active":"2023-10-09T09:06:53.450200+0000","last_peered":"2023-10-09T09:06:53.450200+0000","last_clean":"2023-10-09T09:06:53.450200+0000","last_became_active":"2023-10-09T09:00:31.858675+0000","last_became_peered":"2023-10-09T09:00:31.858675+0000","last_unstale":"2023-10-09T09:06:53.450200+0000","last_undegraded":"2023-10-09T09:06:53.450200+0000","last_fullsized":"2023-10-09T09:06:53.450200+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"218'2591","reported_seq":2606,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:22.683618+0000","last_change":"2023-10-09T09:06:27.620287+0000","last_active":"2023-10-09T09:07:22.683618+0000","last_peered":"2023-10-09T09:07:22.683618+0000","last_clean":"2023-10-09T09:07:22.683618+0000","last_became_active":"2023-10-09T09:06:27.619875+0000","last_became_peered":"2023-10-09T09:06:27.619875+0000","last_unstale":"2023-10-09T09:07:22.683618+0000","last_undegraded":"2023-10-09T09:07:22.683618+0000","last_fullsized":"2023-10-09T09:07:22.683618+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2591,"log_dups_size":0,"ondisk_log_size":2591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:37.325171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169803776,"num_objects":2591,"num_object_clones":0,"num_object_copies":7773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2591,"num_write_kb":165824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932401+0000","last_change":"2023-10-09T09:02:02.649488+0000","last_active":"2023-10-09T09:06:53.932401+0000","last_peered":"2023-10-09T09:06:53.932401+0000","last_clean":"2023-10-09T09:06:53.932401+0000","last_became_active":"2023-10-09T09:02:02.645550+0000","last_became_peered":"2023-10-09T09:02:02.645550+0000","last_unstale":"2023-10-09T09:06:53.932401+0000","last_undegraded":"2023-10-09T09:06:53.932401+0000","last_fullsized":"2023-10-09T09:06:53.932401+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:40.192731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":248,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932517+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:06:53.932517+0000","last_peered":"2023-10-09T09:06:53.932517+0000","last_clean":"2023-10-09T09:06:53.932517+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:06:53.932517+0000","last_undegraded":"2023-10-09T09:06:53.932517+0000","last_fullsized":"2023-10-09T09:06:53.932517+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":208,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.933165+0000","last_change":"2023-10-09T09:02:02.657733+0000","last_active":"2023-10-09T09:06:53.933165+0000","last_peered":"2023-10-09T09:06:53.933165+0000","last_clean":"2023-10-09T09:06:53.933165+0000","last_became_active":"2023-10-09T09:02:02.656701+0000","last_became_peered":"2023-10-09T09:02:02.656701+0000","last_unstale":"2023-10-09T09:06:53.933165+0000","last_undegraded":"2023-10-09T09:06:53.933165+0000","last_fullsized":"2023-10-09T09:06:53.933165+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:39.552401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"218'2698","reported_seq":2712,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:25.342838+0000","last_change":"2023-10-09T09:06:27.613599+0000","last_active":"2023-10-09T09:07:25.342838+0000","last_peered":"2023-10-09T09:07:25.342838+0000","last_clean":"2023-10-09T09:07:25.342838+0000","last_became_active":"2023-10-09T09:06:27.613327+0000","last_became_peered":"2023-10-09T09:06:27.613327+0000","last_unstale":"2023-10-09T09:07:25.342838+0000","last_undegraded":"2023-10-09T09:07:25.342838+0000","last_fullsized":"2023-10-09T09:07:25.342838+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2698,"log_dups_size":0,"ondisk_log_size":2698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176816128,"num_objects":2698,"num_object_clones":0,"num_object_copies":8094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2698,"num_write_kb":172672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":192,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932969+0000","last_change":"2023-10-09T09:02:02.644204+0000","last_active":"2023-10-09T09:06:53.932969+0000","last_peered":"2023-10-09T09:06:53.932969+0000","last_clean":"2023-10-09T09:06:53.932969+0000","last_became_active":"2023-10-09T09:02:02.643713+0000","last_became_peered":"2023-10-09T09:02:02.643713+0000","last_unstale":"2023-10-09T09:06:53.932969+0000","last_undegraded":"2023-10-09T09:06:53.932969+0000","last_fullsized":"2023-10-09T09:06:53.932969+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:20.811353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":187,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:54.506487+0000","last_change":"2023-10-09T09:02:02.647803+0000","last_active":"2023-10-09T09:06:54.506487+0000","last_peered":"2023-10-09T09:06:54.506487+0000","last_clean":"2023-10-09T09:06:54.506487+0000","last_became_active":"2023-10-09T09:02:02.646911+0000","last_became_peered":"2023-10-09T09:02:02.646911+0000","last_unstale":"2023-10-09T09:06:54.506487+0000","last_undegraded":"2023-10-09T09:06:54.506487+0000","last_fullsized":"2023-10-09T09:06:54.506487+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:35.294072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":206,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932775+0000","last_change":"2023-10-09T09:02:52.016015+0000","last_active":"2023-10-09T09:06:53.932775+0000","last_peered":"2023-10-09T09:06:53.932775+0000","last_clean":"2023-10-09T09:06:53.932775+0000","last_became_active":"2023-10-09T09:02:52.015543+0000","last_became_peered":"2023-10-09T09:02:52.015543+0000","last_unstale":"2023-10-09T09:06:53.932775+0000","last_undegraded":"2023-10-09T09:06:53.932775+0000","last_fullsized":"2023-10-09T09:06:53.932775+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.410980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"218'2633","reported_seq":2646,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:25.380755+0000","last_change":"2023-10-09T09:06:27.615162+0000","last_active":"2023-10-09T09:07:25.380755+0000","last_peered":"2023-10-09T09:07:25.380755+0000","last_clean":"2023-10-09T09:07:25.380755+0000","last_became_active":"2023-10-09T09:06:27.614815+0000","last_became_peered":"2023-10-09T09:06:27.614815+0000","last_unstale":"2023-10-09T09:07:25.380755+0000","last_undegraded":"2023-10-09T09:07:25.380755+0000","last_fullsized":"2023-10-09T09:07:25.380755+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2633,"log_dups_size":0,"ondisk_log_size":2633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172556288,"num_objects":2633,"num_object_clones":0,"num_object_copies":7899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2633,"num_write_kb":168512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":17,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:54.506926+0000","last_change":"2023-10-09T09:06:50.681508+0000","last_active":"2023-10-09T09:06:54.506926+0000","last_peered":"2023-10-09T09:06:54.506926+0000","last_clean":"2023-10-09T09:06:54.506926+0000","last_became_active":"2023-10-09T09:06:50.678633+0000","last_became_peered":"2023-10-09T09:06:50.678633+0000","last_unstale":"2023-10-09T09:06:54.506926+0000","last_undegraded":"2023-10-09T09:06:54.506926+0000","last_fullsized":"2023-10-09T09:06:54.506926+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:02.327990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":180,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.449908+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:06:53.449908+0000","last_peered":"2023-10-09T09:06:53.449908+0000","last_clean":"2023-10-09T09:06:53.449908+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:06:53.449908+0000","last_undegraded":"2023-10-09T09:06:53.449908+0000","last_fullsized":"2023-10-09T09:06:53.449908+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":37,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.113306+0000","last_change":"2023-10-09T09:03:19.572475+0000","last_active":"2023-10-09T09:06:59.113306+0000","last_peered":"2023-10-09T09:06:59.113306+0000","last_clean":"2023-10-09T09:06:59.113306+0000","last_became_active":"2023-10-09T09:03:19.572081+0000","last_became_peered":"2023-10-09T09:03:19.572081+0000","last_unstale":"2023-10-09T09:06:59.113306+0000","last_undegraded":"2023-10-09T09:06:59.113306+0000","last_fullsized":"2023-10-09T09:06:59.113306+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:04.771178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"218'2264","reported_seq":2278,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.870127+0000","last_change":"2023-10-09T09:06:27.616627+0000","last_active":"2023-10-09T09:07:20.870127+0000","last_peered":"2023-10-09T09:07:20.870127+0000","last_clean":"2023-10-09T09:07:20.870127+0000","last_became_active":"2023-10-09T09:06:27.616284+0000","last_became_peered":"2023-10-09T09:06:27.616284+0000","last_unstale":"2023-10-09T09:07:20.870127+0000","last_undegraded":"2023-10-09T09:07:20.870127+0000","last_fullsized":"2023-10-09T09:07:20.870127+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":2264,"log_dups_size":0,"ondisk_log_size":2264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:23.626143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2264,"num_write_kb":144896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":67,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.450432+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:06:53.450432+0000","last_peered":"2023-10-09T09:06:53.450432+0000","last_clean":"2023-10-09T09:06:53.450432+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:06:53.450432+0000","last_undegraded":"2023-10-09T09:06:53.450432+0000","last_fullsized":"2023-10-09T09:06:53.450432+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":227,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.933173+0000","last_change":"2023-10-09T09:02:02.674657+0000","last_active":"2023-10-09T09:06:53.933173+0000","last_peered":"2023-10-09T09:06:53.933173+0000","last_clean":"2023-10-09T09:06:53.933173+0000","last_became_active":"2023-10-09T09:02:02.657392+0000","last_became_peered":"2023-10-09T09:02:02.657392+0000","last_unstale":"2023-10-09T09:06:53.933173+0000","last_undegraded":"2023-10-09T09:06:53.933173+0000","last_fullsized":"2023-10-09T09:06:53.933173+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:16.286583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":89,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684760+0000","last_change":"2023-10-09T09:01:36.280537+0000","last_active":"2023-10-09T09:06:52.684760+0000","last_peered":"2023-10-09T09:06:52.684760+0000","last_clean":"2023-10-09T09:06:52.684760+0000","last_became_active":"2023-10-09T09:01:36.280149+0000","last_became_peered":"2023-10-09T09:01:36.280149+0000","last_unstale":"2023-10-09T09:06:52.684760+0000","last_undegraded":"2023-10-09T09:06:52.684760+0000","last_fullsized":"2023-10-09T09:06:52.684760+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":226,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.932745+0000","last_change":"2023-10-09T09:03:19.554339+0000","last_active":"2023-10-09T09:06:53.932745+0000","last_peered":"2023-10-09T09:06:53.932745+0000","last_clean":"2023-10-09T09:06:53.932745+0000","last_became_active":"2023-10-09T09:03:19.553953+0000","last_became_peered":"2023-10-09T09:03:19.553953+0000","last_unstale":"2023-10-09T09:06:53.932745+0000","last_undegraded":"2023-10-09T09:06:53.932745+0000","last_fullsized":"2023-10-09T09:06:53.932745+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:50.601515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":48,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:54.506370+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:06:54.506370+0000","last_peered":"2023-10-09T09:06:54.506370+0000","last_clean":"2023-10-09T09:06:54.506370+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:06:54.506370+0000","last_undegraded":"2023-10-09T09:06:54.506370+0000","last_fullsized":"2023-10-09T09:06:54.506370+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":220,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684702+0000","last_change":"2023-10-09T09:02:52.022788+0000","last_active":"2023-10-09T09:06:52.684702+0000","last_peered":"2023-10-09T09:06:52.684702+0000","last_clean":"2023-10-09T09:06:52.684702+0000","last_became_active":"2023-10-09T09:02:52.016418+0000","last_became_peered":"2023-10-09T09:02:52.016418+0000","last_unstale":"2023-10-09T09:06:52.684702+0000","last_undegraded":"2023-10-09T09:06:52.684702+0000","last_fullsized":"2023-10-09T09:06:52.684702+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":80,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684950+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:06:52.684950+0000","last_peered":"2023-10-09T09:06:52.684950+0000","last_clean":"2023-10-09T09:06:52.684950+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:06:52.684950+0000","last_undegraded":"2023-10-09T09:06:52.684950+0000","last_fullsized":"2023-10-09T09:06:52.684950+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":205,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684849+0000","last_change":"2023-10-09T09:03:19.575218+0000","last_active":"2023-10-09T09:06:52.684849+0000","last_peered":"2023-10-09T09:06:52.684849+0000","last_clean":"2023-10-09T09:06:52.684849+0000","last_became_active":"2023-10-09T09:03:19.557578+0000","last_became_peered":"2023-10-09T09:03:19.557578+0000","last_unstale":"2023-10-09T09:06:52.684849+0000","last_undegraded":"2023-10-09T09:06:52.684849+0000","last_fullsized":"2023-10-09T09:06:52.684849+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":71,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684306+0000","last_change":"2023-10-09T08:59:53.522794+0000","last_active":"2023-10-09T09:06:52.684306+0000","last_peered":"2023-10-09T09:06:52.684306+0000","last_clean":"2023-10-09T09:06:52.684306+0000","last_became_active":"2023-10-09T08:59:53.522419+0000","last_became_peered":"2023-10-09T08:59:53.522419+0000","last_unstale":"2023-10-09T09:06:52.684306+0000","last_undegraded":"2023-10-09T09:06:52.684306+0000","last_fullsized":"2023-10-09T09:06:52.684306+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":220,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.116363+0000","last_change":"2023-10-09T09:00:31.845771+0000","last_active":"2023-10-09T09:06:59.116363+0000","last_peered":"2023-10-09T09:06:59.116363+0000","last_clean":"2023-10-09T09:06:59.116363+0000","last_became_active":"2023-10-09T09:00:31.844775+0000","last_became_peered":"2023-10-09T09:00:31.844775+0000","last_unstale":"2023-10-09T09:06:59.116363+0000","last_undegraded":"2023-10-09T09:06:59.116363+0000","last_fullsized":"2023-10-09T09:06:59.116363+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:33.452539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":36,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.333183+0000","last_change":"2023-10-09T09:03:19.561924+0000","last_active":"2023-10-09T09:06:53.333183+0000","last_peered":"2023-10-09T09:06:53.333183+0000","last_clean":"2023-10-09T09:06:53.333183+0000","last_became_active":"2023-10-09T09:03:19.557692+0000","last_became_peered":"2023-10-09T09:03:19.557692+0000","last_unstale":"2023-10-09T09:06:53.333183+0000","last_undegraded":"2023-10-09T09:06:53.333183+0000","last_fullsized":"2023-10-09T09:06:53.333183+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:39.940109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":279,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:53.933336+0000","last_change":"2023-10-09T09:02:52.019620+0000","last_active":"2023-10-09T09:06:53.933336+0000","last_peered":"2023-10-09T09:06:53.933336+0000","last_clean":"2023-10-09T09:06:53.933336+0000","last_became_active":"2023-10-09T09:02:52.014489+0000","last_became_peered":"2023-10-09T09:02:52.014489+0000","last_unstale":"2023-10-09T09:06:53.933336+0000","last_undegraded":"2023-10-09T09:06:53.933336+0000","last_fullsized":"2023-10-09T09:06:53.933336+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:39.550600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T09:06:52.684252+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:06:52.684252+0000","last_peered":"2023-10-09T09:06:52.684252+0000","last_clean":"2023-10-09T09:06:52.684252+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:06:52.684252+0000","last_undegraded":"2023-10-09T09:06:52.684252+0000","last_fullsized":"2023-10-09T09:06:52.684252+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":13474,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":139264,"data_stored":22108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2566324224,"num_objects":39159,"num_object_clones":0,"num_object_copies":117477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39159,"num_write_kb":2506176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7726366720,"data_stored":7726366720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39159,"ondisk_log_size":39159,"up":48,"acting":48,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430643,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626875,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2558240,"kb_used_data":1650776,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91186912,"statfs":{"total":95995035648,"available":93375397888,"internally_reserved":0,"allocated":1690394624,"data_stored":1689895992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.1639999999999999,"15min":2.3149999999999999},"min":{"1min":0.36799999999999999,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":1.1830000000000001,"5min":5.9459999999999997,"15min":153.63800000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.397,"15min":2.351},"min":{"1min":0.626,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.2529999999999999,"5min":9.1140000000000008,"15min":153.72200000000001},"last":0.86799999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:06:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.69899999999999995,"15min":1.8160000000000001},"min":{"1min":0.42199999999999999,"5min":0.32000000000000001,"15min":0.28299999999999997},"max":{"1min":1.236,"5min":3.6739999999999999,"15min":109.453},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.96499999999999997,"15min":1.931},"min":{"1min":0.46600000000000003,"5min":0.316,"15min":0.314},"max":{"1min":1.28,"5min":9.1869999999999994,"15min":111.47499999999999},"last":0.67100000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":1.04,"15min":1.393},"min":{"1min":0.45800000000000002,"5min":0.31900000000000001,"15min":0.29799999999999999},"max":{"1min":1.125,"5min":5.6820000000000004,"15min":8.3979999999999997},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.998,"5min":1.1719999999999999,"15min":1.488},"min":{"1min":0.49399999999999999,"5min":0.30199999999999999,"15min":0.28100000000000003},"max":{"1min":1.881,"5min":9.1549999999999994,"15min":12.194000000000001},"last":0.78900000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:06:57 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.2629999999999999,"15min":1.8480000000000001},"min":{"1min":0.54100000000000004,"5min":0.311,"15min":0.311},"max":{"1min":4.3029999999999999,"5min":6.992,"15min":55.713000000000001},"last":1.679},{"interface":"front","average":{"1min":1.127,"5min":1.6459999999999999,"15min":1.732},"min":{"1min":0.68000000000000005,"5min":0.46500000000000002,"15min":0.34699999999999998},"max":{"1min":5.1639999999999997,"5min":9.4429999999999996,"15min":56.064},"last":0.71499999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:06:42 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.4430000000000001,"15min":1.534},"min":{"1min":0.59999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":1.796,"5min":32.280999999999999,"15min":32.280999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.01,"5min":1.2889999999999999,"15min":1.177},"min":{"1min":0.70599999999999996,"5min":0.35399999999999998,"15min":0.30599999999999999},"max":{"1min":1.4299999999999999,"5min":34.689,"15min":34.689},"last":0.59599999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:06:42 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.0660000000000001,"15min":1.2949999999999999},"min":{"1min":0.65600000000000003,"5min":0.40300000000000002,"15min":0.35499999999999998},"max":{"1min":1.325,"5min":9.218,"15min":12.404999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.0509999999999999,"5min":0.93500000000000005,"15min":1.115},"min":{"1min":0.64300000000000002,"5min":0.41599999999999998,"15min":0.33900000000000002},"max":{"1min":1.534,"5min":4.2610000000000001,"15min":8.8569999999999993},"last":0.64800000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:06:36 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.081,"15min":1.425},"min":{"1min":0.75600000000000001,"5min":0.434,"15min":0.40500000000000003},"max":{"1min":1.419,"5min":9.2460000000000004,"15min":11.382},"last":0.81499999999999995},{"interface":"front","average":{"1min":1.097,"5min":0.96599999999999997,"15min":1.2350000000000001},"min":{"1min":0.77400000000000002,"5min":0.45400000000000001,"15min":0.377},"max":{"1min":1.621,"5min":4.3259999999999996,"15min":9.4800000000000004},"last":0.91100000000000003}]}]},{"osd":1,"up_from":0,"seq":356482285823,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449349,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2181020,"kb_used_data":1718484,"kb_used_omap":28,"kb_used_meta":462435,"kb_avail":91564132,"statfs":{"total":95995035648,"available":93761671168,"internally_reserved":0,"allocated":1759727616,"data_stored":1758881240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":473534288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.113,"15min":1.4610000000000001},"min":{"1min":0.28799999999999998,"5min":0.22,"15min":0.22},"max":{"1min":1.097,"5min":6.7649999999999997,"15min":9.9879999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.1240000000000001,"15min":1.512},"min":{"1min":0.27200000000000002,"5min":0.254,"15min":0.21199999999999999},"max":{"1min":1.1879999999999999,"5min":10.102,"15min":18.661000000000001},"last":0.47799999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":2.8420000000000001,"15min":2.4430000000000001},"min":{"1min":0.29199999999999998,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":1.056,"5min":71.322999999999993,"15min":71.322999999999993},"last":0.495},{"interface":"front","average":{"1min":0.67400000000000004,"5min":5.21,"15min":3.3410000000000002},"min":{"1min":0.377,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.143,"5min":219.00700000000001,"15min":219.00700000000001},"last":1.0900000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":4.4409999999999998,"15min":3.137},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":3.7519999999999998,"5min":219.041,"15min":219.041},"last":8.9079999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":2.0379999999999998,"15min":2.3029999999999999},"min":{"1min":0.38500000000000001,"5min":0.28899999999999998,"15min":0.25},"max":{"1min":1.091,"5min":20.466000000000001,"15min":107.66500000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.494,"15min":1.5029999999999999},"min":{"1min":0.51700000000000002,"5min":0.40200000000000002,"15min":0.30199999999999999},"max":{"1min":1.268,"5min":10.178000000000001,"15min":10.178000000000001},"last":0.745},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.419,"15min":1.6699999999999999},"min":{"1min":0.53300000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.129,"5min":10.526,"15min":11.654},"last":0.997}]},{"osd":5,"last update":"Mon Oct 9 09:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.5,"15min":1.4079999999999999},"min":{"1min":0.54700000000000004,"5min":0.40500000000000003,"15min":0.34499999999999997},"max":{"1min":1.304,"5min":9.3030000000000008,"15min":9.3689999999999998},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.5409999999999999,"15min":1.605},"min":{"1min":0.496,"5min":0.46000000000000002,"15min":0.33700000000000002},"max":{"1min":1.4279999999999999,"5min":10.140000000000001,"15min":10.140000000000001},"last":1.046}]},{"osd":6,"last update":"Mon Oct 9 09:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":2.3969999999999998,"15min":1.8660000000000001},"min":{"1min":0.48899999999999999,"5min":0.37,"15min":0.37},"max":{"1min":1.425,"5min":62.850999999999999,"15min":62.850999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.98799999999999999,"5min":2.4289999999999998,"15min":2.081},"min":{"1min":0.55400000000000005,"5min":0.434,"15min":0.36699999999999999},"max":{"1min":1.4359999999999999,"5min":62.816000000000003,"15min":62.816000000000003},"last":0.79000000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.5429999999999999,"15min":1.647},"min":{"1min":0.63600000000000001,"5min":0.495,"15min":0.39400000000000002},"max":{"1min":1.3799999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.427,"15min":1.8440000000000001},"min":{"1min":0.59799999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.218,"5min":10.662000000000001,"15min":14.359},"last":0.90500000000000003}]}]},{"osd":2,"up_from":88,"seq":377957122297,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1806096,"kb_used_data":1277704,"kb_used_omap":81,"kb_used_meta":528302,"kb_avail":91939056,"statfs":{"total":95995035648,"available":94145593344,"internally_reserved":0,"allocated":1308368896,"data_stored":1307479210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":540981466},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:06:38 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":3.641,"15min":3.306},"min":{"1min":0.23799999999999999,"5min":0.20699999999999999,"15min":0.191},"max":{"1min":1.925,"5min":68.457999999999998,"15min":134.458},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":3.4809999999999999,"15min":2.9300000000000002},"min":{"1min":0.36499999999999999,"5min":0.26000000000000001,"15min":0.221},"max":{"1min":2.0110000000000001,"5min":68.162999999999997,"15min":138.417},"last":3.2229999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:06:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":2.1899999999999999,"15min":2.2890000000000001},"min":{"1min":0.372,"5min":0.27100000000000002,"15min":0.21299999999999999},"max":{"1min":2.0640000000000001,"5min":57.939999999999998,"15min":57.939999999999998},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.8149999999999999,"15min":2.0649999999999999},"min":{"1min":0.32000000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.9510000000000001,"5min":56.667999999999999,"15min":57.122999999999998},"last":3.1880000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:06:55 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":4.2990000000000004,"15min":2.6669999999999998},"min":{"1min":0.43099999999999999,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":2.1280000000000001,"5min":112.47499999999999,"15min":112.47499999999999},"last":3.1429999999999998},{"interface":"front","average":{"1min":0.89800000000000002,"5min":4.7699999999999996,"15min":2.9700000000000002},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.247},"max":{"1min":5.2450000000000001,"5min":112.929,"15min":112.929},"last":0.27000000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:06:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.8839999999999999,"15min":1.9610000000000001},"min":{"1min":0.47999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":1.9750000000000001,"5min":17.408999999999999,"15min":17.408999999999999},"last":0.374},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.6950000000000001,"15min":1.833},"min":{"1min":0.40200000000000002,"5min":0.38800000000000001,"15min":0.314},"max":{"1min":2.222,"5min":9.2240000000000002,"15min":14.071999999999999},"last":0.495}]},{"osd":5,"last update":"Mon Oct 9 09:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.3129999999999999,"15min":1.4630000000000001},"min":{"1min":0.56799999999999995,"5min":0.41499999999999998,"15min":0.33900000000000002},"max":{"1min":2.4220000000000002,"5min":8.8239999999999998,"15min":12.411},"last":3.266},{"interface":"front","average":{"1min":1.02,"5min":1.881,"15min":1.9570000000000001},"min":{"1min":0.51700000000000002,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":2.3820000000000001,"5min":17.375,"15min":17.375},"last":0.64400000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.6930000000000001,"15min":1.859},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.33900000000000002},"max":{"1min":2.319,"5min":9.1649999999999991,"15min":10.456},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.504,"15min":1.5880000000000001},"min":{"1min":0.5,"5min":0.41499999999999998,"15min":0.39300000000000002},"max":{"1min":2.339,"5min":11.869999999999999,"15min":12.381},"last":3.2469999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":2.3929999999999998,"15min":2.073},"min":{"1min":0.498,"5min":0.42299999999999999,"15min":0.38800000000000001},"max":{"1min":2.3519999999999999,"5min":67.491,"15min":67.491},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.476,"15min":1.534},"min":{"1min":0.66800000000000004,"5min":0.44400000000000001,"15min":0.38500000000000001},"max":{"1min":2.504,"5min":11.933,"15min":12.438000000000001},"last":3.2850000000000001}]}]},{"osd":3,"up_from":93,"seq":399431958771,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2267332,"kb_used_data":1790908,"kb_used_omap":7,"kb_used_meta":476408,"kb_avail":91477820,"statfs":{"total":95995035648,"available":93673287680,"internally_reserved":0,"allocated":1833889792,"data_stored":1833064433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":487842408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":3.7919999999999998,"15min":2.3399999999999999},"min":{"1min":0.30399999999999999,"5min":0.19500000000000001,"15min":0.19400000000000001},"max":{"1min":1.3360000000000001,"5min":233.22900000000001,"15min":233.22900000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":4.0899999999999999,"15min":2.484},"min":{"1min":0.39600000000000002,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":1.2110000000000001,"5min":237.447,"15min":237.447},"last":8.0790000000000006}]},{"osd":1,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":4.6820000000000004,"15min":2.6949999999999998},"min":{"1min":0.315,"5min":0.31,"15min":0.22500000000000001},"max":{"1min":1,"5min":326.75700000000001,"15min":326.75700000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.72499999999999998,"5min":4.4370000000000003,"15min":2.6059999999999999},"min":{"1min":0.39700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.5529999999999999,"5min":327.024,"15min":327.024},"last":0.58899999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":2.431,"15min":2.4089999999999998},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.255},"max":{"1min":1.071,"5min":43.683999999999997,"15min":75.641999999999996},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.70199999999999996,"5min":2.008,"15min":2.1960000000000002},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.034,"5min":45.027999999999999,"15min":75.207999999999998},"last":0.53700000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.1819999999999999,"15min":1.387},"min":{"1min":0.65300000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.208,"5min":8.9879999999999995,"15min":9.2940000000000005},"last":0.628},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.5489999999999999,"15min":1.6519999999999999},"min":{"1min":0.59299999999999997,"5min":0.36399999999999999,"15min":0.33500000000000002},"max":{"1min":1.0720000000000001,"5min":9.4649999999999999,"15min":10.486000000000001},"last":6.5549999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.415,"15min":1.544},"min":{"1min":0.66200000000000003,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":1.21,"5min":9.4990000000000006,"15min":10.577},"last":7.9509999999999996},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.571,"15min":1.657},"min":{"1min":0.55700000000000005,"5min":0.44900000000000001,"15min":0.34300000000000003},"max":{"1min":1.262,"5min":7.5590000000000002,"15min":9.1189999999999998},"last":0.41399999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.639,"15min":1.7310000000000001},"min":{"1min":0.72199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.3069999999999999,"5min":9.1370000000000005,"15min":10.537000000000001},"last":8.0470000000000006},{"interface":"front","average":{"1min":1.032,"5min":1.8859999999999999,"15min":1.833},"min":{"1min":0.52600000000000002,"5min":0.40699999999999997,"15min":0.35099999999999998},"max":{"1min":1.3120000000000001,"5min":9.5090000000000003,"15min":9.5090000000000003},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.6499999999999999,"15min":1.6839999999999999},"min":{"1min":0.65300000000000002,"5min":0.52000000000000002,"15min":0.41299999999999998},"max":{"1min":1.3759999999999999,"5min":7.5899999999999999,"15min":10.284000000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.04,"5min":1.371,"15min":1.5629999999999999},"min":{"1min":0.64300000000000002,"5min":0.47199999999999998,"15min":0.435},"max":{"1min":1.4019999999999999,"5min":9.1560000000000006,"15min":13.339},"last":0.60699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019342,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1872224,"kb_used_data":1278424,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91872928,"statfs":{"total":95995035648,"available":94077878272,"internally_reserved":0,"allocated":1309106176,"data_stored":1308606851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:07:23 2023","interfaces":[{"interface":"back","average":{"1min":3.2400000000000002,"5min":3.2959999999999998,"15min":2.355},"min":{"1min":0.307,"5min":0.248,"15min":0.248},"max":{"1min":46.463000000000001,"5min":141.07400000000001,"15min":141.07400000000001},"last":0.307},{"interface":"front","average":{"1min":3.427,"5min":3.1909999999999998,"15min":2.4249999999999998},"min":{"1min":0.38100000000000001,"5min":0.30399999999999999,"15min":0.255},"max":{"1min":46.823999999999998,"5min":141.15600000000001,"15min":141.15600000000001},"last":1.9990000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.72299999999999998,"15min":1.3580000000000001},"min":{"1min":0.56000000000000005,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":1.093,"5min":2.8199999999999998,"15min":8.9169999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.0129999999999999,"15min":1.476},"min":{"1min":0.57899999999999996,"5min":0.318,"15min":0.29199999999999998},"max":{"1min":1.5349999999999999,"5min":7.6879999999999997,"15min":11.712999999999999},"last":0.46200000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:07:23 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":2.819,"15min":2.1419999999999999},"min":{"1min":0.42699999999999999,"5min":0.34899999999999998,"15min":0.30499999999999999},"max":{"1min":3.6309999999999998,"5min":142.816,"15min":142.816},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.361,"5min":2.7450000000000001,"15min":1.9490000000000001},"min":{"1min":0.52600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.0149999999999997,"5min":143.81800000000001,"15min":143.81800000000001},"last":7.0149999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":2.7090000000000001,"15min":2.1190000000000002},"min":{"1min":0.53200000000000003,"5min":0.31900000000000001,"15min":0.29499999999999998},"max":{"1min":1.2130000000000001,"5min":143.13399999999999,"15min":143.13399999999999},"last":0.497},{"interface":"front","average":{"1min":1.0409999999999999,"5min":3.2370000000000001,"15min":2.1589999999999998},"min":{"1min":0.67400000000000004,"5min":0.41299999999999998,"15min":0.35399999999999998},"max":{"1min":1.29,"5min":152.37299999999999,"15min":152.37299999999999},"last":3.6520000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.0640000000000001,"15min":1.242},"min":{"1min":0.45800000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.6459999999999999,"5min":7.7439999999999998,"15min":11.762},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.95799999999999996,"15min":1.2909999999999999},"min":{"1min":0.40999999999999998,"5min":0.37,"15min":0.28699999999999998},"max":{"1min":1.3680000000000001,"5min":4.1379999999999999,"15min":12.85},"last":2.0579999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.92200000000000004,"15min":1.401},"min":{"1min":0.61399999999999999,"5min":0.35699999999999998,"15min":0.32100000000000001},"max":{"1min":1.5109999999999999,"5min":5.1020000000000003,"15min":10.657999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.133,"15min":1.456},"min":{"1min":0.61299999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.4319999999999999,"5min":7.6470000000000002,"15min":14.755000000000001},"last":0.50800000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.0189999999999999,"15min":1.407},"min":{"1min":0.52300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.125,"5min":3.9039999999999999,"15min":13.933999999999999},"last":2.125},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.97199999999999998,"15min":1.484},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.33200000000000002},"max":{"1min":1.407,"5min":5.1520000000000001,"15min":13.321999999999999},"last":0.57499999999999996}]}]},{"osd":5,"up_from":0,"seq":154618823110,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1798897664,"data_stored":1798897664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1329922048,"data_stored":1329922048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1842085888,"data_stored":1842085888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1052049408,"data_stored":1052049408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1703411712,"data_stored":1703411712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:07:32.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:32.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:32.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:32.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:33.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:33.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:33.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:33.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:33.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:33.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:33 smithi016 bash[75179]: cluster 2023-10-09T09:07:31.930591+0000 mgr.y (mgr.25421) 904 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:07:33.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:33 smithi016 bash[77892]: cluster 2023-10-09T09:07:31.930591+0000 mgr.y (mgr.25421) 904 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:07:33.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:33.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:33.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:33.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:33.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:33.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 51194 51178 88.8394 68.75 0.0182971 0.0112491 2023-10-09T09:07:33.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:33.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:33 smithi094 bash[59677]: cluster 2023-10-09T09:07:31.930591+0000 mgr.y (mgr.25421) 904 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:07:33.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:33.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:33.625 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:33.653 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:33.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:33.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:33.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:33.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:33.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:33.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:33.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:33.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:34.054 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:34.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:34.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:34.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:34.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:34.214 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:34.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:34.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:34.348 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:34.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:34.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:34.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 52307 52291 88.3182 69.5625 0.0112871 0.0113162 2023-10-09T09:07:34.453 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:34.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:34.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:34.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:34.611 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:34.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:34.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:34.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:34.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:34.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:34.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:07:34.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:34.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:07:34.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:34.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:34.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:35.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:35.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:35.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:35.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:35.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:35.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:35.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:35.283 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:35 smithi094 bash[59677]: cluster 2023-10-09T09:07:33.931273+0000 mgr.y (mgr.25421) 905 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:07:35.283 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:35 smithi094 bash[59677]: audit 2023-10-09T09:07:34.248572+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:35.283 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:35 smithi094 bash[59677]: audit 2023-10-09T09:07:34.250084+0000 mon.a (mon.0) 1222 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:35.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:35.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 53324 53308 87.6666 63.5625 0.00819319 0.0113999 2023-10-09T09:07:35.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:35.410 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:35 smithi016 bash[75179]: cluster 2023-10-09T09:07:33.931273+0000 mgr.y (mgr.25421) 905 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:07:35.410 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:35 smithi016 bash[75179]: audit 2023-10-09T09:07:34.248572+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:35.410 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:35 smithi016 bash[75179]: audit 2023-10-09T09:07:34.250084+0000 mon.a (mon.0) 1222 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:35.410 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:35 smithi016 bash[77892]: cluster 2023-10-09T09:07:33.931273+0000 mgr.y (mgr.25421) 905 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:07:35.410 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:35 smithi016 bash[77892]: audit 2023-10-09T09:07:34.248572+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:07:35.410 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:35 smithi016 bash[77892]: audit 2023-10-09T09:07:34.250084+0000 mon.a (mon.0) 1222 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:35.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:35.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:35.565 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:35.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:35.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:35.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:35.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:35.795 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:35.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:35.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:35.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:07:35.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:35.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:35.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:35.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:07:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:07:35] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.33.4" 2023-10-09T09:07:36.067 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:36.087 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:36.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:36.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:36.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:36.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:36.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:36.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:36.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:36.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:36.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:36.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 54359 54343 87.0772 64.6875 0.0104641 0.0114777 2023-10-09T09:07:36.460 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:36.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:36.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:36.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:36.618 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:36.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:36.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:36.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:36.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:36.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:36.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:36.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:36.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:36.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:37.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:37.241 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:37.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:37.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:37 smithi094 bash[59677]: cluster 2023-10-09T09:07:35.931881+0000 mgr.y (mgr.25421) 906 : cluster 0 pgmap v951: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:07:37.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:07:37.395050+0000 min lat: 0.00143057 max lat: 0.310598 avg lat: 0.0115416 2023-10-09T09:07:37.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:07:37.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 55446 55430 86.5986 67.9375 0.0175384 0.0115416 2023-10-09T09:07:37.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:37 smithi016 bash[75179]: cluster 2023-10-09T09:07:35.931881+0000 mgr.y (mgr.25421) 906 : cluster 0 pgmap v951: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:07:37.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:37 smithi016 bash[77892]: cluster 2023-10-09T09:07:35.931881+0000 mgr.y (mgr.25421) 906 : cluster 0 pgmap v951: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:07:37.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:37.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:37.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:37.604 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:37.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:37.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:37.766 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:37.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:37.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:37.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.916 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:37.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:38.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:38.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:38.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:38.154 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:38.179 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:38.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:38.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:38.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 56464 56448 86.0382 63.625 0.022061 0.0116174 2023-10-09T09:07:38.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:38.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:38.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:38.538 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:38.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:38.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:38.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:38.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:38.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:38.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:38.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:38.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:38.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:38.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:39.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:39.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:39.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:39.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:39.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:39.241 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:39.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:39.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:39.366 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:39.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:39.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:39.395 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 57438 57422 85.4389 60.875 0.0164001 0.0116987 2023-10-09T09:07:39.470 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:39.480 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:39 smithi094 bash[59677]: cluster 2023-10-09T09:07:37.932567+0000 mgr.y (mgr.25421) 907 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T09:07:39.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:39.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:39.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:39.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:39.623 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:39.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:39.643 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:39.644 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:39.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:39 smithi016 bash[75179]: cluster 2023-10-09T09:07:37.932567+0000 mgr.y (mgr.25421) 907 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T09:07:39.676 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:39 smithi016 bash[77892]: cluster 2023-10-09T09:07:37.932567+0000 mgr.y (mgr.25421) 907 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T09:07:39.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:39.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:39.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:39.868 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:39.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:39.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:39.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:40.000 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:40.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:40.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:40.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:40.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:40.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:40.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:40.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 58153 58137 84.4911 44.6875 0.0210125 0.0118288 2023-10-09T09:07:40.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:40.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:40.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:40.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:40.666 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:07:40.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:40.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:40.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:40 smithi016 bash[75179]: cluster 2023-10-09T09:07:39.933081+0000 mgr.y (mgr.25421) 908 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T09:07:40.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:40.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:40.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:40.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:40.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:40.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:40.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:40 smithi094 bash[59677]: cluster 2023-10-09T09:07:39.933081+0000 mgr.y (mgr.25421) 908 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T09:07:41.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.124 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:41.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:41.229 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:40 smithi016 bash[77892]: cluster 2023-10-09T09:07:39.933081+0000 mgr.y (mgr.25421) 908 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T09:07:41.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:41.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:41.356 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:41.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:41.398 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 59220 59204 84.0863 66.6875 0.0120267 0.0118868 2023-10-09T09:07:41.486 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:41.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:41.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:41.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:41.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:41.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:41.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:41.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:41.879 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.903 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:41.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:41.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:42.008 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:42.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:42.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:42.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:42.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:42.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:42.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:42.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 60158 60142 83.5204 58.625 0.00982677 0.0119662 2023-10-09T09:07:42.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:42.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:42.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:42.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:42.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:42.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:42.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:42.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:42.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:42.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:42.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:42.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:42.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:42.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:42.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:42.991 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:42 smithi016 bash[75179]: cluster 2023-10-09T09:07:41.934015+0000 mgr.y (mgr.25421) 909 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T09:07:42.992 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:42 smithi016 bash[77892]: cluster 2023-10-09T09:07:41.934015+0000 mgr.y (mgr.25421) 909 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T09:07:43.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:43.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:43.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:43.173 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:43.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:42 smithi094 bash[59677]: cluster 2023-10-09T09:07:41.934015+0000 mgr.y (mgr.25421) 909 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T09:07:43.199 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:43.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:43.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:43.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 61034 61018 82.8949 54.75 0.0112519 0.0120571 2023-10-09T09:07:43.431 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:43.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:43.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:43.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:43.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:43.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:43.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:43.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:43.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:43.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:43.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:43.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:43.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:43.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:44.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:44.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:44.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:44.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:44.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:44.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:44.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:44.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:44.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 62080 62064 82.522 65.375 0.0118271 0.0121114 2023-10-09T09:07:44.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:44.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:44.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:44.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:44.527 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:44.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:44.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:44.633 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:44.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:44.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:44.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:44.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:44.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:44.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:44.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:44.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:44.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:45.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:45.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:45.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:45.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:45.174 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:44 smithi016 bash[75179]: cluster 2023-10-09T09:07:43.934622+0000 mgr.y (mgr.25421) 910 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:07:45.174 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:45 smithi016 bash[77892]: cluster 2023-10-09T09:07:43.934622+0000 mgr.y (mgr.25421) 910 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:07:45.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:45.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:45.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:45.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:45.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:45.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 15 63041 63026 82.0553 60.125 0.0129697 0.0121804 2023-10-09T09:07:45.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:45.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:45.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:45.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:44 smithi094 bash[59677]: cluster 2023-10-09T09:07:43.934622+0000 mgr.y (mgr.25421) 910 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:07:45.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:45.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:45.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:45.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:45.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:45.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:45.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:45.844 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:07:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:07:45] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.33.4" 2023-10-09T09:07:45.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:45.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:45.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:45.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:46.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:46.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:46.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:46.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:46.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:46.239 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:46.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:46.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:46.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:46.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:46.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:46.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 63940 63924 81.526 56.125 0.0143707 0.0122592 2023-10-09T09:07:46.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:46.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:46.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:46.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:46.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:46.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:46.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:46.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:46.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:46.768 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:46.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:46.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:46.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:46.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:46.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:46.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:47.008 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:46 smithi094 bash[59677]: cluster 2023-10-09T09:07:45.935126+0000 mgr.y (mgr.25421) 911 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 57 MiB/s wr, 916 op/s 2023-10-09T09:07:47.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:47.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:47.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:47.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:47.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:47.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:46 smithi016 bash[75179]: cluster 2023-10-09T09:07:45.935126+0000 mgr.y (mgr.25421) 911 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 57 MiB/s wr, 916 op/s 2023-10-09T09:07:47.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:46 smithi016 bash[77892]: cluster 2023-10-09T09:07:45.935126+0000 mgr.y (mgr.25421) 911 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 57 MiB/s wr, 916 op/s 2023-10-09T09:07:47.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:47.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:47.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 64894 64878 81.0879 59.625 0.00535061 0.0123111 2023-10-09T09:07:47.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:47.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:47.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:47.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:47.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:47.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:47.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:47.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:47.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:47.914 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:47.937 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:47.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:48.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:48.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:48.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:48.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:48.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:48.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:48.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:48.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 65840 65824 80.6571 59.125 0.01401 0.0123915 2023-10-09T09:07:48.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:48.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:48.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:48.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:48.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:48.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:48.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:48.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:48.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:48.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:48.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:48.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:48.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:48.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:49.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:49.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:49.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:49.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:49.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:49.234 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:49.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:49.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:49 smithi016 bash[77892]: cluster 2023-10-09T09:07:47.935789+0000 mgr.y (mgr.25421) 912 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:07:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:49 smithi016 bash[75179]: cluster 2023-10-09T09:07:47.935789+0000 mgr.y (mgr.25421) 912 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:07:49.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:49.347 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:49 smithi094 bash[59677]: cluster 2023-10-09T09:07:47.935789+0000 mgr.y (mgr.25421) 912 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:07:49.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:49.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:49.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 66650 66634 80.0795 50.625 0.00821965 0.0124818 2023-10-09T09:07:49.474 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:49.499 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:49.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:49.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:49.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:49.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:49.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:49.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:49.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:49.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:49.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:49.890 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:49.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:49.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:49.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:50.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:50.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:50.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:50.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:50 smithi094 bash[59677]: audit 2023-10-09T09:07:49.238008+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:50.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:50.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:50 smithi016 bash[75179]: audit 2023-10-09T09:07:49.238008+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:50.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:50 smithi016 bash[77892]: audit 2023-10-09T09:07:49.238008+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:50.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:50.396 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:50.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 67637 67621 79.7324 61.6875 0.0178162 0.0125355 2023-10-09T09:07:50.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:50.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:50.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:50.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:50.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:50.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:50.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:50.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:50.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:50.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:50.937 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:50.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:50.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:51.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:51.065 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:51.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:51.091 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:51 smithi016 bash[75179]: cluster 2023-10-09T09:07:49.936285+0000 mgr.y (mgr.25421) 913 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-09T09:07:51.091 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:51 smithi016 bash[77892]: cluster 2023-10-09T09:07:49.936285+0000 mgr.y (mgr.25421) 913 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-09T09:07:51.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:51.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:51 smithi094 bash[59677]: cluster 2023-10-09T09:07:49.936285+0000 mgr.y (mgr.25421) 913 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-09T09:07:51.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:51.396 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 68619 68603 79.3923 61.375 0.0170362 0.0125891 2023-10-09T09:07:51.427 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:51.455 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:51.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:51.701 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:51.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:51.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:51.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:51.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:51.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:51.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:52.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:52.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:52.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:52.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:52.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:52.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:52.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:52.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:52.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:52.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:52.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 69503 69487 78.9533 55.25 0.00350335 0.0126483 2023-10-09T09:07:52.489 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:52.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:52.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:52.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:52.647 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:52.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:52.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:52.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:52.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:52.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:52.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:52.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:52.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:53.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:53.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:53 smithi016 bash[77892]: cluster 2023-10-09T09:07:51.937013+0000 mgr.y (mgr.25421) 914 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:07:53.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:53 smithi016 bash[75179]: cluster 2023-10-09T09:07:51.937013+0000 mgr.y (mgr.25421) 914 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:07:53.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:53.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:53.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:53.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:53.297 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:53 smithi094 bash[59677]: cluster 2023-10-09T09:07:51.937013+0000 mgr.y (mgr.25421) 914 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:07:53.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:53.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 70325 70309 78.4607 51.375 0.00824325 0.0127384 2023-10-09T09:07:53.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:53.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:53.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:53.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:53.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:53.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:53.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:53.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:53.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:53.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:53.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:53.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:53.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:54.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:54.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:54.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:54.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:54.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:54.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:54.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:54.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:54.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:54.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:54.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 71186 71170 78.0282 53.8125 0.0124623 0.0128092 2023-10-09T09:07:54.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:54.510 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:54.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:54.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:54.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:54.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:54.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:54.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:54.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:54.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:54.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:54.879 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:54.902 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:54.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:55.005 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:55.030 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:55.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:55.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:55.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:55 smithi094 bash[59677]: cluster 2023-10-09T09:07:53.937597+0000 mgr.y (mgr.25421) 915 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T09:07:55.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:55 smithi016 bash[75179]: cluster 2023-10-09T09:07:53.937597+0000 mgr.y (mgr.25421) 915 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T09:07:55.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:55 smithi016 bash[77892]: cluster 2023-10-09T09:07:53.937597+0000 mgr.y (mgr.25421) 915 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T09:07:55.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:55.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:55.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 72039 72023 77.602 53.3125 0.0172837 0.0128791 2023-10-09T09:07:55.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:55.437 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:55.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:55.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:55.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:55.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:55.696 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:55.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:55.808 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:07:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:07:55] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.33.4" 2023-10-09T09:07:55.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:55.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:55.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:55.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:55.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:56.048 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:07:56.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:56.075 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:56 smithi016 bash[77892]: audit 2023-10-09T09:07:55.609817+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.16:0/3700520921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:07:56.075 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:56 smithi016 bash[77892]: audit 2023-10-09T09:07:55.610538+0000 mon.a (mon.0) 1224 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:07:56.076 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:56 smithi016 bash[75179]: audit 2023-10-09T09:07:55.609817+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.16:0/3700520921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:07:56.076 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:56 smithi016 bash[75179]: audit 2023-10-09T09:07:55.610538+0000 mon.a (mon.0) 1224 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:07:56.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:56.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:56.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:56.238 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:07:56.256 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:07:56.260 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:07:56.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:56.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:56.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:56.287 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:07:56.309 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:07:56.309 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T09:07:56.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35034.0:166331 7.c 7:381feb68:::benchmark_data_smithi016_128518_object65813:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:59:23.622575+0000", 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 512.67280255799994, 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.114597013, 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35034", 2023-10-09T09:07:56.310 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 166331 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622575+0000", 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622575+0000", 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622574+0000", 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.312 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622582+0000", 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1009999999999992e-06 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622596+0000", 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4652e-05 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622607+0000", 2023-10-09T09:07:56.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0788000000000001e-05 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622656+0000", 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8838000000000002e-05 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.622663+0000", 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3440000000000003e-06 2023-10-09T09:07:56.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:24.493258+0000", 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.87059415900000003 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:24.493679+0000", 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042171999999999999 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.712718+0000", 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.219039164 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.714565+0000", 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018464659999999999 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T09:07:56.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.714671+0000", 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000105925 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.723986+0000", 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0093156409999999995 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.726417+0000", 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002430977 2023-10-09T09:07:56.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.730625+0000", 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0042074729999999998 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.736857+0000", 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0062317919999999999 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.737037+0000", 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018081799999999999 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.737172+0000", 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134969 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.319 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35034.0:166347 7.c 7:35274746:::benchmark_data_smithi016_128518_object65829:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:59:23.647072+0000", 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 512.648305553, 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0987690749999999, 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35034", 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 166347 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.320 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647072+0000", 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647072+0000", 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647071+0000", 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647078+0000", 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4980000000000003e-06 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647092+0000", 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4331999999999999e-05 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647102+0000", 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7580000000000004e-06 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647151+0000", 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9187999999999997e-05 2023-10-09T09:07:56.323 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:23.647158+0000", 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9199999999999998e-06 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:24.493708+0000", 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.846549471 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.324 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:24.493717+0000", 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5829999999999996e-06 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.718116+0000", 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.224398455 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.325 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.718222+0000", 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000106573 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.718306+0000", 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4037999999999997e-05 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.723998+0000", 2023-10-09T09:07:56.326 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0056918230000000004 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.726622+0000", 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00262444 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.731654+0000", 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0050316900000000001 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.327 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.745503+0000", 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.013848849999999999 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.745710+0000", 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000207309 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.328 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:25.745842+0000", 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013125600000000001 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35034.0:177366 7.8 7:1826e336:::benchmark_data_smithi016_128518_object76848:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:59:45.131587+0000", 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 491.16379107099999, 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25954799099999998, 2023-10-09T09:07:56.329 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35034", 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177366 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.131587+0000", 2023-10-09T09:07:56.330 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.131587+0000", 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.131586+0000", 2023-10-09T09:07:56.331 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.131590+0000", 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4050000000000002e-06 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.131604+0000", 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4233000000000001e-05 2023-10-09T09:07:56.332 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.131611+0000", 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9330000000000004e-06 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.132127+0000", 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00051528900000000005 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.333 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.132250+0000", 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012370099999999999 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.132410+0000", 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015921300000000001 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.334 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.151216+0000", 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018806461 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.151235+0000", 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9244000000000001e-05 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.389113+0000", 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.237877375 2023-10-09T09:07:56.335 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.389168+0000", 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5481e-05 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.391135+0000", 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001966766 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.336 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35034.0:177378 7.8 7:142bb1a6:::benchmark_data_smithi016_128518_object76860:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:59:45.153178+0000", 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 491.14220021599999, 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25445004999999998, 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35034", 2023-10-09T09:07:56.337 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177378 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153178+0000", 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153178+0000", 2023-10-09T09:07:56.338 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153177+0000", 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153183+0000", 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5429999999999999e-06 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.339 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153197+0000", 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4110999999999999e-05 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153204+0000", 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9659999999999999e-06 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.340 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153249+0000", 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4641999999999997e-05 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153353+0000", 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010407 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.153519+0000", 2023-10-09T09:07:56.341 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016574600000000001 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.160067+0000", 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0065483470000000004 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.160090+0000", 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2776e-05 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.342 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.405708+0000", 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24561833799999999 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.405766+0000", 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7623e-05 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.343 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.407628+0000", 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001862067 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35034.0:177389 7.8 7:13d09ef8:::benchmark_data_smithi016_128518_object76871:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T08:59:45.170968+0000", 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 491.12441026300002, 2023-10-09T09:07:56.344 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.328452517, 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35034", 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1878585322", 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177389 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.170968+0000", 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.345 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.170968+0000", 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.170967+0000", 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.346 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.170972+0000", 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7320000000000003e-06 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.170984+0000", 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2033e-05 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.347 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.170990+0000", 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8449999999999999e-06 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.171033+0000", 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2898999999999997e-05 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for rw locks", 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.171123+0000", 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0425000000000002e-05 2023-10-09T09:07:56.348 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.434972+0000", 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26384824299999998 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.435074+0000", 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010214599999999999 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.435224+0000", 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000150144 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.439486+0000", 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0042619759999999998 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.448846+0000", 2023-10-09T09:07:56.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0093601780000000002 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.497472+0000", 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.048626139999999998 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.497593+0000", 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012034799999999999 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T08:59:45.499420+0000", 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018275909999999999 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45122.0:7967 8.d 8:b2f76e1b:::benchmark_data_smithi016_151580_object7966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:01:41.878805+0000", 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 374.41657323800001, 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.39735309699999999, 2023-10-09T09:07:56.352 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45122", 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 7967 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.878805+0000", 2023-10-09T09:07:56.353 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.878805+0000", 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.878803+0000", 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.878832+0000", 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.94e-05 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.878844+0000", 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1882000000000001e-05 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.878865+0000", 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0424e-05 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.878907+0000", 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2020000000000001e-05 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.879034+0000", 2023-10-09T09:07:56.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012785499999999999 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,6", 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.879133+0000", 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8181000000000001e-05 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.880275+0000", 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011425770000000001 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.357 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:41.880739+0000", 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00046392200000000002 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:42.275984+0000", 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.39524477000000002 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:42.276110+0000", 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012635599999999999 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:01:42.276158+0000", 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7559000000000003e-05 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45122.0:55048 8.a 8:5504c4c8:::benchmark_data_smithi016_151580_object55047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:02:23.277899+0000", 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 333.01747895900002, 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29209282600000003, 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45122", 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 55048 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.360 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.277899+0000", 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.277899+0000", 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.277898+0000", 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.277927+0000", 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9526000000000002e-05 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.277958+0000", 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0626999999999997e-05 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.277966+0000", 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9929999999999995e-06 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.278867+0000", 2023-10-09T09:07:56.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00090116999999999997 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.279049+0000", 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018160299999999999 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.279236+0000", 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018698599999999999 2023-10-09T09:07:56.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.285100+0000", 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0058642649999999996 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.286498+0000", 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001398622 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.365 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.568055+0000", 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28155681100000002 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.568146+0000", 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0759000000000003e-05 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.569992+0000", 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018458770000000001 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45122.0:55073 8.a 8:5d809ed9:::benchmark_data_smithi016_151580_object55072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:02:23.312523+0000", 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 332.98285484000002, 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25921481699999999, 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.367 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45122", 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 55073 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312523+0000", 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312523+0000", 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312522+0000", 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312546+0000", 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3734e-05 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312558+0000", 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2357999999999999e-05 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312566+0000", 2023-10-09T09:07:56.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6240000000000002e-06 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312613+0000", 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6968000000000002e-05 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312739+0000", 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012644800000000001 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.371 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.312860+0000", 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012077400000000001 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.314621+0000", 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001761405 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.372 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.316338+0000", 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0017168330000000001 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.570117+0000", 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25377887500000001 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.570191+0000", 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3621999999999994e-05 2023-10-09T09:07:56.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:02:23.571738+0000", 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001547477 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45122.0:139871 8.f e201/190 8:f2a239e6:::benchmark_data_smithi016_151580_object51632:head v 201'8826, mlcod=0'0)", 2023-10-09T09:07:56.374 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.112870+0000", 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.18250840499999, 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35716489400000001, 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.112870+0000", 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.375 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.112870+0000", 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.112869+0000", 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.376 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.112874+0000", 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3839999999999999e-06 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.112877+0000", 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9490000000000001e-06 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.112880+0000", 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9579999999999999e-06 2023-10-09T09:07:56.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.114131+0000", 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0012512109999999999 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.114174+0000", 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2855000000000002e-05 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.469969+0000", 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35579506799999999 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470034+0000", 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5580000000000006e-05 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45122.0:139874 8.7 e201/190 8:edc26fd4:::benchmark_data_smithi016_151580_object51635:head v 201'8986, mlcod=0'0)", 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.122607+0000", 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.172770595, 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34749914799999998, 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122607+0000", 2023-10-09T09:07:56.380 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122607+0000", 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122607+0000", 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:07:56.381 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122615+0000", 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6729999999999992e-06 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122623+0000", 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3550000000000005e-06 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.382 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122629+0000", 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9109999999999998e-06 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.125576+0000", 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0029472909999999999 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.125595+0000", 2023-10-09T09:07:56.383 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8428e-05 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470046+0000", 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.344451597 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470107+0000", 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0318000000000001e-05 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.384 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45122.0:139875 8.7 e201/190 8:e3622471:::benchmark_data_smithi016_151580_object51636:head v 201'8987, mlcod=0'0)", 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.122641+0000", 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.17273663500001, 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34749361400000001, 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.385 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122641+0000", 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122641+0000", 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122641+0000", 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122645+0000", 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7759999999999999e-06 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122650+0000", 2023-10-09T09:07:56.387 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5439999999999998e-06 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.122655+0000", 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.33e-06 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.133631+0000", 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.01097586 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.388 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.133659+0000", 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8793999999999999e-05 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470110+0000", 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33645107899999999 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470135+0000", 2023-10-09T09:07:56.389 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4490000000000001e-05 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45122.0:139885 8.7 e201/190 8:e2f2c51b:::benchmark_data_smithi016_151580_object51646:head v 201'8988, mlcod=0'0)", 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.129260+0000", 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.16611755, 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34089903100000002, 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.390 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129260+0000", 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129260+0000", 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.391 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129260+0000", 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129265+0000", 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2410000000000003e-06 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.392 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129273+0000", 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8229999999999995e-06 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129278+0000", 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6410000000000001e-06 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.146235+0000", 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016957863 2023-10-09T09:07:56.393 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.146268+0000", 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2178999999999998e-05 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470138+0000", 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32387022700000001 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.394 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470159+0000", 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1691e-05 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45122.0:139890 8.a 8:5f6f80bf:::benchmark_data_smithi016_151580_object51651:head [delete] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.129290+0000", 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.16608822800001, 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34128804899999998, 2023-10-09T09:07:56.395 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45122", 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139890 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129290+0000", 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.396 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129290+0000", 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129289+0000", 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.397 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129303+0000", 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4286e-05 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129309+0000", 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0939999999999996e-06 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.398 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129313+0000", 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0450000000000002e-06 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129366+0000", 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2382000000000001e-05 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129506+0000", 2023-10-09T09:07:56.399 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013979800000000001 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.129644+0000", 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000138417 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.400 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.137073+0000", 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0074292200000000003 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.142528+0000", 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0054544219999999996 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150232+0000", 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0077040609999999999 2023-10-09T09:07:56.401 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150280+0000", 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8677000000000002e-05 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470578+0000", 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32029743799999999 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.402 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45122.0:139894 8.a 8:54ffc0a8:::benchmark_data_smithi016_151580_object51655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.139635+0000", 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.15574252299999, 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33392813799999999, 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.403 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45122", 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139894 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139635+0000", 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139635+0000", 2023-10-09T09:07:56.404 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139634+0000", 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139640+0000", 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5740000000000003e-06 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.405 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139649+0000", 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7520000000000002e-06 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139655+0000", 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5910000000000001e-06 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.142547+0000", 2023-10-09T09:07:56.406 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028923339999999999 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.142662+0000", 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000114948 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.142781+0000", 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011859299999999999 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.407 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150047+0000", 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0072664410000000002 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.472956+0000", 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32290869999999999 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.472996+0000", 2023-10-09T09:07:56.408 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9999000000000001e-05 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.473061+0000", 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4622000000000006e-05 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.473564+0000", 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00050282899999999999 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.409 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45122.0:139898 8.a 8:570110fa:::benchmark_data_smithi016_151580_object51659:head [delete] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.139667+0000", 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.15571145300001, 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34826606599999999, 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45122", 2023-10-09T09:07:56.410 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139898 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139667+0000", 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139667+0000", 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.411 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139666+0000", 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139668+0000", 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.863e-06 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139672+0000", 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.641e-06 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.139675+0000", 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9380000000000001e-06 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.146091+0000", 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0064158239999999997 2023-10-09T09:07:56.413 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.146231+0000", 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000140739 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.146359+0000", 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012783299999999999 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.414 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150057+0000", 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0036981119999999999 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.473604+0000", 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32354724400000001 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.415 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.487201+0000", 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.013596205 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.487358+0000", 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015764899999999999 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.487933+0000", 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00057432900000000005 2023-10-09T09:07:56.416 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45122.0:139890 8.a e201/182 ondisk, result = 0)", 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.150152+0000", 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.14522568699999, 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32043914699999998, 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.417 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150152+0000", 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150152+0000", 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.418 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150152+0000", 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150156+0000", 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3100000000000002e-06 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150164+0000", 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.32e-06 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.419 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150170+0000", 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3260000000000003e-06 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150221+0000", 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1019000000000001e-05 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150227+0000", 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8129999999999998e-06 2023-10-09T09:07:56.420 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470591+0000", 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32036400300000001 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45122.0:139894 8.a e201/182 ondisk, result = 0)", 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.150181+0000", 2023-10-09T09:07:56.421 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.14519656799999, 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.322779914, 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150181+0000", 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.422 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150181+0000", 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150181+0000", 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150183+0000", 2023-10-09T09:07:56.423 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.888e-06 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150187+0000", 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.631e-06 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150189+0000", 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6230000000000001e-06 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.424 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.472936+0000", 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32274675000000003 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.472948+0000", 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2147000000000001e-05 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.425 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.472961+0000", 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3094e-05 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45122.0:139902 8.8 e201/191 8:1ead571d:::benchmark_data_smithi016_151580_object51663:head v 201'8776, mlcod=0'0)", 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.150587+0000", 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.14479122399999, 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33309264100000002, 2023-10-09T09:07:56.426 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150587+0000", 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150587+0000", 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.427 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150586+0000", 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.150592+0000", 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.767e-06 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.428 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470821+0000", 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32022887500000002 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470828+0000", 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8990000000000004e-06 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470871+0000", 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3120000000000001e-05 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470900+0000", 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9257000000000001e-05 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.483637+0000", 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012736903000000001 2023-10-09T09:07:56.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.483679+0000", 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2344e-05 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45122.0:139904 8.b e201/194 8:d37ee7eb:::benchmark_data_smithi016_151580_object51665:head v 201'8805, mlcod=201'8803)", 2023-10-09T09:07:56.431 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:04:32.152026+0000", 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 204.143352056, 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33170649899999999, 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.152026+0000", 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.432 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.152026+0000", 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.152025+0000", 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.152033+0000", 2023-10-09T09:07:56.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3300000000000001e-06 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470528+0000", 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31849581500000002 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470537+0000", 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3410000000000003e-06 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470972+0000", 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00043513200000000002 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.470994+0000", 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.213e-05 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.435 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.483698+0000", 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012704259000000001 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:04:32.483732+0000", 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4205000000000002e-05 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.436 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:57864 9.6 9:67037055:::benchmark_data_smithi016_172771_object57863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:07:39.814839+0000", 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 16.480538523, 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26235065200000002, 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 57864 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:07:56.437 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.814839+0000", 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.814839+0000", 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.438 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.814838+0000", 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.814869+0000", 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1319000000000003e-05 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.814879+0000", 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4539999999999996e-06 2023-10-09T09:07:56.439 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.814885+0000", 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4339999999999997e-06 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.814946+0000", 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1247000000000005e-05 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.440 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.815096+0000", 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000150049 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.815253+0000", 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015629700000000001 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:07:56.441 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.817538+0000", 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022848069999999998 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:39.818253+0000", 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00071531800000000005 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:40.076837+0000", 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.258584219 2023-10-09T09:07:56.442 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:40.077081+0000", 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00024362200000000001 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:07:40.077190+0000", 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109415 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.443 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.444 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.444 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:07:56.444 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:07:56.444 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:07:56.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:56.445 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:07:56.445 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:07:56.446 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 72781 72765 77.0726 46.375 0.0136794 0.012968 2023-10-09T09:07:56.446 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:07:56.452 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:07:56.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:56 smithi094 bash[59677]: audit 2023-10-09T09:07:55.609817+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.16:0/3700520921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:07:56.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:56 smithi094 bash[59677]: audit 2023-10-09T09:07:55.610538+0000 mon.a (mon.0) 1224 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:07:56.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:56.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:56.491 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:07:56.530 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:07:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.535 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:07:56.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:56.602 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:56.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:56.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:56.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:56.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:56.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:56.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:56.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:56.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:56.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:56.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:57.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:57.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:57.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:57.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: cluster 2023-10-09T09:07:55.938125+0000 mgr.y (mgr.25421) 916 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: cluster 2023-10-09T09:07:56.014066+0000 mon.a (mon.0) 1225 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: cluster 2023-10-09T09:07:56.024705+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: cluster 2023-10-09T09:07:56.027622+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: audit 2023-10-09T09:07:56.035294+0000 mon.a (mon.0) 1226 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: cluster 2023-10-09T09:07:56.035535+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e220: 8 total, 8 up, 5 in 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: cluster 2023-10-09T09:07:56.056481+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: audit 2023-10-09T09:07:56.191865+0000 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.16:0/2821761191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[75179]: audit 2023-10-09T09:07:56.227344+0000 mgr.y (mgr.25421) 917 : audit 0 from='client.55087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: cluster 2023-10-09T09:07:55.938125+0000 mgr.y (mgr.25421) 916 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: cluster 2023-10-09T09:07:56.014066+0000 mon.a (mon.0) 1225 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: cluster 2023-10-09T09:07:56.024705+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: cluster 2023-10-09T09:07:56.027622+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: audit 2023-10-09T09:07:56.035294+0000 mon.a (mon.0) 1226 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: cluster 2023-10-09T09:07:56.035535+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e220: 8 total, 8 up, 5 in 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: cluster 2023-10-09T09:07:56.056481+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: audit 2023-10-09T09:07:56.191865+0000 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.16:0/2821761191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:07:57.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:57 smithi016 bash[77892]: audit 2023-10-09T09:07:56.227344+0000 mgr.y (mgr.25421) 917 : audit 0 from='client.55087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:07:57.252 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:57.282 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:57.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: cluster 2023-10-09T09:07:55.938125+0000 mgr.y (mgr.25421) 916 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T09:07:57.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: cluster 2023-10-09T09:07:56.014066+0000 mon.a (mon.0) 1225 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:07:57.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: cluster 2023-10-09T09:07:56.024705+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: cluster 2023-10-09T09:07:56.027622+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: audit 2023-10-09T09:07:56.035294+0000 mon.a (mon.0) 1226 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:07:57.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: cluster 2023-10-09T09:07:56.035535+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e220: 8 total, 8 up, 5 in 2023-10-09T09:07:57.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: cluster 2023-10-09T09:07:56.056481+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T09:07:57.287 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: audit 2023-10-09T09:07:56.191865+0000 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.16:0/2821761191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:07:57.287 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:57 smithi094 bash[59677]: audit 2023-10-09T09:07:56.227344+0000 mgr.y (mgr.25421) 917 : audit 0 from='client.55087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:07:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:57.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:57.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:07:57.397009+0000 min lat: 0.00143057 max lat: 0.340643 avg lat: 0.0130177 2023-10-09T09:07:57.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:07:57.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 73736 73720 76.7828 59.6875 0.0330521 0.0130177 2023-10-09T09:07:57.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:57.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:57.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:57.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:57.648 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:57.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:57.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:57.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:57.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:57.918 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:57.942 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:57.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:58.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:58.076 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:58.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:58.181 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:58.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:58.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:58.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:58.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:07:58.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 74633 74617 76.443 56.0625 0.0132151 0.0130759 2023-10-09T09:07:58.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:58.463 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:58.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:58.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:07:58.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:07:58.694 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:58.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:07:58.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:07:58.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:58.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:58.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:58.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:58.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:07:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:58.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:07:59.083 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:07:59 smithi016 bash[75179]: cluster 2023-10-09T09:07:57.938695+0000 mgr.y (mgr.25421) 918 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T09:07:59.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:59.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:59.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:59.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:59.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:07:59 smithi094 bash[59677]: cluster 2023-10-09T09:07:57.938695+0000 mgr.y (mgr.25421) 918 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T09:07:59.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:59.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:59.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:59.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 75586 75570 76.1707 59.5625 0.0151377 0.0131223 2023-10-09T09:07:59.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:59.428 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:07:59.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:59.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:07:59.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:07:59 smithi016 bash[77892]: cluster 2023-10-09T09:07:57.938695+0000 mgr.y (mgr.25421) 918 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T09:07:59.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:59.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:07:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:59.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:07:59.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:59.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:07:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:59.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:07:59.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:59.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:07:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:59.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:07:59.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:59.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:07:59.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:07:59.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:00.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:00.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:00.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:00.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:00.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:00.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:00.256 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":962,"stamp":"2023-10-09T09:07:56.025229+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4548466592,"num_objects":69419,"num_object_clones":0,"num_object_copies":208257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69419,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":69655,"num_write_kb":4443725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":2374946,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69589,"ondisk_log_size":69589,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":16597860,"kb_used_data":13398780,"kb_used_omap":117,"kb_used_meta":3198922,"kb_avail":452127900,"statfs":{"total":479975178240,"available":462978969600,"internally_reserved":0,"allocated":13720350720,"data_stored":13716778137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":3275696786},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,4,6,2,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":614072320,"num_objects":9370,"num_object_clones":0,"num_object_copies":28110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9370,"num_write_kb":599680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.090426"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":81,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110806+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:06:59.110806+0000","last_peered":"2023-10-09T09:06:59.110806+0000","last_clean":"2023-10-09T09:06:59.110806+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:06:59.110806+0000","last_undegraded":"2023-10-09T09:06:59.110806+0000","last_fullsized":"2023-10-09T09:06:59.110806+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":231,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.872206+0000","last_change":"2023-10-09T09:02:02.625626+0000","last_active":"2023-10-09T09:07:20.872206+0000","last_peered":"2023-10-09T09:07:20.872206+0000","last_clean":"2023-10-09T09:07:20.872206+0000","last_became_active":"2023-10-09T09:02:02.625312+0000","last_became_peered":"2023-10-09T09:02:02.625312+0000","last_unstale":"2023-10-09T09:07:20.872206+0000","last_undegraded":"2023-10-09T09:07:20.872206+0000","last_fullsized":"2023-10-09T09:07:20.872206+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:01.640943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":184,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:00.848561+0000","last_change":"2023-10-09T09:02:02.626274+0000","last_active":"2023-10-09T09:07:00.848561+0000","last_peered":"2023-10-09T09:07:00.848561+0000","last_clean":"2023-10-09T09:07:00.848561+0000","last_became_active":"2023-10-09T09:02:02.625973+0000","last_became_peered":"2023-10-09T09:02:02.625973+0000","last_unstale":"2023-10-09T09:07:00.848561+0000","last_undegraded":"2023-10-09T09:07:00.848561+0000","last_fullsized":"2023-10-09T09:07:00.848561+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:44.223794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":232,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.081498+0000","last_change":"2023-10-09T09:03:19.554441+0000","last_active":"2023-10-09T09:07:29.081498+0000","last_peered":"2023-10-09T09:07:29.081498+0000","last_clean":"2023-10-09T09:07:29.081498+0000","last_became_active":"2023-10-09T09:03:19.553967+0000","last_became_peered":"2023-10-09T09:03:19.553967+0000","last_unstale":"2023-10-09T09:07:29.081498+0000","last_undegraded":"2023-10-09T09:07:29.081498+0000","last_fullsized":"2023-10-09T09:07:29.081498+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:26.065993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":220,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.113622+0000","last_change":"2023-10-09T09:02:02.656821+0000","last_active":"2023-10-09T09:06:59.113622+0000","last_peered":"2023-10-09T09:06:59.113622+0000","last_clean":"2023-10-09T09:06:59.113622+0000","last_became_active":"2023-10-09T09:02:02.656523+0000","last_became_peered":"2023-10-09T09:02:02.656523+0000","last_unstale":"2023-10-09T09:06:59.113622+0000","last_undegraded":"2023-10-09T09:06:59.113622+0000","last_fullsized":"2023-10-09T09:06:59.113622+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:21.635518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":95,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115455+0000","last_change":"2023-10-09T09:03:19.582769+0000","last_active":"2023-10-09T09:06:59.115455+0000","last_peered":"2023-10-09T09:06:59.115455+0000","last_clean":"2023-10-09T09:06:59.115455+0000","last_became_active":"2023-10-09T09:03:19.577023+0000","last_became_peered":"2023-10-09T09:03:19.577023+0000","last_unstale":"2023-10-09T09:06:59.115455+0000","last_undegraded":"2023-10-09T09:06:59.115455+0000","last_fullsized":"2023-10-09T09:06:59.115455+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":194,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.083340+0000","last_change":"2023-10-09T09:01:36.280403+0000","last_active":"2023-10-09T09:07:29.083340+0000","last_peered":"2023-10-09T09:07:29.083340+0000","last_clean":"2023-10-09T09:07:29.083340+0000","last_became_active":"2023-10-09T09:01:36.280070+0000","last_became_peered":"2023-10-09T09:01:36.280070+0000","last_unstale":"2023-10-09T09:07:29.083340+0000","last_undegraded":"2023-10-09T09:07:29.083340+0000","last_fullsized":"2023-10-09T09:07:29.083340+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:33.796899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":231,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114417+0000","last_change":"2023-10-09T09:02:02.630162+0000","last_active":"2023-10-09T09:06:59.114417+0000","last_peered":"2023-10-09T09:06:59.114417+0000","last_clean":"2023-10-09T09:06:59.114417+0000","last_became_active":"2023-10-09T09:02:02.629768+0000","last_became_peered":"2023-10-09T09:02:02.629768+0000","last_unstale":"2023-10-09T09:06:59.114417+0000","last_undegraded":"2023-10-09T09:06:59.114417+0000","last_fullsized":"2023-10-09T09:06:59.114417+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:36.859890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":37,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.116682+0000","last_change":"2023-10-09T09:03:19.562130+0000","last_active":"2023-10-09T09:06:59.116682+0000","last_peered":"2023-10-09T09:06:59.116682+0000","last_clean":"2023-10-09T09:06:59.116682+0000","last_became_active":"2023-10-09T09:03:19.557757+0000","last_became_peered":"2023-10-09T09:03:19.557757+0000","last_unstale":"2023-10-09T09:06:59.116682+0000","last_undegraded":"2023-10-09T09:06:59.116682+0000","last_fullsized":"2023-10-09T09:06:59.116682+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:46.546761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":192,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.113874+0000","last_change":"2023-10-09T09:01:36.279247+0000","last_active":"2023-10-09T09:06:59.113874+0000","last_peered":"2023-10-09T09:06:59.113874+0000","last_clean":"2023-10-09T09:06:59.113874+0000","last_became_active":"2023-10-09T09:01:36.278376+0000","last_became_peered":"2023-10-09T09:01:36.278376+0000","last_unstale":"2023-10-09T09:06:59.113874+0000","last_undegraded":"2023-10-09T09:06:59.113874+0000","last_fullsized":"2023-10-09T09:06:59.113874+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:19.166567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":177,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114321+0000","last_change":"2023-10-09T08:59:53.544302+0000","last_active":"2023-10-09T09:06:59.114321+0000","last_peered":"2023-10-09T09:06:59.114321+0000","last_clean":"2023-10-09T09:06:59.114321+0000","last_became_active":"2023-10-09T08:59:53.531404+0000","last_became_peered":"2023-10-09T08:59:53.531404+0000","last_unstale":"2023-10-09T09:06:59.114321+0000","last_undegraded":"2023-10-09T09:06:59.114321+0000","last_fullsized":"2023-10-09T09:06:59.114321+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:48.670745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":178,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114374+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:06:59.114374+0000","last_peered":"2023-10-09T09:06:59.114374+0000","last_clean":"2023-10-09T09:06:59.114374+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:06:59.114374+0000","last_undegraded":"2023-10-09T09:06:59.114374+0000","last_fullsized":"2023-10-09T09:06:59.114374+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":101,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111306+0000","last_change":"2023-10-09T09:00:31.867387+0000","last_active":"2023-10-09T09:06:59.111306+0000","last_peered":"2023-10-09T09:06:59.111306+0000","last_clean":"2023-10-09T09:06:59.111306+0000","last_became_active":"2023-10-09T09:00:31.859752+0000","last_became_peered":"2023-10-09T09:00:31.859752+0000","last_unstale":"2023-10-09T09:06:59.111306+0000","last_undegraded":"2023-10-09T09:06:59.111306+0000","last_fullsized":"2023-10-09T09:06:59.111306+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":90,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110950+0000","last_change":"2023-10-09T09:02:02.630025+0000","last_active":"2023-10-09T09:06:59.110950+0000","last_peered":"2023-10-09T09:06:59.110950+0000","last_clean":"2023-10-09T09:06:59.110950+0000","last_became_active":"2023-10-09T09:02:02.629474+0000","last_became_peered":"2023-10-09T09:02:02.629474+0000","last_unstale":"2023-10-09T09:06:59.110950+0000","last_undegraded":"2023-10-09T09:06:59.110950+0000","last_fullsized":"2023-10-09T09:06:59.110950+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":93,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111209+0000","last_change":"2023-10-09T09:00:31.959283+0000","last_active":"2023-10-09T09:06:59.111209+0000","last_peered":"2023-10-09T09:06:59.111209+0000","last_clean":"2023-10-09T09:06:59.111209+0000","last_became_active":"2023-10-09T09:00:31.861638+0000","last_became_peered":"2023-10-09T09:00:31.861638+0000","last_unstale":"2023-10-09T09:06:59.111209+0000","last_undegraded":"2023-10-09T09:06:59.111209+0000","last_fullsized":"2023-10-09T09:06:59.111209+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":213,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.113924+0000","last_change":"2023-10-09T09:02:02.636035+0000","last_active":"2023-10-09T09:06:59.113924+0000","last_peered":"2023-10-09T09:06:59.113924+0000","last_clean":"2023-10-09T09:06:59.113924+0000","last_became_active":"2023-10-09T09:02:02.635621+0000","last_became_peered":"2023-10-09T09:02:02.635621+0000","last_unstale":"2023-10-09T09:06:59.113924+0000","last_undegraded":"2023-10-09T09:06:59.113924+0000","last_fullsized":"2023-10-09T09:06:59.113924+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:46.294650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":221,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.108967+0000","last_change":"2023-10-09T09:03:19.564077+0000","last_active":"2023-10-09T09:06:59.108967+0000","last_peered":"2023-10-09T09:06:59.108967+0000","last_clean":"2023-10-09T09:06:59.108967+0000","last_became_active":"2023-10-09T09:03:19.563764+0000","last_became_peered":"2023-10-09T09:03:19.563764+0000","last_unstale":"2023-10-09T09:06:59.108967+0000","last_undegraded":"2023-10-09T09:06:59.108967+0000","last_fullsized":"2023-10-09T09:06:59.108967+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:38.043450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":138,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115705+0000","last_change":"2023-10-09T09:06:50.678855+0000","last_active":"2023-10-09T09:06:59.115705+0000","last_peered":"2023-10-09T09:06:59.115705+0000","last_clean":"2023-10-09T09:06:59.115705+0000","last_became_active":"2023-10-09T09:06:50.678286+0000","last_became_peered":"2023-10-09T09:06:50.678286+0000","last_unstale":"2023-10-09T09:06:59.115705+0000","last_undegraded":"2023-10-09T09:06:59.115705+0000","last_fullsized":"2023-10-09T09:06:59.115705+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110767+0000","last_change":"2023-10-09T09:00:31.851267+0000","last_active":"2023-10-09T09:06:59.110767+0000","last_peered":"2023-10-09T09:06:59.110767+0000","last_clean":"2023-10-09T09:06:59.110767+0000","last_became_active":"2023-10-09T09:00:31.850918+0000","last_became_peered":"2023-10-09T09:00:31.850918+0000","last_unstale":"2023-10-09T09:06:59.110767+0000","last_undegraded":"2023-10-09T09:06:59.110767+0000","last_fullsized":"2023-10-09T09:06:59.110767+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":194,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.081643+0000","last_change":"2023-10-09T08:54:16.081441+0000","last_active":"2023-10-09T09:07:29.081643+0000","last_peered":"2023-10-09T09:07:29.081643+0000","last_clean":"2023-10-09T09:07:29.081643+0000","last_became_active":"2023-10-09T08:54:16.081045+0000","last_became_peered":"2023-10-09T08:54:16.081045+0000","last_unstale":"2023-10-09T09:07:29.081643+0000","last_undegraded":"2023-10-09T09:07:29.081643+0000","last_fullsized":"2023-10-09T09:07:29.081643+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:36.710600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":97,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114506+0000","last_change":"2023-10-09T09:01:36.301852+0000","last_active":"2023-10-09T09:06:59.114506+0000","last_peered":"2023-10-09T09:06:59.114506+0000","last_clean":"2023-10-09T09:06:59.114506+0000","last_became_active":"2023-10-09T09:01:36.284416+0000","last_became_peered":"2023-10-09T09:01:36.284416+0000","last_unstale":"2023-10-09T09:06:59.114506+0000","last_undegraded":"2023-10-09T09:06:59.114506+0000","last_fullsized":"2023-10-09T09:06:59.114506+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":37,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.116932+0000","last_change":"2023-10-09T09:03:19.575211+0000","last_active":"2023-10-09T09:06:59.116932+0000","last_peered":"2023-10-09T09:06:59.116932+0000","last_clean":"2023-10-09T09:06:59.116932+0000","last_became_active":"2023-10-09T09:03:19.574829+0000","last_became_peered":"2023-10-09T09:03:19.574829+0000","last_unstale":"2023-10-09T09:06:59.116932+0000","last_undegraded":"2023-10-09T09:06:59.116932+0000","last_fullsized":"2023-10-09T09:06:59.116932+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:14.763790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"219'4310","reported_seq":4324,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:52.740506+0000","last_change":"2023-10-09T09:06:27.619755+0000","last_active":"2023-10-09T09:07:52.740506+0000","last_peered":"2023-10-09T09:07:52.740506+0000","last_clean":"2023-10-09T09:07:52.740506+0000","last_became_active":"2023-10-09T09:06:27.619440+0000","last_became_peered":"2023-10-09T09:06:27.619440+0000","last_unstale":"2023-10-09T09:07:52.740506+0000","last_undegraded":"2023-10-09T09:07:52.740506+0000","last_fullsized":"2023-10-09T09:07:52.740506+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4310,"log_dups_size":0,"ondisk_log_size":4310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:12.821748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282460160,"num_objects":4310,"num_object_clones":0,"num_object_copies":12930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4310,"num_write_kb":275840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":18,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115560+0000","last_change":"2023-10-09T09:06:50.677770+0000","last_active":"2023-10-09T09:06:59.115560+0000","last_peered":"2023-10-09T09:06:59.115560+0000","last_clean":"2023-10-09T09:06:59.115560+0000","last_became_active":"2023-10-09T09:06:50.674696+0000","last_became_peered":"2023-10-09T09:06:50.674696+0000","last_unstale":"2023-10-09T09:06:59.115560+0000","last_undegraded":"2023-10-09T09:06:59.115560+0000","last_fullsized":"2023-10-09T09:06:59.115560+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:24:52.193110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":196,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114125+0000","last_change":"2023-10-09T09:03:19.582540+0000","last_active":"2023-10-09T09:06:59.114125+0000","last_peered":"2023-10-09T09:06:59.114125+0000","last_clean":"2023-10-09T09:06:59.114125+0000","last_became_active":"2023-10-09T09:03:19.577044+0000","last_became_peered":"2023-10-09T09:03:19.577044+0000","last_unstale":"2023-10-09T09:06:59.114125+0000","last_undegraded":"2023-10-09T09:06:59.114125+0000","last_fullsized":"2023-10-09T09:06:59.114125+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"219'4554","reported_seq":4569,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:55.399500+0000","last_change":"2023-10-09T09:06:27.616114+0000","last_active":"2023-10-09T09:07:55.399500+0000","last_peered":"2023-10-09T09:07:55.399500+0000","last_clean":"2023-10-09T09:07:55.399500+0000","last_became_active":"2023-10-09T09:06:27.615815+0000","last_became_peered":"2023-10-09T09:06:27.615815+0000","last_unstale":"2023-10-09T09:07:55.399500+0000","last_undegraded":"2023-10-09T09:07:55.399500+0000","last_fullsized":"2023-10-09T09:07:55.399500+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4554,"log_dups_size":0,"ondisk_log_size":4554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298450944,"num_objects":4554,"num_object_clones":0,"num_object_copies":13662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4554,"num_write_kb":291456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":89,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111709+0000","last_change":"2023-10-09T09:02:02.724809+0000","last_active":"2023-10-09T09:06:59.111709+0000","last_peered":"2023-10-09T09:06:59.111709+0000","last_clean":"2023-10-09T09:06:59.111709+0000","last_became_active":"2023-10-09T09:02:02.666202+0000","last_became_peered":"2023-10-09T09:02:02.666202+0000","last_unstale":"2023-10-09T09:06:59.111709+0000","last_undegraded":"2023-10-09T09:06:59.111709+0000","last_fullsized":"2023-10-09T09:06:59.111709+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":99,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110639+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:06:59.110639+0000","last_peered":"2023-10-09T09:06:59.110639+0000","last_clean":"2023-10-09T09:06:59.110639+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:06:59.110639+0000","last_undegraded":"2023-10-09T09:06:59.110639+0000","last_fullsized":"2023-10-09T09:06:59.110639+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"219'4136","reported_seq":4152,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.077401+0000","last_change":"2023-10-09T09:06:27.614853+0000","last_active":"2023-10-09T09:07:51.077401+0000","last_peered":"2023-10-09T09:07:51.077401+0000","last_clean":"2023-10-09T09:07:51.077401+0000","last_became_active":"2023-10-09T09:06:27.614466+0000","last_became_peered":"2023-10-09T09:06:27.614466+0000","last_unstale":"2023-10-09T09:07:51.077401+0000","last_undegraded":"2023-10-09T09:07:51.077401+0000","last_fullsized":"2023-10-09T09:07:51.077401+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4136,"log_dups_size":0,"ondisk_log_size":4136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:20.558954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271056896,"num_objects":4136,"num_object_clones":0,"num_object_copies":12408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4136,"num_write_kb":264704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":128,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115595+0000","last_change":"2023-10-09T09:01:36.305615+0000","last_active":"2023-10-09T09:06:59.115595+0000","last_peered":"2023-10-09T09:06:59.115595+0000","last_clean":"2023-10-09T09:06:59.115595+0000","last_became_active":"2023-10-09T09:01:36.292828+0000","last_became_peered":"2023-10-09T09:01:36.292828+0000","last_unstale":"2023-10-09T09:06:59.115595+0000","last_undegraded":"2023-10-09T09:06:59.115595+0000","last_fullsized":"2023-10-09T09:06:59.115595+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114638+0000","last_change":"2023-10-09T09:03:19.564369+0000","last_active":"2023-10-09T09:06:59.114638+0000","last_peered":"2023-10-09T09:06:59.114638+0000","last_clean":"2023-10-09T09:06:59.114638+0000","last_became_active":"2023-10-09T09:03:19.563989+0000","last_became_peered":"2023-10-09T09:03:19.563989+0000","last_unstale":"2023-10-09T09:06:59.114638+0000","last_undegraded":"2023-10-09T09:06:59.114638+0000","last_fullsized":"2023-10-09T09:06:59.114638+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"219'4347","reported_seq":4363,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:53.229131+0000","last_change":"2023-10-09T09:06:27.620026+0000","last_active":"2023-10-09T09:07:53.229131+0000","last_peered":"2023-10-09T09:07:53.229131+0000","last_clean":"2023-10-09T09:07:53.229131+0000","last_became_active":"2023-10-09T09:06:27.619665+0000","last_became_peered":"2023-10-09T09:06:27.619665+0000","last_unstale":"2023-10-09T09:07:53.229131+0000","last_undegraded":"2023-10-09T09:07:53.229131+0000","last_fullsized":"2023-10-09T09:07:53.229131+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4347,"log_dups_size":0,"ondisk_log_size":4347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:13.420364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284884992,"num_objects":4347,"num_object_clones":0,"num_object_copies":13041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4347,"num_write_kb":278208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":187,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114093+0000","last_change":"2023-10-09T09:02:02.664617+0000","last_active":"2023-10-09T09:06:59.114093+0000","last_peered":"2023-10-09T09:06:59.114093+0000","last_clean":"2023-10-09T09:06:59.114093+0000","last_became_active":"2023-10-09T09:02:02.664305+0000","last_became_peered":"2023-10-09T09:02:02.664305+0000","last_unstale":"2023-10-09T09:06:59.114093+0000","last_undegraded":"2023-10-09T09:06:59.114093+0000","last_fullsized":"2023-10-09T09:06:59.114093+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:14.520177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":212,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111360+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:06:59.111360+0000","last_peered":"2023-10-09T09:06:59.111360+0000","last_clean":"2023-10-09T09:06:59.111360+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:06:59.111360+0000","last_undegraded":"2023-10-09T09:06:59.111360+0000","last_fullsized":"2023-10-09T09:06:59.111360+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"219'4306","reported_seq":4321,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.044883+0000","last_change":"2023-10-09T09:06:27.616764+0000","last_active":"2023-10-09T09:07:51.044883+0000","last_peered":"2023-10-09T09:07:51.044883+0000","last_clean":"2023-10-09T09:07:51.044883+0000","last_became_active":"2023-10-09T09:06:27.615844+0000","last_became_peered":"2023-10-09T09:06:27.615844+0000","last_unstale":"2023-10-09T09:07:51.044883+0000","last_undegraded":"2023-10-09T09:07:51.044883+0000","last_fullsized":"2023-10-09T09:07:51.044883+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4306,"log_dups_size":0,"ondisk_log_size":4306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:11.236973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282198016,"num_objects":4306,"num_object_clones":0,"num_object_copies":12918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4306,"num_write_kb":275584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":221,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.033740+0000","last_change":"2023-10-09T08:59:53.642095+0000","last_active":"2023-10-09T09:07:29.033740+0000","last_peered":"2023-10-09T09:07:29.033740+0000","last_clean":"2023-10-09T09:07:29.033740+0000","last_became_active":"2023-10-09T08:59:53.531588+0000","last_became_peered":"2023-10-09T08:59:53.531588+0000","last_unstale":"2023-10-09T09:07:29.033740+0000","last_undegraded":"2023-10-09T09:07:29.033740+0000","last_fullsized":"2023-10-09T09:07:29.033740+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.878230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111194+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:06:59.111194+0000","last_peered":"2023-10-09T09:06:59.111194+0000","last_clean":"2023-10-09T09:06:59.111194+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:06:59.111194+0000","last_undegraded":"2023-10-09T09:06:59.111194+0000","last_fullsized":"2023-10-09T09:06:59.111194+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"219'4308","reported_seq":4323,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.111358+0000","last_change":"2023-10-09T09:06:27.621010+0000","last_active":"2023-10-09T09:07:51.111358+0000","last_peered":"2023-10-09T09:07:51.111358+0000","last_clean":"2023-10-09T09:07:51.111358+0000","last_became_active":"2023-10-09T09:06:27.620675+0000","last_became_peered":"2023-10-09T09:06:27.620675+0000","last_unstale":"2023-10-09T09:07:51.111358+0000","last_undegraded":"2023-10-09T09:07:51.111358+0000","last_fullsized":"2023-10-09T09:07:51.111358+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4308,"log_dups_size":0,"ondisk_log_size":4308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282329088,"num_objects":4308,"num_object_clones":0,"num_object_copies":12924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4308,"num_write_kb":275712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114514+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:06:59.114514+0000","last_peered":"2023-10-09T09:06:59.114514+0000","last_clean":"2023-10-09T09:06:59.114514+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:06:59.114514+0000","last_undegraded":"2023-10-09T09:06:59.114514+0000","last_fullsized":"2023-10-09T09:06:59.114514+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":81,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111665+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:06:59.111665+0000","last_peered":"2023-10-09T09:06:59.111665+0000","last_clean":"2023-10-09T09:06:59.111665+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:06:59.111665+0000","last_undegraded":"2023-10-09T09:06:59.111665+0000","last_fullsized":"2023-10-09T09:06:59.111665+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"219'4335","reported_seq":4351,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.050995+0000","last_change":"2023-10-09T09:06:27.621711+0000","last_active":"2023-10-09T09:07:51.050995+0000","last_peered":"2023-10-09T09:07:51.050995+0000","last_clean":"2023-10-09T09:07:51.050995+0000","last_became_active":"2023-10-09T09:06:27.620532+0000","last_became_peered":"2023-10-09T09:06:27.620532+0000","last_unstale":"2023-10-09T09:07:51.050995+0000","last_undegraded":"2023-10-09T09:07:51.050995+0000","last_fullsized":"2023-10-09T09:07:51.050995+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4335,"log_dups_size":0,"ondisk_log_size":4335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:42.373323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284098560,"num_objects":4335,"num_object_clones":0,"num_object_copies":13005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4335,"num_write_kb":277440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":90,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110564+0000","last_change":"2023-10-09T09:03:19.563963+0000","last_active":"2023-10-09T09:06:59.110564+0000","last_peered":"2023-10-09T09:06:59.110564+0000","last_clean":"2023-10-09T09:06:59.110564+0000","last_became_active":"2023-10-09T09:03:19.563629+0000","last_became_peered":"2023-10-09T09:03:19.563629+0000","last_unstale":"2023-10-09T09:06:59.110564+0000","last_undegraded":"2023-10-09T09:06:59.110564+0000","last_fullsized":"2023-10-09T09:06:59.110564+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":90,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111616+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:06:59.111616+0000","last_peered":"2023-10-09T09:06:59.111616+0000","last_clean":"2023-10-09T09:06:59.111616+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:06:59.111616+0000","last_undegraded":"2023-10-09T09:06:59.111616+0000","last_fullsized":"2023-10-09T09:06:59.111616+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"219'4220","reported_seq":4235,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.062903+0000","last_change":"2023-10-09T09:06:27.617347+0000","last_active":"2023-10-09T09:07:51.062903+0000","last_peered":"2023-10-09T09:07:51.062903+0000","last_clean":"2023-10-09T09:07:51.062903+0000","last_became_active":"2023-10-09T09:06:27.615423+0000","last_became_peered":"2023-10-09T09:06:27.615423+0000","last_unstale":"2023-10-09T09:07:51.062903+0000","last_undegraded":"2023-10-09T09:07:51.062903+0000","last_fullsized":"2023-10-09T09:07:51.062903+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4220,"log_dups_size":0,"ondisk_log_size":4220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:53.926159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276561920,"num_objects":4220,"num_object_clones":0,"num_object_copies":12660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4220,"num_write_kb":270080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":38,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.077653+0000","last_change":"2023-10-09T09:03:19.572389+0000","last_active":"2023-10-09T09:07:29.077653+0000","last_peered":"2023-10-09T09:07:29.077653+0000","last_clean":"2023-10-09T09:07:29.077653+0000","last_became_active":"2023-10-09T09:03:19.572091+0000","last_became_peered":"2023-10-09T09:03:19.572091+0000","last_unstale":"2023-10-09T09:07:29.077653+0000","last_undegraded":"2023-10-09T09:07:29.077653+0000","last_fullsized":"2023-10-09T09:07:29.077653+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:27.702718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":175,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.117279+0000","last_change":"2023-10-09T09:02:02.636996+0000","last_active":"2023-10-09T09:06:59.117279+0000","last_peered":"2023-10-09T09:06:59.117279+0000","last_clean":"2023-10-09T09:06:59.117279+0000","last_became_active":"2023-10-09T09:02:02.636702+0000","last_became_peered":"2023-10-09T09:02:02.636702+0000","last_unstale":"2023-10-09T09:06:59.117279+0000","last_undegraded":"2023-10-09T09:06:59.117279+0000","last_fullsized":"2023-10-09T09:06:59.117279+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:45.392645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"219'4342","reported_seq":4358,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.034370+0000","last_change":"2023-10-09T09:06:27.616361+0000","last_active":"2023-10-09T09:07:51.034370+0000","last_peered":"2023-10-09T09:07:51.034370+0000","last_clean":"2023-10-09T09:07:51.034370+0000","last_became_active":"2023-10-09T09:06:27.616004+0000","last_became_peered":"2023-10-09T09:06:27.616004+0000","last_unstale":"2023-10-09T09:07:51.034370+0000","last_undegraded":"2023-10-09T09:07:51.034370+0000","last_fullsized":"2023-10-09T09:07:51.034370+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4342,"log_dups_size":0,"ondisk_log_size":4342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:10.699277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284557312,"num_objects":4342,"num_object_clones":0,"num_object_copies":13026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4342,"num_write_kb":277888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":162,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111276+0000","last_change":"2023-10-09T09:03:19.556450+0000","last_active":"2023-10-09T09:06:59.111276+0000","last_peered":"2023-10-09T09:06:59.111276+0000","last_clean":"2023-10-09T09:06:59.111276+0000","last_became_active":"2023-10-09T09:03:19.556190+0000","last_became_peered":"2023-10-09T09:03:19.556190+0000","last_unstale":"2023-10-09T09:06:59.111276+0000","last_undegraded":"2023-10-09T09:06:59.111276+0000","last_fullsized":"2023-10-09T09:06:59.111276+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:23.704455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":157,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.116107+0000","last_change":"2023-10-09T09:01:36.285844+0000","last_active":"2023-10-09T09:06:59.116107+0000","last_peered":"2023-10-09T09:06:59.116107+0000","last_clean":"2023-10-09T09:06:59.116107+0000","last_became_active":"2023-10-09T09:01:36.285525+0000","last_became_peered":"2023-10-09T09:01:36.285525+0000","last_unstale":"2023-10-09T09:06:59.116107+0000","last_undegraded":"2023-10-09T09:06:59.116107+0000","last_fullsized":"2023-10-09T09:06:59.116107+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:05.624756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":198,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110965+0000","last_change":"2023-10-09T09:02:52.021749+0000","last_active":"2023-10-09T09:06:59.110965+0000","last_peered":"2023-10-09T09:06:59.110965+0000","last_clean":"2023-10-09T09:06:59.110965+0000","last_became_active":"2023-10-09T09:02:52.021416+0000","last_became_peered":"2023-10-09T09:02:52.021416+0000","last_unstale":"2023-10-09T09:06:59.110965+0000","last_undegraded":"2023-10-09T09:06:59.110965+0000","last_fullsized":"2023-10-09T09:06:59.110965+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"219'4332","reported_seq":4348,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:52.696143+0000","last_change":"2023-10-09T09:06:27.620398+0000","last_active":"2023-10-09T09:07:52.696143+0000","last_peered":"2023-10-09T09:07:52.696143+0000","last_clean":"2023-10-09T09:07:52.696143+0000","last_became_active":"2023-10-09T09:06:27.619485+0000","last_became_peered":"2023-10-09T09:06:27.619485+0000","last_unstale":"2023-10-09T09:07:52.696143+0000","last_undegraded":"2023-10-09T09:07:52.696143+0000","last_fullsized":"2023-10-09T09:07:52.696143+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4332,"log_dups_size":0,"ondisk_log_size":4332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:19.177497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283901952,"num_objects":4332,"num_object_clones":0,"num_object_copies":12996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4332,"num_write_kb":277248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":186,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114485+0000","last_change":"2023-10-09T09:03:19.564803+0000","last_active":"2023-10-09T09:06:59.114485+0000","last_peered":"2023-10-09T09:06:59.114485+0000","last_clean":"2023-10-09T09:06:59.114485+0000","last_became_active":"2023-10-09T09:03:19.564520+0000","last_became_peered":"2023-10-09T09:03:19.564520+0000","last_unstale":"2023-10-09T09:06:59.114485+0000","last_undegraded":"2023-10-09T09:06:59.114485+0000","last_fullsized":"2023-10-09T09:06:59.114485+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111183+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:06:59.111183+0000","last_peered":"2023-10-09T09:06:59.111183+0000","last_clean":"2023-10-09T09:06:59.111183+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:06:59.111183+0000","last_undegraded":"2023-10-09T09:06:59.111183+0000","last_fullsized":"2023-10-09T09:06:59.111183+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":186,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115282+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:06:59.115282+0000","last_peered":"2023-10-09T09:06:59.115282+0000","last_clean":"2023-10-09T09:06:59.115282+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:06:59.115282+0000","last_undegraded":"2023-10-09T09:06:59.115282+0000","last_fullsized":"2023-10-09T09:06:59.115282+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"219'4230","reported_seq":4246,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.060610+0000","last_change":"2023-10-09T09:06:27.617113+0000","last_active":"2023-10-09T09:07:51.060610+0000","last_peered":"2023-10-09T09:07:51.060610+0000","last_clean":"2023-10-09T09:07:51.060610+0000","last_became_active":"2023-10-09T09:06:27.615429+0000","last_became_peered":"2023-10-09T09:06:27.615429+0000","last_unstale":"2023-10-09T09:07:51.060610+0000","last_undegraded":"2023-10-09T09:07:51.060610+0000","last_fullsized":"2023-10-09T09:07:51.060610+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4230,"log_dups_size":0,"ondisk_log_size":4230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:07.620043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277217280,"num_objects":4230,"num_object_clones":0,"num_object_copies":12690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4230,"num_write_kb":270720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":245,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.086912+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:07:29.086912+0000","last_peered":"2023-10-09T09:07:29.086912+0000","last_clean":"2023-10-09T09:07:29.086912+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:07:29.086912+0000","last_undegraded":"2023-10-09T09:07:29.086912+0000","last_fullsized":"2023-10-09T09:07:29.086912+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":179,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114142+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:06:59.114142+0000","last_peered":"2023-10-09T09:06:59.114142+0000","last_clean":"2023-10-09T09:06:59.114142+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:06:59.114142+0000","last_undegraded":"2023-10-09T09:06:59.114142+0000","last_fullsized":"2023-10-09T09:06:59.114142+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":226,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.112349+0000","last_change":"2023-10-09T09:03:19.575546+0000","last_active":"2023-10-09T09:06:59.112349+0000","last_peered":"2023-10-09T09:06:59.112349+0000","last_clean":"2023-10-09T09:06:59.112349+0000","last_became_active":"2023-10-09T09:03:19.575075+0000","last_became_peered":"2023-10-09T09:03:19.575075+0000","last_unstale":"2023-10-09T09:06:59.112349+0000","last_undegraded":"2023-10-09T09:06:59.112349+0000","last_fullsized":"2023-10-09T09:06:59.112349+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:14.350886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"219'4351","reported_seq":4365,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.043797+0000","last_change":"2023-10-09T09:06:27.620038+0000","last_active":"2023-10-09T09:07:51.043797+0000","last_peered":"2023-10-09T09:07:51.043797+0000","last_clean":"2023-10-09T09:07:51.043797+0000","last_became_active":"2023-10-09T09:06:27.619703+0000","last_became_peered":"2023-10-09T09:06:27.619703+0000","last_unstale":"2023-10-09T09:07:51.043797+0000","last_undegraded":"2023-10-09T09:07:51.043797+0000","last_fullsized":"2023-10-09T09:07:51.043797+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4351,"log_dups_size":0,"ondisk_log_size":4351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:51.146805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285147136,"num_objects":4351,"num_object_clones":0,"num_object_copies":13053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4351,"num_write_kb":278464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":151,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.090756+0000","last_change":"2023-10-09T09:01:36.274911+0000","last_active":"2023-10-09T09:07:29.090756+0000","last_peered":"2023-10-09T09:07:29.090756+0000","last_clean":"2023-10-09T09:07:29.090756+0000","last_became_active":"2023-10-09T09:01:36.274560+0000","last_became_peered":"2023-10-09T09:01:36.274560+0000","last_unstale":"2023-10-09T09:07:29.090756+0000","last_undegraded":"2023-10-09T09:07:29.090756+0000","last_fullsized":"2023-10-09T09:07:29.090756+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:55.305969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":459,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.107089+0000","last_change":"2023-10-09T09:02:05.534070+0000","last_active":"2023-10-09T09:06:59.107089+0000","last_peered":"2023-10-09T09:06:59.107089+0000","last_clean":"2023-10-09T09:06:59.107089+0000","last_became_active":"2023-10-09T09:02:05.533567+0000","last_became_peered":"2023-10-09T09:02:05.533567+0000","last_unstale":"2023-10-09T09:06:59.107089+0000","last_undegraded":"2023-10-09T09:06:59.107089+0000","last_fullsized":"2023-10-09T09:06:59.107089+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:59.568077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":38,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.082473+0000","last_change":"2023-10-09T09:03:19.562659+0000","last_active":"2023-10-09T09:07:29.082473+0000","last_peered":"2023-10-09T09:07:29.082473+0000","last_clean":"2023-10-09T09:07:29.082473+0000","last_became_active":"2023-10-09T09:03:19.562284+0000","last_became_peered":"2023-10-09T09:03:19.562284+0000","last_unstale":"2023-10-09T09:07:29.082473+0000","last_undegraded":"2023-10-09T09:07:29.082473+0000","last_fullsized":"2023-10-09T09:07:29.082473+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:26.537954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115816+0000","last_change":"2023-10-09T09:00:31.859121+0000","last_active":"2023-10-09T09:06:59.115816+0000","last_peered":"2023-10-09T09:06:59.115816+0000","last_clean":"2023-10-09T09:06:59.115816+0000","last_became_active":"2023-10-09T09:00:31.858675+0000","last_became_peered":"2023-10-09T09:00:31.858675+0000","last_unstale":"2023-10-09T09:06:59.115816+0000","last_undegraded":"2023-10-09T09:06:59.115816+0000","last_fullsized":"2023-10-09T09:06:59.115816+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"219'4536","reported_seq":4552,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:52.737349+0000","last_change":"2023-10-09T09:06:27.620287+0000","last_active":"2023-10-09T09:07:52.737349+0000","last_peered":"2023-10-09T09:07:52.737349+0000","last_clean":"2023-10-09T09:07:52.737349+0000","last_became_active":"2023-10-09T09:06:27.619875+0000","last_became_peered":"2023-10-09T09:06:27.619875+0000","last_unstale":"2023-10-09T09:07:52.737349+0000","last_undegraded":"2023-10-09T09:07:52.737349+0000","last_fullsized":"2023-10-09T09:07:52.737349+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4536,"log_dups_size":0,"ondisk_log_size":4536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:37.325171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297271296,"num_objects":4536,"num_object_clones":0,"num_object_copies":13608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4536,"num_write_kb":290304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":113,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.107191+0000","last_change":"2023-10-09T09:02:02.649488+0000","last_active":"2023-10-09T09:06:59.107191+0000","last_peered":"2023-10-09T09:06:59.107191+0000","last_clean":"2023-10-09T09:06:59.107191+0000","last_became_active":"2023-10-09T09:02:02.645550+0000","last_became_peered":"2023-10-09T09:02:02.645550+0000","last_unstale":"2023-10-09T09:06:59.107191+0000","last_undegraded":"2023-10-09T09:06:59.107191+0000","last_fullsized":"2023-10-09T09:06:59.107191+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:40.192731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":249,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.109386+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:06:59.109386+0000","last_peered":"2023-10-09T09:06:59.109386+0000","last_clean":"2023-10-09T09:06:59.109386+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:06:59.109386+0000","last_undegraded":"2023-10-09T09:06:59.109386+0000","last_fullsized":"2023-10-09T09:06:59.109386+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":210,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:27.711840+0000","last_change":"2023-10-09T09:02:02.657733+0000","last_active":"2023-10-09T09:07:27.711840+0000","last_peered":"2023-10-09T09:07:27.711840+0000","last_clean":"2023-10-09T09:07:27.711840+0000","last_became_active":"2023-10-09T09:02:02.656701+0000","last_became_peered":"2023-10-09T09:02:02.656701+0000","last_unstale":"2023-10-09T09:07:27.711840+0000","last_undegraded":"2023-10-09T09:07:27.711840+0000","last_fullsized":"2023-10-09T09:07:27.711840+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:39.552401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"219'4506","reported_seq":4521,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:55.409914+0000","last_change":"2023-10-09T09:06:27.613599+0000","last_active":"2023-10-09T09:07:55.409914+0000","last_peered":"2023-10-09T09:07:55.409914+0000","last_clean":"2023-10-09T09:07:55.409914+0000","last_became_active":"2023-10-09T09:06:27.613327+0000","last_became_peered":"2023-10-09T09:06:27.613327+0000","last_unstale":"2023-10-09T09:07:55.409914+0000","last_undegraded":"2023-10-09T09:07:55.409914+0000","last_fullsized":"2023-10-09T09:07:55.409914+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295305216,"num_objects":4506,"num_object_clones":0,"num_object_copies":13518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4506,"num_write_kb":288384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":193,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111676+0000","last_change":"2023-10-09T09:02:02.644204+0000","last_active":"2023-10-09T09:06:59.111676+0000","last_peered":"2023-10-09T09:06:59.111676+0000","last_clean":"2023-10-09T09:06:59.111676+0000","last_became_active":"2023-10-09T09:02:02.643713+0000","last_became_peered":"2023-10-09T09:02:02.643713+0000","last_unstale":"2023-10-09T09:06:59.111676+0000","last_undegraded":"2023-10-09T09:06:59.111676+0000","last_fullsized":"2023-10-09T09:06:59.111676+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:20.811353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":188,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115221+0000","last_change":"2023-10-09T09:02:02.647803+0000","last_active":"2023-10-09T09:06:59.115221+0000","last_peered":"2023-10-09T09:06:59.115221+0000","last_clean":"2023-10-09T09:06:59.115221+0000","last_became_active":"2023-10-09T09:02:02.646911+0000","last_became_peered":"2023-10-09T09:02:02.646911+0000","last_unstale":"2023-10-09T09:06:59.115221+0000","last_undegraded":"2023-10-09T09:06:59.115221+0000","last_fullsized":"2023-10-09T09:06:59.115221+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:35.294072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":207,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110098+0000","last_change":"2023-10-09T09:02:52.016015+0000","last_active":"2023-10-09T09:06:59.110098+0000","last_peered":"2023-10-09T09:06:59.110098+0000","last_clean":"2023-10-09T09:06:59.110098+0000","last_became_active":"2023-10-09T09:02:52.015543+0000","last_became_peered":"2023-10-09T09:02:52.015543+0000","last_unstale":"2023-10-09T09:06:59.110098+0000","last_undegraded":"2023-10-09T09:06:59.110098+0000","last_fullsized":"2023-10-09T09:06:59.110098+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.410980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"219'4416","reported_seq":4431,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:55.382273+0000","last_change":"2023-10-09T09:06:27.615162+0000","last_active":"2023-10-09T09:07:55.382273+0000","last_peered":"2023-10-09T09:07:55.382273+0000","last_clean":"2023-10-09T09:07:55.382273+0000","last_became_active":"2023-10-09T09:06:27.614815+0000","last_became_peered":"2023-10-09T09:06:27.614815+0000","last_unstale":"2023-10-09T09:07:55.382273+0000","last_undegraded":"2023-10-09T09:07:55.382273+0000","last_fullsized":"2023-10-09T09:07:55.382273+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4416,"log_dups_size":0,"ondisk_log_size":4416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4416,"num_write_kb":282624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":18,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115457+0000","last_change":"2023-10-09T09:06:50.681508+0000","last_active":"2023-10-09T09:06:59.115457+0000","last_peered":"2023-10-09T09:06:59.115457+0000","last_clean":"2023-10-09T09:06:59.115457+0000","last_became_active":"2023-10-09T09:06:50.678633+0000","last_became_peered":"2023-10-09T09:06:50.678633+0000","last_unstale":"2023-10-09T09:06:59.115457+0000","last_undegraded":"2023-10-09T09:06:59.115457+0000","last_fullsized":"2023-10-09T09:06:59.115457+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:02.327990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":181,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115266+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:06:59.115266+0000","last_peered":"2023-10-09T09:06:59.115266+0000","last_clean":"2023-10-09T09:06:59.115266+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:06:59.115266+0000","last_undegraded":"2023-10-09T09:06:59.115266+0000","last_fullsized":"2023-10-09T09:06:59.115266+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.872097+0000","last_change":"2023-10-09T09:03:19.572475+0000","last_active":"2023-10-09T09:07:20.872097+0000","last_peered":"2023-10-09T09:07:20.872097+0000","last_clean":"2023-10-09T09:07:20.872097+0000","last_became_active":"2023-10-09T09:03:19.572081+0000","last_became_peered":"2023-10-09T09:03:19.572081+0000","last_unstale":"2023-10-09T09:07:20.872097+0000","last_undegraded":"2023-10-09T09:07:20.872097+0000","last_fullsized":"2023-10-09T09:07:20.872097+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:04.771178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"219'4166","reported_seq":4182,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:51.091432+0000","last_change":"2023-10-09T09:06:27.616627+0000","last_active":"2023-10-09T09:07:51.091432+0000","last_peered":"2023-10-09T09:07:51.091432+0000","last_clean":"2023-10-09T09:07:51.091432+0000","last_became_active":"2023-10-09T09:06:27.616284+0000","last_became_peered":"2023-10-09T09:06:27.616284+0000","last_unstale":"2023-10-09T09:07:51.091432+0000","last_undegraded":"2023-10-09T09:07:51.091432+0000","last_fullsized":"2023-10-09T09:07:51.091432+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":4166,"log_dups_size":0,"ondisk_log_size":4166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:23.626143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273022976,"num_objects":4166,"num_object_clones":0,"num_object_copies":12498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4166,"num_write_kb":266624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":68,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.115559+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:06:59.115559+0000","last_peered":"2023-10-09T09:06:59.115559+0000","last_clean":"2023-10-09T09:06:59.115559+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:06:59.115559+0000","last_undegraded":"2023-10-09T09:06:59.115559+0000","last_fullsized":"2023-10-09T09:06:59.115559+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":229,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:27.711855+0000","last_change":"2023-10-09T09:02:02.674657+0000","last_active":"2023-10-09T09:07:27.711855+0000","last_peered":"2023-10-09T09:07:27.711855+0000","last_clean":"2023-10-09T09:07:27.711855+0000","last_became_active":"2023-10-09T09:02:02.657392+0000","last_became_peered":"2023-10-09T09:02:02.657392+0000","last_unstale":"2023-10-09T09:07:27.711855+0000","last_undegraded":"2023-10-09T09:07:27.711855+0000","last_fullsized":"2023-10-09T09:07:27.711855+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:16.286583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":90,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110654+0000","last_change":"2023-10-09T09:01:36.280537+0000","last_active":"2023-10-09T09:06:59.110654+0000","last_peered":"2023-10-09T09:06:59.110654+0000","last_clean":"2023-10-09T09:06:59.110654+0000","last_became_active":"2023-10-09T09:01:36.280149+0000","last_became_peered":"2023-10-09T09:01:36.280149+0000","last_unstale":"2023-10-09T09:06:59.110654+0000","last_undegraded":"2023-10-09T09:06:59.110654+0000","last_fullsized":"2023-10-09T09:06:59.110654+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":227,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110013+0000","last_change":"2023-10-09T09:03:19.554339+0000","last_active":"2023-10-09T09:06:59.110013+0000","last_peered":"2023-10-09T09:06:59.110013+0000","last_clean":"2023-10-09T09:06:59.110013+0000","last_became_active":"2023-10-09T09:03:19.553953+0000","last_became_peered":"2023-10-09T09:03:19.553953+0000","last_unstale":"2023-10-09T09:06:59.110013+0000","last_undegraded":"2023-10-09T09:06:59.110013+0000","last_fullsized":"2023-10-09T09:06:59.110013+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:50.601515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":50,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T09:07:29.087322+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:07:29.087322+0000","last_peered":"2023-10-09T09:07:29.087322+0000","last_clean":"2023-10-09T09:07:29.087322+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:07:29.087322+0000","last_undegraded":"2023-10-09T09:07:29.087322+0000","last_fullsized":"2023-10-09T09:07:29.087322+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":221,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111143+0000","last_change":"2023-10-09T09:02:52.022788+0000","last_active":"2023-10-09T09:06:59.111143+0000","last_peered":"2023-10-09T09:06:59.111143+0000","last_clean":"2023-10-09T09:06:59.111143+0000","last_became_active":"2023-10-09T09:02:52.016418+0000","last_became_peered":"2023-10-09T09:02:52.016418+0000","last_unstale":"2023-10-09T09:06:59.111143+0000","last_undegraded":"2023-10-09T09:06:59.111143+0000","last_fullsized":"2023-10-09T09:06:59.111143+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":81,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111031+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:06:59.111031+0000","last_peered":"2023-10-09T09:06:59.111031+0000","last_clean":"2023-10-09T09:06:59.111031+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:06:59.111031+0000","last_undegraded":"2023-10-09T09:06:59.111031+0000","last_fullsized":"2023-10-09T09:06:59.111031+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":206,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.111204+0000","last_change":"2023-10-09T09:03:19.575218+0000","last_active":"2023-10-09T09:06:59.111204+0000","last_peered":"2023-10-09T09:06:59.111204+0000","last_clean":"2023-10-09T09:06:59.111204+0000","last_became_active":"2023-10-09T09:03:19.557578+0000","last_became_peered":"2023-10-09T09:03:19.557578+0000","last_unstale":"2023-10-09T09:06:59.111204+0000","last_undegraded":"2023-10-09T09:06:59.111204+0000","last_fullsized":"2023-10-09T09:06:59.111204+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":72,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110623+0000","last_change":"2023-10-09T08:59:53.522794+0000","last_active":"2023-10-09T09:06:59.110623+0000","last_peered":"2023-10-09T09:06:59.110623+0000","last_clean":"2023-10-09T09:06:59.110623+0000","last_became_active":"2023-10-09T08:59:53.522419+0000","last_became_peered":"2023-10-09T08:59:53.522419+0000","last_unstale":"2023-10-09T09:06:59.110623+0000","last_undegraded":"2023-10-09T09:06:59.110623+0000","last_fullsized":"2023-10-09T09:06:59.110623+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":221,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:07:20.872230+0000","last_change":"2023-10-09T09:00:31.845771+0000","last_active":"2023-10-09T09:07:20.872230+0000","last_peered":"2023-10-09T09:07:20.872230+0000","last_clean":"2023-10-09T09:07:20.872230+0000","last_became_active":"2023-10-09T09:00:31.844775+0000","last_became_peered":"2023-10-09T09:00:31.844775+0000","last_unstale":"2023-10-09T09:07:20.872230+0000","last_undegraded":"2023-10-09T09:07:20.872230+0000","last_fullsized":"2023-10-09T09:07:20.872230+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:33.452539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":37,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.114119+0000","last_change":"2023-10-09T09:03:19.561924+0000","last_active":"2023-10-09T09:06:59.114119+0000","last_peered":"2023-10-09T09:06:59.114119+0000","last_clean":"2023-10-09T09:06:59.114119+0000","last_became_active":"2023-10-09T09:03:19.557692+0000","last_became_peered":"2023-10-09T09:03:19.557692+0000","last_unstale":"2023-10-09T09:06:59.114119+0000","last_undegraded":"2023-10-09T09:06:59.114119+0000","last_fullsized":"2023-10-09T09:06:59.114119+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:39.940109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":280,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.112450+0000","last_change":"2023-10-09T09:02:52.019620+0000","last_active":"2023-10-09T09:06:59.112450+0000","last_peered":"2023-10-09T09:06:59.112450+0000","last_clean":"2023-10-09T09:06:59.112450+0000","last_became_active":"2023-10-09T09:02:52.014489+0000","last_became_peered":"2023-10-09T09:02:52.014489+0000","last_unstale":"2023-10-09T09:06:59.112450+0000","last_undegraded":"2023-10-09T09:06:59.112450+0000","last_fullsized":"2023-10-09T09:06:59.112450+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:39.550600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":99,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T09:06:59.110478+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:06:59.110478+0000","last_peered":"2023-10-09T09:06:59.110478+0000","last_clean":"2023-10-09T09:06:59.110478+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:06:59.110478+0000","last_undegraded":"2023-10-09T09:06:59.110478+0000","last_fullsized":"2023-10-09T09:06:59.110478+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":13474,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":139264,"data_stored":22108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4547870720,"num_objects":69395,"num_object_clones":0,"num_object_copies":208185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":69395,"num_write_kb":4441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13655670784,"data_stored":13655670784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69395,"ondisk_log_size":69395,"up":48,"acting":48,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430649,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626881,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3894184,"kb_used_data":2990816,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":89850968,"statfs":{"total":95995035648,"available":92007391232,"internally_reserved":0,"allocated":3062595584,"data_stored":3062096570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:07:39 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.2789999999999999,"15min":2.3719999999999999},"min":{"1min":0.51900000000000002,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":19.568999999999999,"5min":19.568999999999999,"15min":153.63800000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.048,"5min":1.5569999999999999,"15min":2.3929999999999998},"min":{"1min":0.53700000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":19.498000000000001,"5min":19.498000000000001,"15min":153.72200000000001},"last":0.70799999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:06:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.69899999999999995,"15min":1.8160000000000001},"min":{"1min":0.42199999999999999,"5min":0.32000000000000001,"15min":0.28299999999999997},"max":{"1min":1.236,"5min":3.6739999999999999,"15min":109.453},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.96499999999999997,"15min":1.931},"min":{"1min":0.46600000000000003,"5min":0.316,"15min":0.314},"max":{"1min":1.28,"5min":9.1869999999999994,"15min":111.47499999999999},"last":1.1220000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:07:26 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.1779999999999999,"15min":1.411},"min":{"1min":0.315,"5min":0.315,"15min":0.29799999999999999},"max":{"1min":18.756,"5min":18.756,"15min":18.756},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.865,"5min":1.3680000000000001,"15min":1.5229999999999999},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.28100000000000003},"max":{"1min":18.841999999999999,"5min":18.841999999999999,"15min":18.841999999999999},"last":0.435}]},{"osd":3,"last update":"Mon Oct 9 09:06:57 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.2629999999999999,"15min":1.8480000000000001},"min":{"1min":0.54100000000000004,"5min":0.311,"15min":0.311},"max":{"1min":4.3029999999999999,"5min":6.992,"15min":55.713000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.127,"5min":1.6459999999999999,"15min":1.732},"min":{"1min":0.68000000000000005,"5min":0.46500000000000002,"15min":0.34699999999999998},"max":{"1min":5.1639999999999997,"5min":9.4429999999999996,"15min":56.064},"last":0.747}]},{"osd":4,"last update":"Mon Oct 9 09:07:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.109,"15min":1.452},"min":{"1min":0.49399999999999999,"5min":0.314,"15min":0.314},"max":{"1min":2.194,"5min":9.2639999999999993,"15min":32.280999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.268,"5min":0.96199999999999997,"15min":1.1259999999999999},"min":{"1min":0.44700000000000001,"5min":0.35399999999999998,"15min":0.30599999999999999},"max":{"1min":4.2830000000000004,"5min":4.2830000000000004,"15min":34.689},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:07:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.1599999999999999,"15min":1.23},"min":{"1min":0.61499999999999999,"5min":0.41399999999999998,"15min":0.35499999999999998},"max":{"1min":2.6779999999999999,"5min":9.218,"15min":10.349},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.89100000000000001,"15min":1.0649999999999999},"min":{"1min":0.52700000000000002,"5min":0.41599999999999998,"15min":0.33900000000000002},"max":{"1min":1.6240000000000001,"5min":3.1259999999999999,"15min":8.8569999999999993},"last":0.69399999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:07:39 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1439999999999999,"15min":1.3919999999999999},"min":{"1min":0.68700000000000006,"5min":0.47599999999999998,"15min":0.42999999999999999},"max":{"1min":2.2320000000000002,"5min":9.2460000000000004,"15min":11.382},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.048,"5min":1.018,"15min":1.2130000000000001},"min":{"1min":0.57499999999999996,"5min":0.45400000000000001,"15min":0.39000000000000001},"max":{"1min":1.677,"5min":4.3259999999999996,"15min":9.4800000000000004},"last":0.78700000000000003}]}]},{"osd":1,"up_from":0,"seq":356482285829,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449355,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3583976,"kb_used_data":3049312,"kb_used_omap":28,"kb_used_meta":534627,"kb_avail":90161176,"statfs":{"total":95995035648,"available":92325044224,"internally_reserved":0,"allocated":3122495488,"data_stored":3121644965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":547458896},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.113,"15min":1.4610000000000001},"min":{"1min":0.28799999999999998,"5min":0.22,"15min":0.22},"max":{"1min":1.097,"5min":6.7649999999999997,"15min":9.9879999999999995},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.1240000000000001,"15min":1.512},"min":{"1min":0.27200000000000002,"5min":0.254,"15min":0.21199999999999999},"max":{"1min":1.1879999999999999,"5min":10.102,"15min":18.661000000000001},"last":0.77900000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":2.0569999999999999,"15min":2.4350000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.835,"5min":10.202999999999999,"15min":71.322999999999993},"last":0.624},{"interface":"front","average":{"1min":1.3149999999999999,"5min":4.3680000000000003,"15min":3.3340000000000001},"min":{"1min":0.33200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.399,"5min":219.00700000000001,"15min":219.00700000000001},"last":0.91700000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:07:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":4.5019999999999998,"15min":2.8260000000000001},"min":{"1min":0.53200000000000003,"5min":0.32300000000000001,"15min":0.27700000000000002},"max":{"1min":8.9079999999999995,"5min":219.041,"15min":219.041},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.631,"5min":2.0270000000000001,"15min":1.9650000000000001},"min":{"1min":0.52500000000000002,"5min":0.29499999999999998,"15min":0.25},"max":{"1min":6.4459999999999997,"5min":20.466000000000001,"15min":38.270000000000003},"last":0.72999999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":1.4890000000000001,"15min":1.504},"min":{"1min":0.53000000000000003,"5min":0.443,"15min":0.30199999999999999},"max":{"1min":3.298,"5min":10.178000000000001,"15min":10.178000000000001},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.464,"15min":1.6699999999999999},"min":{"1min":0.36799999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.7910000000000004,"5min":10.526,"15min":11.654},"last":3.665}]},{"osd":5,"last update":"Mon Oct 9 09:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":1.54,"15min":1.429},"min":{"1min":0.41299999999999998,"5min":0.40500000000000003,"15min":0.34499999999999997},"max":{"1min":5.3019999999999996,"5min":9.3030000000000008,"15min":9.3689999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.03,"5min":1.5429999999999999,"15min":1.591},"min":{"1min":0.52000000000000002,"5min":0.48399999999999999,"15min":0.33700000000000002},"max":{"1min":3.4380000000000002,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.83999999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.7230000000000001,"15min":1.8660000000000001},"min":{"1min":0.48599999999999999,"5min":0.37,"15min":0.37},"max":{"1min":5.3410000000000002,"5min":10.196999999999999,"15min":62.850999999999999},"last":1.3400000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.712,"15min":2.0739999999999998},"min":{"1min":0.54900000000000004,"5min":0.48999999999999999,"15min":0.36699999999999999},"max":{"1min":3.3719999999999999,"5min":10.632999999999999,"15min":62.816000000000003},"last":3.5190000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.52,"15min":1.643},"min":{"1min":0.56599999999999995,"5min":0.54800000000000004,"15min":0.42499999999999999},"max":{"1min":3.4740000000000002,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.345,"5min":1.4530000000000001,"15min":1.8180000000000001},"min":{"1min":0.50700000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.944,"5min":10.662000000000001,"15min":14.359},"last":3.7029999999999998}]}]},{"osd":2,"up_from":88,"seq":377957122303,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2862620,"kb_used_data":2239892,"kb_used_omap":81,"kb_used_meta":622638,"kb_avail":90882532,"statfs":{"total":95995035648,"available":93063712768,"internally_reserved":0,"allocated":2293649408,"data_stored":2292755575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":637581530},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:07:41 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":3.3919999999999999,"15min":3.4689999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":57.420999999999999,"5min":68.457999999999998,"15min":134.458},"last":1.4199999999999999},{"interface":"front","average":{"1min":3.673,"5min":3.234,"15min":3.0830000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.221},"max":{"1min":56.779000000000003,"5min":68.162999999999997,"15min":138.417},"last":4.0990000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:07:41 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.2969999999999999,"15min":2.2349999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.21299999999999999},"max":{"1min":3.927,"5min":8.1560000000000006,"15min":57.939999999999998},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.0780000000000001,"15min":1.998},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.1880000000000002,"5min":8.9139999999999997,"15min":57.122999999999998},"last":0.60899999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:06:55 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":4.2990000000000004,"15min":2.6669999999999998},"min":{"1min":0.43099999999999999,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":2.1280000000000001,"5min":112.47499999999999,"15min":112.47499999999999},"last":4.133},{"interface":"front","average":{"1min":0.89800000000000002,"5min":4.7699999999999996,"15min":2.9700000000000002},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.247},"max":{"1min":5.2450000000000001,"5min":112.929,"15min":112.929},"last":3.4329999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:07:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.746,"15min":1.883},"min":{"1min":0.34499999999999997,"5min":0.316,"15min":0.316},"max":{"1min":2.7010000000000001,"5min":17.408999999999999,"15min":17.408999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.554,"15min":1.8360000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":4.3470000000000004,"5min":8.1850000000000005,"15min":14.071999999999999},"last":0.69999999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:07:41 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.3049999999999999,"15min":1.448},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":3.266,"5min":8.8239999999999998,"15min":12.411},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.776,"15min":1.8879999999999999},"min":{"1min":0.39700000000000002,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":2.649,"5min":17.375,"15min":17.375},"last":0.64200000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:07:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.6180000000000001,"15min":1.865},"min":{"1min":0.47699999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.4370000000000003,"5min":8.1129999999999995,"15min":10.456},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.4119999999999999,"15min":1.5580000000000001},"min":{"1min":0.49199999999999999,"5min":0.41499999999999998,"15min":0.39300000000000002},"max":{"1min":3.2469999999999999,"5min":6.4720000000000004,"15min":12.381},"last":0.72099999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:07:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":2.3149999999999999,"15min":2.069},"min":{"1min":0.51400000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":4.3979999999999997,"5min":67.491,"15min":67.491},"last":0.83399999999999996},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.393,"15min":1.506},"min":{"1min":0.48199999999999998,"5min":0.44400000000000001,"15min":0.38500000000000001},"max":{"1min":3.2850000000000001,"5min":8.8789999999999996,"15min":12.438000000000001},"last":4.0220000000000002}]}]},{"osd":3,"up_from":93,"seq":399431958777,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3801872,"kb_used_data":3249160,"kb_used_omap":7,"kb_used_meta":552696,"kb_avail":89943280,"statfs":{"total":95995035648,"available":92101918720,"internally_reserved":0,"allocated":3327139840,"data_stored":3326310334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":565961320},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":3.7919999999999998,"15min":2.3399999999999999},"min":{"1min":0.30399999999999999,"5min":0.19500000000000001,"15min":0.19400000000000001},"max":{"1min":1.3360000000000001,"5min":233.22900000000001,"15min":233.22900000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.63700000000000001,"5min":4.0899999999999999,"15min":2.484},"min":{"1min":0.39600000000000002,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":1.2110000000000001,"5min":237.447,"15min":237.447},"last":3.9470000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":4.6820000000000004,"15min":2.6949999999999998},"min":{"1min":0.315,"5min":0.31,"15min":0.22500000000000001},"max":{"1min":1,"5min":326.75700000000001,"15min":326.75700000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":4.4370000000000003,"15min":2.6059999999999999},"min":{"1min":0.39700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.5529999999999999,"5min":327.024,"15min":327.024},"last":0.61699999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":2.431,"15min":2.4089999999999998},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.255},"max":{"1min":1.071,"5min":43.683999999999997,"15min":75.641999999999996},"last":0.371},{"interface":"front","average":{"1min":0.70199999999999996,"5min":2.008,"15min":2.1960000000000002},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.034,"5min":45.027999999999999,"15min":75.207999999999998},"last":2.254}]},{"osd":4,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.1819999999999999,"15min":1.387},"min":{"1min":0.65300000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.208,"5min":8.9879999999999995,"15min":9.2940000000000005},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.5489999999999999,"15min":1.6519999999999999},"min":{"1min":0.59299999999999997,"5min":0.36399999999999999,"15min":0.33500000000000002},"max":{"1min":1.0720000000000001,"5min":9.4649999999999999,"15min":10.486000000000001},"last":3.9830000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.415,"15min":1.544},"min":{"1min":0.66200000000000003,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":1.21,"5min":9.4990000000000006,"15min":10.577},"last":4.0380000000000003},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.571,"15min":1.657},"min":{"1min":0.55700000000000005,"5min":0.44900000000000001,"15min":0.34300000000000003},"max":{"1min":1.262,"5min":7.5590000000000002,"15min":9.1189999999999998},"last":0.52200000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.639,"15min":1.7310000000000001},"min":{"1min":0.72199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.3069999999999999,"5min":9.1370000000000005,"15min":10.537000000000001},"last":4.0099999999999998},{"interface":"front","average":{"1min":1.032,"5min":1.8859999999999999,"15min":1.833},"min":{"1min":0.52600000000000002,"5min":0.40699999999999997,"15min":0.35099999999999998},"max":{"1min":1.3120000000000001,"5min":9.5090000000000003,"15min":9.5090000000000003},"last":0.49099999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.6499999999999999,"15min":1.6839999999999999},"min":{"1min":0.65300000000000002,"5min":0.52000000000000002,"15min":0.41299999999999998},"max":{"1min":1.3759999999999999,"5min":7.5899999999999999,"15min":10.284000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.04,"5min":1.371,"15min":1.5629999999999999},"min":{"1min":0.64300000000000002,"5min":0.47199999999999998,"15min":0.435},"max":{"1min":1.4019999999999999,"5min":9.1560000000000006,"15min":13.339},"last":0.90000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019348,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2455208,"kb_used_data":1869600,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91289944,"statfs":{"total":95995035648,"available":93480902656,"internally_reserved":0,"allocated":1914470400,"data_stored":1913970693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:07:23 2023","interfaces":[{"interface":"back","average":{"1min":3.2400000000000002,"5min":3.2959999999999998,"15min":2.355},"min":{"1min":0.307,"5min":0.248,"15min":0.248},"max":{"1min":46.463000000000001,"5min":141.07400000000001,"15min":141.07400000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":3.427,"5min":3.1909999999999998,"15min":2.4249999999999998},"min":{"1min":0.38100000000000001,"5min":0.30399999999999999,"15min":0.255},"max":{"1min":46.823999999999998,"5min":141.15600000000001,"15min":141.15600000000001},"last":0.77000000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.73699999999999999,"15min":1.304},"min":{"1min":0.32300000000000001,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":2.403,"5min":2.601,"15min":8.3970000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.89600000000000002,"15min":1.3919999999999999},"min":{"1min":0.39600000000000002,"5min":0.318,"15min":0.29199999999999998},"max":{"1min":3.2189999999999999,"5min":7.6879999999999997,"15min":11.712999999999999},"last":0.39800000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:07:23 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":2.819,"15min":2.1419999999999999},"min":{"1min":0.42699999999999999,"5min":0.34899999999999998,"15min":0.30499999999999999},"max":{"1min":3.6309999999999998,"5min":142.816,"15min":142.816},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.361,"5min":2.7450000000000001,"15min":1.9490000000000001},"min":{"1min":0.52600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.0149999999999997,"5min":143.81800000000001,"15min":143.81800000000001},"last":0.55900000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:07:51 2023","interfaces":[{"interface":"back","average":{"1min":8.125,"5min":2.641,"15min":2.5129999999999999},"min":{"1min":0.36499999999999999,"5min":0.31900000000000001,"15min":0.309},"max":{"1min":64.667000000000002,"5min":64.667000000000002,"15min":143.13399999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":9.2729999999999997,"5min":3.1720000000000002,"15min":2.6629999999999998},"min":{"1min":0.58399999999999996,"5min":0.46300000000000002,"15min":0.36299999999999999},"max":{"1min":65.271000000000001,"5min":65.271000000000001,"15min":152.37299999999999},"last":0.63700000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.0640000000000001,"15min":1.242},"min":{"1min":0.45800000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.6459999999999999,"5min":7.7439999999999998,"15min":11.762},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.95799999999999996,"15min":1.2909999999999999},"min":{"1min":0.40999999999999998,"5min":0.37,"15min":0.28699999999999998},"max":{"1min":1.3680000000000001,"5min":4.1379999999999999,"15min":12.85},"last":0.68999999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:07:45 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.85999999999999999,"15min":1.294},"min":{"1min":0.41499999999999998,"5min":0.36799999999999999,"15min":0.32100000000000001},"max":{"1min":1.3009999999999999,"5min":5.1020000000000003,"15min":10.657999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.0600000000000001,"15min":1.4059999999999999},"min":{"1min":0.47399999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.702,"5min":7.6470000000000002,"15min":14.755000000000001},"last":0.59499999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.0189999999999999,"15min":1.407},"min":{"1min":0.52300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.125,"5min":3.9039999999999999,"15min":13.933999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.97199999999999998,"15min":1.484},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.33200000000000002},"max":{"1min":1.407,"5min":5.1520000000000001,"15min":13.321999999999999},"last":0.67500000000000004}]}]},{"osd":5,"up_from":0,"seq":154618823116,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3151495168,"data_stored":3151495168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2309488640,"data_stored":2309488640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3353018368,"data_stored":3353018368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1768423424,"data_stored":1768423424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3073245184,"data_stored":3073245184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:08:00.262 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T09:08:00.262 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:08:00.263 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T09:08:00.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T09:08:00.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:08:00.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 76353 76337 75.7225 47.9375 0.0222964 0.0131977 2023-10-09T09:08:00.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:00.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:00.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:00.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:00.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:00.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:00.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.863 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:00 smithi016 bash[75179]: cluster 2023-10-09T09:07:59.939767+0000 mgr.y (mgr.25421) 919 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T09:08:00.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:00 smithi016 bash[77892]: cluster 2023-10-09T09:07:59.939767+0000 mgr.y (mgr.25421) 919 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T09:08:00.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:00.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:00.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:00.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:00.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:00 smithi094 bash[59677]: cluster 2023-10-09T09:07:59.939767+0000 mgr.y (mgr.25421) 919 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T09:08:01.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:01.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:01.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:01.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:01.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.397 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 77175 77159 75.342 51.375 0.0109845 0.0132667 2023-10-09T09:08:01.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:01.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:01.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:01.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:01.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:01.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:01.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:01.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:01.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:01.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:02.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:02.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:02.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:02.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:02.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:08:02.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:02.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:02.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:02.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:08:02.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.398 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 78005 77989 74.9801 51.875 0.0212863 0.0133285 2023-10-09T09:08:02.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:02.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:02.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:02.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:02.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:02.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:02.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:02.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:02.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:02.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:02.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:02.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:03.065 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:03.089 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:03.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:03.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:03.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:03.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:03.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:03.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:02 smithi016 bash[75179]: cluster 2023-10-09T09:08:01.940493+0000 mgr.y (mgr.25421) 920 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 865 op/s 2023-10-09T09:08:03.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:02 smithi016 bash[77892]: cluster 2023-10-09T09:08:01.940493+0000 mgr.y (mgr.25421) 920 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 865 op/s 2023-10-09T09:08:03.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:03.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:03.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:03.398 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 78813 78797 74.6091 50.5 0.00909563 0.0133952 2023-10-09T09:08:03.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:03.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:02 smithi094 bash[59677]: cluster 2023-10-09T09:08:01.940493+0000 mgr.y (mgr.25421) 920 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 865 op/s 2023-10-09T09:08:03.493 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:03.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:03.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:03.628 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:03.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:03.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:03.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:03.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:03.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:03.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:03.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:03.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:03.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:03.997 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:04.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:04.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:04.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:04.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:04.260 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:04.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:04.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:04.398 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 79656 79640 74.2819 52.6875 0.018278 0.0134559 2023-10-09T09:08:04.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:04.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:04.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:04.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:04.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:04.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:04.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:04.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:04.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:04.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:04.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:05.073 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:05.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:05.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:05.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:05.194 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:05.195 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:05.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:05.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:04 smithi016 bash[77892]: cluster 2023-10-09T09:08:03.941008+0000 mgr.y (mgr.25421) 921 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T09:08:05.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:04 smithi016 bash[77892]: audit 2023-10-09T09:08:04.238580+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:05.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:04 smithi016 bash[75179]: cluster 2023-10-09T09:08:03.941008+0000 mgr.y (mgr.25421) 921 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T09:08:05.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:04 smithi016 bash[75179]: audit 2023-10-09T09:08:04.238580+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:05.226 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:05.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:05.332 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:05.343 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:04 smithi094 bash[59677]: cluster 2023-10-09T09:08:03.941008+0000 mgr.y (mgr.25421) 921 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T09:08:05.343 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:04 smithi094 bash[59677]: audit 2023-10-09T09:08:04.238580+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:05.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:05.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:05.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:05.398 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 80498 80482 73.9633 52.625 0.0112702 0.0135137 2023-10-09T09:08:05.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:05.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:05.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:05.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:05.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:05.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:05.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:05.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:05.727 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:05.752 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:05.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:05.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:05.881 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:05.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:05.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:05.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:05.997 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:08:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:08:05] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-10-09T09:08:06.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:06.126 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:06.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:06.213 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:06.214 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:06.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:06.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:06.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:06.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:06.400 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 81360 81344 73.6706 53.875 0.0125755 0.0135668 2023-10-09T09:08:06.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:06.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:06.516 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:06.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:06.648 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:06.674 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:06.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:06.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:06.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:06.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:06.912 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:06.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:06.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:07.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:07.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:07.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:07.174 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:07.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:07 smithi094 bash[59677]: cluster 2023-10-09T09:08:05.942655+0000 mgr.y (mgr.25421) 922 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T09:08:07.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:07.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:07.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:07 smithi016 bash[75179]: cluster 2023-10-09T09:08:05.942655+0000 mgr.y (mgr.25421) 922 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T09:08:07.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:07 smithi016 bash[77892]: cluster 2023-10-09T09:08:05.942655+0000 mgr.y (mgr.25421) 922 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T09:08:07.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:07.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:07.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:07.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:07.400 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 82277 82261 73.4368 57.3125 0.0325693 0.0136109 2023-10-09T09:08:07.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:07.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:07.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:07.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:07.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:07.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:07.709 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:07.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:07.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:07.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:07.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:07.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:07.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:08.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:08.110 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:08.136 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:08.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:08.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:08.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:08.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:08.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:08.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:08.400 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 83139 83123 73.1613 53.875 0.0233731 0.0136618 2023-10-09T09:08:08.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:08.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:08 smithi094 bash[59677]: audit 2023-10-09T09:08:07.579904+0000 mon.a (mon.0) 1230 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:08.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:08.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:08 smithi016 bash[75179]: audit 2023-10-09T09:08:07.579904+0000 mon.a (mon.0) 1230 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:08.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:08 smithi016 bash[77892]: audit 2023-10-09T09:08:07.579904+0000 mon.a (mon.0) 1230 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:08.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:08.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:08.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:08.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:08.782 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:08.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:08.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:08.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:08.935 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:08.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:08.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:09.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:09.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:09.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:09.180 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:09 smithi094 bash[59677]: cluster 2023-10-09T09:08:07.943337+0000 mgr.y (mgr.25421) 923 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T09:08:09.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:09.281 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:09 smithi016 bash[75179]: cluster 2023-10-09T09:08:07.943337+0000 mgr.y (mgr.25421) 923 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T09:08:09.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:09 smithi016 bash[77892]: cluster 2023-10-09T09:08:07.943337+0000 mgr.y (mgr.25421) 923 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T09:08:09.302 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:09.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:09.400 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 84020 84004 72.9098 55.0625 0.0187587 0.0137095 2023-10-09T09:08:09.430 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:09.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:09.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:09.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:09.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:09.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:09.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:09.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:09.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:09.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:09.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:09.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:09.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:09.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:10.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:10.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:10.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:10.221 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:10.246 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:10.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:10.349 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:10.373 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:10.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:10.400 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 84922 84906 72.6833 56.375 0.0107099 0.0137518 2023-10-09T09:08:10.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:10.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:10.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:10.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:10.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:10.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:10.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:10 smithi094 bash[59677]: cluster 2023-10-09T09:08:09.943818+0000 mgr.y (mgr.25421) 924 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T09:08:10.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:10.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:10.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:10.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:10.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:10.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:10.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:10.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:10.910 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:10 smithi016 bash[75179]: cluster 2023-10-09T09:08:09.943818+0000 mgr.y (mgr.25421) 924 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T09:08:10.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:10 smithi016 bash[77892]: cluster 2023-10-09T09:08:09.943818+0000 mgr.y (mgr.25421) 924 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T09:08:11.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:11.030 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:11.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:11.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:11.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:11.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:11.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 85720 85704 72.375 49.875 0.0142964 0.013811 2023-10-09T09:08:11.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:11.430 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:11.532 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:11.557 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:11.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:11.660 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:11.687 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:11.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:11.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:11.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:11.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:11.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:11.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:11.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:12.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:12.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:12.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:12.216 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:12.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:12.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:12.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:12.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 86597 86581 72.1408 54.8125 0.00815298 0.0138562 2023-10-09T09:08:12.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:12.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:12.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:12.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:12.602 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:12.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:12.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:12.838 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:12.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:12.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:12.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:12.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:13.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:13.098 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:13 smithi094 bash[59677]: cluster 2023-10-09T09:08:11.944479+0000 mgr.y (mgr.25421) 925 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T09:08:13.115 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:13.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:13.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:13.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:13 smithi016 bash[75179]: cluster 2023-10-09T09:08:11.944479+0000 mgr.y (mgr.25421) 925 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T09:08:13.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:13 smithi016 bash[77892]: cluster 2023-10-09T09:08:11.944479+0000 mgr.y (mgr.25421) 925 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T09:08:13.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:13.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:13.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:13.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:13.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:13.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:13.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 87366 87350 71.8239 48.0625 0.0122803 0.0139155 2023-10-09T09:08:13.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:13.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:13.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:13.626 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:13.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:13.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:13.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:13.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:13.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:13.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:13.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:13.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:13.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:14.039 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:14.063 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:14.166 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:14.193 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:14.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:14.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:14.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:14.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:14.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 88138 88122 71.5177 48.25 0.0194312 0.0139765 2023-10-09T09:08:14.434 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:14.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:14.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:14.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:14.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:14.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:14.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:14.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:14.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:14.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:14.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:14.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:14.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:14.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:15.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:15.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:14 smithi016 bash[75179]: audit 2023-10-09T09:08:13.932347+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:15.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:14 smithi016 bash[75179]: audit 2023-10-09T09:08:13.938088+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:15.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:14 smithi016 bash[75179]: cluster 2023-10-09T09:08:13.945192+0000 mgr.y (mgr.25421) 926 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T09:08:15.096 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:14 smithi016 bash[77892]: audit 2023-10-09T09:08:13.932347+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:15.096 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:14 smithi016 bash[77892]: audit 2023-10-09T09:08:13.938088+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:15.096 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:14 smithi016 bash[77892]: cluster 2023-10-09T09:08:13.945192+0000 mgr.y (mgr.25421) 926 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T09:08:15.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:15.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:15.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:15.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:15.228 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:14 smithi094 bash[59677]: audit 2023-10-09T09:08:13.932347+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:15.229 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:14 smithi094 bash[59677]: audit 2023-10-09T09:08:13.938088+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:15.229 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:14 smithi094 bash[59677]: cluster 2023-10-09T09:08:13.945192+0000 mgr.y (mgr.25421) 926 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T09:08:15.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:15.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:15.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:15.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:15.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:15.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:15.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 15 88889 88874 71.2033 47 0.00472307 0.0140385 2023-10-09T09:08:15.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:15.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:15.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:15.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:15.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:15.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:15.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:15.738 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:15.765 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:15.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:15.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:15.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:15.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:15.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:15.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:15.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:08:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:08:15] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-10-09T09:08:16.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:16.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:16.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:16.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:16.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:16.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:16.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:16.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 89675 89659 70.923 49.0625 0.0154098 0.0140939 2023-10-09T09:08:16.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:16.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:16.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:16.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:16.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:16.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:16.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:16.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:16.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:16.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:16.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:16.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:16.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:17.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:17.078 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:17 smithi016 bash[75179]: cluster 2023-10-09T09:08:15.945705+0000 mgr.y (mgr.25421) 927 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T09:08:17.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:17 smithi016 bash[77892]: cluster 2023-10-09T09:08:15.945705+0000 mgr.y (mgr.25421) 927 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T09:08:17.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:17.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:17.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:17.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:17.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:17.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:17.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:17.357 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:17 smithi094 bash[59677]: cluster 2023-10-09T09:08:15.945705+0000 mgr.y (mgr.25421) 927 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T09:08:17.371 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:17.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:17.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:08:17.401018+0000 min lat: 0.00143057 max lat: 0.340643 avg lat: 0.0141369 2023-10-09T09:08:17.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:08:17.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 90530 90514 70.7044 53.4375 0.018487 0.0141369 2023-10-09T09:08:17.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:17.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:17.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:17.608 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:17.639 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:17.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:17.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:17.742 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:17.768 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:17.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:17.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:17.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:17.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:18.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:18.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:18.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:18.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:18.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:18.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:18.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:18.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:18.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 91380 91364 70.4873 53.125 0.0247046 0.0141806 2023-10-09T09:08:18.405 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:18.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:18.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:18.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:18.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:18.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:18.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:18.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:18.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:18.795 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:18.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:18.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:18.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:18.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:19.059 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:19.067 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:19 smithi094 bash[59677]: cluster 2023-10-09T09:08:17.946405+0000 mgr.y (mgr.25421) 928 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T09:08:19.088 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:19.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:19.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:19.194 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:19.219 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:19.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:19.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:19.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:19 smithi016 bash[75179]: cluster 2023-10-09T09:08:17.946405+0000 mgr.y (mgr.25421) 928 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T09:08:19.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:19 smithi016 bash[77892]: cluster 2023-10-09T09:08:17.946405+0000 mgr.y (mgr.25421) 928 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T09:08:19.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:19.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:19.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:19.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 92256 92240 70.2953 54.75 0.0102073 0.0142202 2023-10-09T09:08:19.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:19.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:19.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:19.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:19.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:19.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:19.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:19.728 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:19.760 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:19.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:19.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:19.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:19.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:19.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:19.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:19.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:20.009 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:20 smithi094 bash[59677]: audit 2023-10-09T09:08:19.238644+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:20.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:20.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:20 smithi016 bash[75179]: audit 2023-10-09T09:08:19.238644+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:20.019 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:20 smithi016 bash[77892]: audit 2023-10-09T09:08:19.238644+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:20.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:20.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:20.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:20.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:20.254 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:20.280 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:20.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:20.401 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 93107 93091 70.0892 53.1875 0.00952889 0.014261 2023-10-09T09:08:20.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:20.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:20.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:20.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:20.671 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:20.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:20.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:20.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:20.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:20.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:20.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:20.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:21.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:21.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:21.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:21.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:21.210 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:21 smithi016 bash[75179]: cluster 2023-10-09T09:08:19.946895+0000 mgr.y (mgr.25421) 929 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T09:08:21.211 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:21 smithi016 bash[77892]: cluster 2023-10-09T09:08:19.946895+0000 mgr.y (mgr.25421) 929 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T09:08:21.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:21 smithi094 bash[59677]: cluster 2023-10-09T09:08:19.946895+0000 mgr.y (mgr.25421) 929 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T09:08:21.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:21.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:21.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:21.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:21.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:21.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:21.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 93818 93802 69.7838 44.4375 0.00832586 0.0143197 2023-10-09T09:08:21.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:21.515 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:21.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:21.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:21.648 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:21.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:21.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:21.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:21.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:21.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:21.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:21.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:21.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:22.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:22.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:22.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:22.163 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:22.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:22.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:22.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 94542 94526 69.4951 45.25 0.00600052 0.0143837 2023-10-09T09:08:22.405 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:22.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:22.531 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:22.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:22.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:22.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:22.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:22.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:22.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:22.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:22.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:22.960 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:22 smithi016 bash[75179]: audit 2023-10-09T09:08:21.897825+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:22.961 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:22 smithi016 bash[75179]: audit 2023-10-09T09:08:21.913508+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:22.961 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:22 smithi016 bash[75179]: cluster 2023-10-09T09:08:21.947555+0000 mgr.y (mgr.25421) 930 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:08:22.961 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:22 smithi016 bash[77892]: audit 2023-10-09T09:08:21.897825+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:22.961 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:22 smithi016 bash[77892]: audit 2023-10-09T09:08:21.913508+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:22.961 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:22 smithi016 bash[77892]: cluster 2023-10-09T09:08:21.947555+0000 mgr.y (mgr.25421) 930 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:08:22.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:22.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:23.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:23.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:23.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:23.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:23.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:22 smithi094 bash[59677]: audit 2023-10-09T09:08:21.897825+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:23.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:22 smithi094 bash[59677]: audit 2023-10-09T09:08:21.913508+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:23.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:22 smithi094 bash[59677]: cluster 2023-10-09T09:08:21.947555+0000 mgr.y (mgr.25421) 930 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:08:23.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:23.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:23.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:23.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:23.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:23.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:23.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 95347 95331 69.272 50.3125 0.0227311 0.0144298 2023-10-09T09:08:23.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:23.527 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:23.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:23.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:23.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:23.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:23.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:23.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:23.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:23.898 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:23.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[75179]: debug 2023-10-09T09:08:23.851+0000 7fa046a4d700 -1 mon.a@0(leader).osd e220 definitely_dead 0 2023-10-09T09:08:23.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:23.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:23.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:23.976 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.5. 2023-10-09T09:08:24.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:24.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:24.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:24.178 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[75179]: audit 2023-10-09T09:08:22.956459+0000 mon.a (mon.0) 1236 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:24.178 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[75179]: audit 2023-10-09T09:08:22.958970+0000 mon.a (mon.0) 1237 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:24.178 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[75179]: audit 2023-10-09T09:08:22.970829+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:24.178 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[75179]: audit 2023-10-09T09:08:23.848153+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.16:0/3014706354' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:08:24.178 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[75179]: audit 2023-10-09T09:08:23.851877+0000 mon.a (mon.0) 1239 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:08:24.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[77892]: audit 2023-10-09T09:08:22.956459+0000 mon.a (mon.0) 1236 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:24.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[77892]: audit 2023-10-09T09:08:22.958970+0000 mon.a (mon.0) 1237 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:24.179 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[77892]: audit 2023-10-09T09:08:22.970829+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:24.179 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[77892]: audit 2023-10-09T09:08:23.848153+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.16:0/3014706354' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:08:24.179 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:23 smithi016 bash[77892]: audit 2023-10-09T09:08:23.851877+0000 mon.a (mon.0) 1239 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:08:24.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:24.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:23 smithi094 bash[59677]: audit 2023-10-09T09:08:22.956459+0000 mon.a (mon.0) 1236 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:24.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:23 smithi094 bash[59677]: audit 2023-10-09T09:08:22.958970+0000 mon.a (mon.0) 1237 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:24.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:23 smithi094 bash[59677]: audit 2023-10-09T09:08:22.970829+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:24.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:23 smithi094 bash[59677]: audit 2023-10-09T09:08:23.848153+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.16:0/3014706354' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:08:24.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:23 smithi094 bash[59677]: audit 2023-10-09T09:08:23.851877+0000 mon.a (mon.0) 1239 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:08:24.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:24.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:24.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:24.402 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:08:24.402 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:08:24.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 95802 95786 68.8026 28.4375 0.0393709 0.0144571 2023-10-09T09:08:24.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:24.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:24.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:24.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:24.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:24.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:24.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:24.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:24.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:24.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:24.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:24.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:24.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:25.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:25.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:23.948238+0000 mgr.y (mgr.25421) 931 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T09:08:25.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:23.965619+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: audit 2023-10-09T09:08:23.969015+0000 mon.a (mon.0) 1240 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T09:08:25.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:23.969664+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e221: 8 total, 8 up, 6 in 2023-10-09T09:08:25.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:23.971616+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:23.973831+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.626487+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.091 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.627892+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e222: 8 total, 8 up, 6 in 2023-10-09T09:08:25.091 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.628124+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.091 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.631602+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.091 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.633509+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.091 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:24 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.654042+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.104 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:25.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:25.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:25.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:25.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:25.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:23.948238+0000 mgr.y (mgr.25421) 931 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T09:08:25.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:23.965619+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: audit 2023-10-09T09:08:23.969015+0000 mon.a (mon.0) 1240 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T09:08:25.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:23.969664+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e221: 8 total, 8 up, 6 in 2023-10-09T09:08:25.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:23.971616+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:23.973831+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.626487+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.627892+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e222: 8 total, 8 up, 6 in 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.628124+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.631602+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.633509+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.654042+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:23.948238+0000 mgr.y (mgr.25421) 931 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:23.965619+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: audit 2023-10-09T09:08:23.969015+0000 mon.a (mon.0) 1240 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T09:08:25.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:23.969664+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e221: 8 total, 8 up, 6 in 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:23.971616+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:23.973831+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.626487+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.627892+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e222: 8 total, 8 up, 6 in 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.628124+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.631602+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.633509+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.654042+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:25.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:25.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:25.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:25.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:25.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 95802 95786 68.0207 0 - 0.0144571 2023-10-09T09:08:25.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:25.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:25.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:25.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:25.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:25.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:25.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:25.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:25.748 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:25.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:08:25] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.33.4" 2023-10-09T09:08:25.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:25.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:25.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:25.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:25.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:25.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:25.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.627300+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.632111+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:24.633820+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.625924+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.629418+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e223: 8 total, 8 up, 6 in 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.630036+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.632640+0000 osd.0 (osd.0) 10 : cluster 0 9.5 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5889 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.634444+0000 osd.0 (osd.0) 11 : cluster 0 9.9 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5965 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.634621+0000 osd.0 (osd.0) 12 : cluster 0 9.b starting backfill to osd.5 from (0'0,0'0] MAX to 220'6133 2023-10-09T09:08:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.636222+0000 osd.2 (osd.2) 2 : cluster 0 9.a starting backfill to osd.5 from (0'0,0'0] MAX to 220'5995 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.637620+0000 osd.2 (osd.2) 3 : cluster 0 9.f starting backfill to osd.5 from (0'0,0'0] MAX to 220'6058 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.653237+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.655061+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.656194+0000 osd.2 (osd.2) 4 : cluster 0 9.7 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5869 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.627300+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.632111+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:24.633820+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.625924+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.629418+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e223: 8 total, 8 up, 6 in 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.630036+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.632640+0000 osd.0 (osd.0) 10 : cluster 0 9.5 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5889 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.634444+0000 osd.0 (osd.0) 11 : cluster 0 9.9 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5965 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.634621+0000 osd.0 (osd.0) 12 : cluster 0 9.b starting backfill to osd.5 from (0'0,0'0] MAX to 220'6133 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.636222+0000 osd.2 (osd.2) 2 : cluster 0 9.a starting backfill to osd.5 from (0'0,0'0] MAX to 220'5995 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.637620+0000 osd.2 (osd.2) 3 : cluster 0 9.f starting backfill to osd.5 from (0'0,0'0] MAX to 220'6058 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.653237+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.655061+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:25 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.656194+0000 osd.2 (osd.2) 4 : cluster 0 9.7 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5869 2023-10-09T09:08:26.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:26.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:26.155 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:26.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.627300+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.632111+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:24.633820+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.625924+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.629418+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e223: 8 total, 8 up, 6 in 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.630036+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.632640+0000 osd.0 (osd.0) 10 : cluster 0 9.5 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5889 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.634444+0000 osd.0 (osd.0) 11 : cluster 0 9.9 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5965 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.634621+0000 osd.0 (osd.0) 12 : cluster 0 9.b starting backfill to osd.5 from (0'0,0'0] MAX to 220'6133 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.636222+0000 osd.2 (osd.2) 2 : cluster 0 9.a starting backfill to osd.5 from (0'0,0'0] MAX to 220'5995 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.637620+0000 osd.2 (osd.2) 3 : cluster 0 9.f starting backfill to osd.5 from (0'0,0'0] MAX to 220'6058 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.653237+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.655061+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:26.163 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:25 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.656194+0000 osd.2 (osd.2) 4 : cluster 0 9.7 starting backfill to osd.5 from (0'0,0'0] MAX to 220'5869 2023-10-09T09:08:26.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:26.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:26.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 95904 95888 67.3281 3.1875 0.0648392 0.0148411 2023-10-09T09:08:26.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.442 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:26.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:26.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:26.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.703 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:26.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:26.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:26.934 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:26.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:26.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:27.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:27.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:27.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:27.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:27.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:27.202 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.628612+0000 osd.3 (osd.3) 4 : cluster 0 9.8 starting backfill to osd.5 from (0'0,0'0] MAX to 220'6096 2023-10-09T09:08:27.202 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.634785+0000 osd.3 (osd.3) 5 : cluster 0 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 220'6091 2023-10-09T09:08:27.202 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.948953+0000 mgr.y (mgr.25421) 932 : cluster 0 pgmap v980: 89 pgs: 1 remapped+peering, 88 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T09:08:27.202 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[75179]: cluster 2023-10-09T09:08:25.950785+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:27.203 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.628612+0000 osd.3 (osd.3) 4 : cluster 0 9.8 starting backfill to osd.5 from (0'0,0'0] MAX to 220'6096 2023-10-09T09:08:27.203 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.634785+0000 osd.3 (osd.3) 5 : cluster 0 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 220'6091 2023-10-09T09:08:27.203 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.948953+0000 mgr.y (mgr.25421) 932 : cluster 0 pgmap v980: 89 pgs: 1 remapped+peering, 88 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T09:08:27.203 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:26 smithi016 bash[77892]: cluster 2023-10-09T09:08:25.950785+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:27.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:27.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:27.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:26 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.628612+0000 osd.3 (osd.3) 4 : cluster 0 9.8 starting backfill to osd.5 from (0'0,0'0] MAX to 220'6096 2023-10-09T09:08:27.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:26 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.634785+0000 osd.3 (osd.3) 5 : cluster 0 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 220'6091 2023-10-09T09:08:27.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:26 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.948953+0000 mgr.y (mgr.25421) 932 : cluster 0 pgmap v980: 89 pgs: 1 remapped+peering, 88 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T09:08:27.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:26 smithi094 bash[59677]: cluster 2023-10-09T09:08:25.950785+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T09:08:27.340 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:27.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:27.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:27.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:27.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 5 96583 96578 67.0591 43.125 0.0444811 0.0149056 2023-10-09T09:08:27.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:27.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:27.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:27.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:27.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:27.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:27.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:27.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:27.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:27.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:27.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:27.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:27.860 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T09:08:27.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:27.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:27.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:28.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:28.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:28.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:28.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:28.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:28.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:28.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:28.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:28.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:28.402 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0141 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 96583 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 67.061 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 24.9997 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 155.812 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 0 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1072 2023-10-09T09:08:28.403 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 399.996 2023-10-09T09:08:28.404 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2493 2023-10-09T09:08:28.404 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 0 2023-10-09T09:08:28.404 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0149065 2023-10-09T09:08:28.404 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.031367 2023-10-09T09:08:28.404 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 2.36215 2023-10-09T09:08:28.404 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00143057 2023-10-09T09:08:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:28.439 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:08:28.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:28.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:28.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:28.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:28.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:28.798 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:28.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:28.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:28.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:28.975 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:28 smithi094 bash[59677]: cluster 2023-10-09T09:08:27.950047+0000 mgr.y (mgr.25421) 933 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 1 remapped+peering, 81 active+clean; 5.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 28 MiB/s wr, 450 op/s; 42022/287031 objects misplaced (14.640%); 619 KiB/s, 10 objects/s recovering 2023-10-09T09:08:29.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:29.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:29.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:29.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:29.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:29.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:28 smithi016 bash[77892]: cluster 2023-10-09T09:08:27.950047+0000 mgr.y (mgr.25421) 933 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 1 remapped+peering, 81 active+clean; 5.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 28 MiB/s wr, 450 op/s; 42022/287031 objects misplaced (14.640%); 619 KiB/s, 10 objects/s recovering 2023-10-09T09:08:29.171 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:28 smithi016 bash[75179]: cluster 2023-10-09T09:08:27.950047+0000 mgr.y (mgr.25421) 933 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 1 remapped+peering, 81 active+clean; 5.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 28 MiB/s wr, 450 op/s; 42022/287031 objects misplaced (14.640%); 619 KiB/s, 10 objects/s recovering 2023-10-09T09:08:29.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:29.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:29.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:29.268 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:08:29.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:29.308 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:08:29.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:08:29.324 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:29.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:29.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:29.383 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:08:29.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:29.428 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:08:29.472 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:08:29.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:29.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:29.512 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:08:29.561 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:08:29.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:29.611 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:29.612 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:08:29.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:29.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:29.621 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:08:29.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:29.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:29.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:29.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:29.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:29.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:29.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:29.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:29 smithi016 bash[75179]: audit 2023-10-09T09:08:29.218578+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.16:0/3331765299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:08:29.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:30.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:30.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:30.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:30.157 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:29 smithi016 bash[77892]: audit 2023-10-09T09:08:29.218578+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.16:0/3331765299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:08:30.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:30.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:29 smithi094 bash[59677]: audit 2023-10-09T09:08:29.218578+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.16:0/3331765299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:08:30.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:30.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:30.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:30.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:30.444 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:30.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:30.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:30.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:30.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:30.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:30.814 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:30.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:30.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:30.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:30.980 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:30 smithi016 bash[75179]: cluster 2023-10-09T09:08:29.950794+0000 mgr.y (mgr.25421) 934 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 8 active+remapped+backfill_wait, 1 remapped+peering, 78 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 24 MiB/s wr, 381 op/s; 60038/288699 objects misplaced (20.796%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T09:08:30.981 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:30 smithi016 bash[75179]: audit 2023-10-09T09:08:29.996345+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.16:0/3612124800' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:08:30.981 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:30 smithi016 bash[75179]: audit 2023-10-09T09:08:29.997394+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:08:30.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:30.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:31.011 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:08:31.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:31.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:30 smithi094 bash[59677]: cluster 2023-10-09T09:08:29.950794+0000 mgr.y (mgr.25421) 934 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 8 active+remapped+backfill_wait, 1 remapped+peering, 78 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 24 MiB/s wr, 381 op/s; 60038/288699 objects misplaced (20.796%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T09:08:31.100 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:30 smithi094 bash[59677]: audit 2023-10-09T09:08:29.996345+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.16:0/3612124800' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:08:31.101 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:30 smithi094 bash[59677]: audit 2023-10-09T09:08:29.997394+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:08:31.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:31.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:31.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:31.233 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:30 smithi016 bash[77892]: cluster 2023-10-09T09:08:29.950794+0000 mgr.y (mgr.25421) 934 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 8 active+remapped+backfill_wait, 1 remapped+peering, 78 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 24 MiB/s wr, 381 op/s; 60038/288699 objects misplaced (20.796%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T09:08:31.233 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:30 smithi016 bash[77892]: audit 2023-10-09T09:08:29.996345+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.16:0/3612124800' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:08:31.233 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:30 smithi016 bash[77892]: audit 2023-10-09T09:08:29.997394+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:08:31.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:31.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:31.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:31.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:31.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:31.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:31.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:31.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:31.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:31.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:31.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:31.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:31.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:31.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:31.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:31.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:31.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:31.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:31.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:31.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:32.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:32.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:32.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:32.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[75179]: cluster 2023-10-09T09:08:30.975613+0000 mon.a (mon.0) 1245 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:08:32.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[75179]: cluster 2023-10-09T09:08:30.996964+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[75179]: audit 2023-10-09T09:08:31.002697+0000 mon.a (mon.0) 1246 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:08:32.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[75179]: cluster 2023-10-09T09:08:31.002792+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e224: 8 total, 8 up, 6 in 2023-10-09T09:08:32.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[75179]: cluster 2023-10-09T09:08:31.004882+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[75179]: cluster 2023-10-09T09:08:31.004944+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[75179]: cluster 2023-10-09T09:08:31.151992+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[77892]: cluster 2023-10-09T09:08:30.975613+0000 mon.a (mon.0) 1245 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:08:32.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[77892]: cluster 2023-10-09T09:08:30.996964+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[77892]: audit 2023-10-09T09:08:31.002697+0000 mon.a (mon.0) 1246 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:08:32.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[77892]: cluster 2023-10-09T09:08:31.002792+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e224: 8 total, 8 up, 6 in 2023-10-09T09:08:32.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[77892]: cluster 2023-10-09T09:08:31.004882+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[77892]: cluster 2023-10-09T09:08:31.004944+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:31 smithi016 bash[77892]: cluster 2023-10-09T09:08:31.151992+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:32.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:32.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:31 smithi094 bash[59677]: cluster 2023-10-09T09:08:30.975613+0000 mon.a (mon.0) 1245 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:08:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:31 smithi094 bash[59677]: cluster 2023-10-09T09:08:30.996964+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:31 smithi094 bash[59677]: audit 2023-10-09T09:08:31.002697+0000 mon.a (mon.0) 1246 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:08:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:31 smithi094 bash[59677]: cluster 2023-10-09T09:08:31.002792+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e224: 8 total, 8 up, 6 in 2023-10-09T09:08:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:31 smithi094 bash[59677]: cluster 2023-10-09T09:08:31.004882+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:31 smithi094 bash[59677]: cluster 2023-10-09T09:08:31.004944+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:31 smithi094 bash[59677]: cluster 2023-10-09T09:08:31.151992+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T09:08:32.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:32.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:32.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:32.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:32.457 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:32.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:32.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:32.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:32.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:32.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:32.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:32.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:32.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:32.861 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5] out_osds: [7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:08:32.861 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:08:32.861 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T09:08:32.861 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-10-09T09:08:32.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-10-09T09:08:32.875 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:32.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:32.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:32.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:32.985 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:32 smithi016 bash[75179]: cluster 2023-10-09T09:08:31.951468+0000 mgr.y (mgr.25421) 935 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 23 MiB/s wr, 483 op/s; 6003/287196 objects degraded (2.090%); 59865/287196 objects misplaced (20.845%); 926 KiB/s, 15 objects/s recovering 2023-10-09T09:08:32.985 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:32 smithi016 bash[75179]: cluster 2023-10-09T09:08:31.990417+0000 mon.a (mon.0) 1248 : cluster 3 Health check failed: Degraded data redundancy: 6003/287196 objects degraded (2.090%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:33.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:33.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:33.129 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:32 smithi094 bash[59677]: cluster 2023-10-09T09:08:31.951468+0000 mgr.y (mgr.25421) 935 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 23 MiB/s wr, 483 op/s; 6003/287196 objects degraded (2.090%); 59865/287196 objects misplaced (20.845%); 926 KiB/s, 15 objects/s recovering 2023-10-09T09:08:33.129 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:32 smithi094 bash[59677]: cluster 2023-10-09T09:08:31.990417+0000 mon.a (mon.0) 1248 : cluster 3 Health check failed: Degraded data redundancy: 6003/287196 objects degraded (2.090%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:33.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:33.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:33.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:32 smithi016 bash[77892]: cluster 2023-10-09T09:08:31.951468+0000 mgr.y (mgr.25421) 935 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 23 MiB/s wr, 483 op/s; 6003/287196 objects degraded (2.090%); 59865/287196 objects misplaced (20.845%); 926 KiB/s, 15 objects/s recovering 2023-10-09T09:08:33.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:32 smithi016 bash[77892]: cluster 2023-10-09T09:08:31.990417+0000 mon.a (mon.0) 1248 : cluster 3 Health check failed: Degraded data redundancy: 6003/287196 objects degraded (2.090%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:33.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:33.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:33.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:33.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:33.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:33.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:33.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:33.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:33.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:33.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:33.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:33.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:33.832 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:33.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:33.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:33.970 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:33.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:33.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:34.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:34.101 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:34.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:34.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:08:34.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:34.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:34.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:34.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:34.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:34.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:34.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:34.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:34.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:34.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:34.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:34.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:34.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:34.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:34.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:34.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:34.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:34.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:34.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:34.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:34.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:34.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:35.068 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:35.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:35.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:35.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:08:35.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:35.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:35.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:35.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:35.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:35.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:35.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:35.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:35 smithi016 bash[75179]: cluster 2023-10-09T09:08:33.952143+0000 mgr.y (mgr.25421) 936 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 122 B/s rd, 18 MiB/s wr, 567 op/s; 6003/282693 objects degraded (2.124%); 58580/282693 objects misplaced (20.722%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:08:35.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:35 smithi016 bash[75179]: audit 2023-10-09T09:08:34.244444+0000 mon.a (mon.0) 1249 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:35.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:35 smithi016 bash[75179]: audit 2023-10-09T09:08:34.248572+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:35.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:35 smithi016 bash[77892]: cluster 2023-10-09T09:08:33.952143+0000 mgr.y (mgr.25421) 936 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 122 B/s rd, 18 MiB/s wr, 567 op/s; 6003/282693 objects degraded (2.124%); 58580/282693 objects misplaced (20.722%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:08:35.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:35 smithi016 bash[77892]: audit 2023-10-09T09:08:34.244444+0000 mon.a (mon.0) 1249 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:35.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:35 smithi016 bash[77892]: audit 2023-10-09T09:08:34.248572+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:35.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:35.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:35.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:35.614 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:35.624 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:35 smithi094 bash[59677]: cluster 2023-10-09T09:08:33.952143+0000 mgr.y (mgr.25421) 936 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 122 B/s rd, 18 MiB/s wr, 567 op/s; 6003/282693 objects degraded (2.124%); 58580/282693 objects misplaced (20.722%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:08:35.624 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:35 smithi094 bash[59677]: audit 2023-10-09T09:08:34.244444+0000 mon.a (mon.0) 1249 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:35.624 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:35 smithi094 bash[59677]: audit 2023-10-09T09:08:34.248572+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:35.639 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:35.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:35.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:35.702 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:35.740 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:08:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:08:35] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.33.4" 2023-10-09T09:08:35.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:35.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:35.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:35.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:35.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:35.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:36.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:36.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:36.145 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:36.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:36.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:36.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:36.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:36.416 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:36.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:36.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:36.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:36.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:36.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:36.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:36.720 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:36.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:36.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:36.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:36.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:36.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:36.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:36.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:37.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:37.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:37.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:37.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:37.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:37.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:37.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:37.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:37.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:37.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:37.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:37.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:37 smithi016 bash[75179]: cluster 2023-10-09T09:08:35.952789+0000 mgr.y (mgr.25421) 937 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 15 MiB/s wr, 516 op/s; 6003/281376 objects degraded (2.133%); 58580/281376 objects misplaced (20.819%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T09:08:37.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:37 smithi016 bash[77892]: cluster 2023-10-09T09:08:35.952789+0000 mgr.y (mgr.25421) 937 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 15 MiB/s wr, 516 op/s; 6003/281376 objects degraded (2.133%); 58580/281376 objects misplaced (20.819%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T09:08:37.511 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:37.515 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:37 smithi094 bash[59677]: cluster 2023-10-09T09:08:35.952789+0000 mgr.y (mgr.25421) 937 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 15 MiB/s wr, 516 op/s; 6003/281376 objects degraded (2.133%); 58580/281376 objects misplaced (20.819%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T09:08:37.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:37.616 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:37.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:37.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:37.754 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:37.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:37.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:37.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:37.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:37.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:38.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:38.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:38.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:38.154 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:38.178 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:38.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:38.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:38 smithi094 bash[59677]: cluster 2023-10-09T09:08:38.251545+0000 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 6003/274308 objects degraded (2.188%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:38.542 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:38.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:38.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:38.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:38.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:38.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:38 smithi016 bash[75179]: cluster 2023-10-09T09:08:38.251545+0000 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 6003/274308 objects degraded (2.188%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:38.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:38 smithi016 bash[77892]: cluster 2023-10-09T09:08:38.251545+0000 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 6003/274308 objects degraded (2.188%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:38.757 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:38.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:38.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:38.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:38.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:39.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:39.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:39.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:39.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:39.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:39.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:39.359 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:39.368 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:39 smithi094 bash[59677]: cluster 2023-10-09T09:08:37.953492+0000 mgr.y (mgr.25421) 938 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 610 op/s; 6003/274308 objects degraded (2.188%); 57395/274308 objects misplaced (20.924%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:08:39.394 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:39.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:39.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:39.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:39 smithi016 bash[75179]: cluster 2023-10-09T09:08:37.953492+0000 mgr.y (mgr.25421) 938 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 610 op/s; 6003/274308 objects degraded (2.188%); 57395/274308 objects misplaced (20.924%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:08:39.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:39 smithi016 bash[77892]: cluster 2023-10-09T09:08:37.953492+0000 mgr.y (mgr.25421) 938 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 610 op/s; 6003/274308 objects degraded (2.188%); 57395/274308 objects misplaced (20.924%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:08:39.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:39.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:39.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:39.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:39.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:39.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:39.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:39.778 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:08:39.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:39.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:39.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:39.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:39.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:39.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:40.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:40.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:40.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:40.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:40.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:40.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:40.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:40.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:40.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:40.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:40.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:40.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:40.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:40.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:40.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:40.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:40.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:40.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:40.728 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:40.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:40.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:40.859 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:40.891 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:40.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:40.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:41.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:41.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:41.142 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:40 smithi094 bash[59677]: audit 2023-10-09T09:08:39.952335+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:41.142 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:40 smithi094 bash[59677]: cluster 2023-10-09T09:08:39.954026+0000 mgr.y (mgr.25421) 939 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 2.3 MiB/s wr, 619 op/s; 6003/272358 objects degraded (2.204%); 56745/272358 objects misplaced (20.835%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:08:41.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:41.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:40 smithi016 bash[75179]: audit 2023-10-09T09:08:39.952335+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:41.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:40 smithi016 bash[75179]: cluster 2023-10-09T09:08:39.954026+0000 mgr.y (mgr.25421) 939 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 2.3 MiB/s wr, 619 op/s; 6003/272358 objects degraded (2.204%); 56745/272358 objects misplaced (20.835%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:08:41.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:40 smithi016 bash[77892]: audit 2023-10-09T09:08:39.952335+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:08:41.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:40 smithi016 bash[77892]: cluster 2023-10-09T09:08:39.954026+0000 mgr.y (mgr.25421) 939 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 2.3 MiB/s wr, 619 op/s; 6003/272358 objects degraded (2.204%); 56745/272358 objects misplaced (20.835%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:08:41.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:41.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:41.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:41.398 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:41.434 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:41.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:41.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:41.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:41.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:41.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:41.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:41.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:41.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:41.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:41.944 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:41.972 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:41.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:42.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:42.101 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:42.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:42.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:42.239 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:42.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:42.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:42.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:42.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:42.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:42.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:42.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:42.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:42.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:42.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:42.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:42.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:42.770 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:42.798 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:42.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:42.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:42.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:42.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:42.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:43.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:43.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:43.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:43.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:43.181 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:43 smithi094 bash[59677]: cluster 2023-10-09T09:08:41.954798+0000 mgr.y (mgr.25421) 940 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 93 B/s rd, 1.5 MiB/s wr, 705 op/s; 6003/266949 objects degraded (2.249%); 56247/266949 objects misplaced (21.070%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:08:43.195 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:43.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:43.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:43 smithi016 bash[75179]: cluster 2023-10-09T09:08:41.954798+0000 mgr.y (mgr.25421) 940 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 93 B/s rd, 1.5 MiB/s wr, 705 op/s; 6003/266949 objects degraded (2.249%); 56247/266949 objects misplaced (21.070%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:08:43.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:43 smithi016 bash[77892]: cluster 2023-10-09T09:08:41.954798+0000 mgr.y (mgr.25421) 940 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 93 B/s rd, 1.5 MiB/s wr, 705 op/s; 6003/266949 objects degraded (2.249%); 56247/266949 objects misplaced (21.070%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:08:43.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:43.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:43.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:43.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:43.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:43.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:43.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:43.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:43.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:43.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:43.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:43.759 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:43.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:43.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:43.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:43.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:43.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:43.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:44.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:44.123 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:44.163 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:44.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:44.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:44 smithi016 bash[75179]: cluster 2023-10-09T09:08:44.012163+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: Degraded data redundancy: 6003/261765 objects degraded (2.293%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:44.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:44 smithi016 bash[77892]: cluster 2023-10-09T09:08:44.012163+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: Degraded data redundancy: 6003/261765 objects degraded (2.293%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:44.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:44.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:44.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:44.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:44.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:44.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:44.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:44 smithi094 bash[59677]: cluster 2023-10-09T09:08:44.012163+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: Degraded data redundancy: 6003/261765 objects degraded (2.293%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:44.532 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:44.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:44.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:44.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:44.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:44.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:44.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:44.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:44.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:44.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:44.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:45.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:45.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:45.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:45.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:45.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:45.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:45.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:45.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:45.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:45 smithi016 bash[75179]: cluster 2023-10-09T09:08:43.955295+0000 mgr.y (mgr.25421) 941 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 706 op/s; 6003/261765 objects degraded (2.293%); 54822/261765 objects misplaced (20.943%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:08:45.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:45 smithi016 bash[77892]: cluster 2023-10-09T09:08:43.955295+0000 mgr.y (mgr.25421) 941 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 706 op/s; 6003/261765 objects degraded (2.293%); 54822/261765 objects misplaced (20.943%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:08:45.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:45.338 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:45 smithi094 bash[59677]: cluster 2023-10-09T09:08:43.955295+0000 mgr.y (mgr.25421) 941 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 706 op/s; 6003/261765 objects degraded (2.293%); 54822/261765 objects misplaced (20.943%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:08:45.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:45.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:45.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:45.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:45.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:45.600 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:45.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:45.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:45.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:45.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:45.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:45.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:45.899 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:45.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:45.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:08:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:08:45] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.33.4" 2023-10-09T09:08:46.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:46.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:46.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:46.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:46.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:46.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:46.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:46.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:46.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:46.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:46.550 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:46.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:46.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:46.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:46.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:46.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:46.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:46.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:46.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:46.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:46.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:46.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:47.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:47.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:47.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:47.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:47.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:47 smithi016 bash[77892]: cluster 2023-10-09T09:08:45.955913+0000 mgr.y (mgr.25421) 942 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 6003/260490 objects degraded (2.305%); 54822/260490 objects misplaced (21.046%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T09:08:47.214 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:47 smithi016 bash[75179]: cluster 2023-10-09T09:08:45.955913+0000 mgr.y (mgr.25421) 942 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 6003/260490 objects degraded (2.305%); 54822/260490 objects misplaced (21.046%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T09:08:47.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:47.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:47.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:47.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:47.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:47.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:47.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:47.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:47.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:47 smithi094 bash[59677]: cluster 2023-10-09T09:08:45.955913+0000 mgr.y (mgr.25421) 942 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 6003/260490 objects degraded (2.305%); 54822/260490 objects misplaced (21.046%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T09:08:47.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:47.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:47.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:47.654 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:47.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:47.759 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:47.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:47.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:47.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:47.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:47.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:47.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:48.030 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:48.058 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:48.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:48.191 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:48.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:48.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:48.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:48.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:48.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:48.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:48.560 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:48.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:48.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:48.694 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:48.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:48.825 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:48.851 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:48.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:48.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:48.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:49.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:49.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:49.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:49 smithi016 bash[75179]: cluster 2023-10-09T09:08:47.960059+0000 mgr.y (mgr.25421) 943 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 763 op/s; 6003/253866 objects degraded (2.365%); 53752/253866 objects misplaced (21.173%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T09:08:49.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:49 smithi016 bash[77892]: cluster 2023-10-09T09:08:47.960059+0000 mgr.y (mgr.25421) 943 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 763 op/s; 6003/253866 objects degraded (2.365%); 53752/253866 objects misplaced (21.173%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T09:08:49.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:49.237 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:49.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:49 smithi094 bash[59677]: cluster 2023-10-09T09:08:47.960059+0000 mgr.y (mgr.25421) 943 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 763 op/s; 6003/253866 objects degraded (2.365%); 53752/253866 objects misplaced (21.173%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T09:08:49.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:49.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:49.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:49.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:49.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:49.498 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:49.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:49.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:49.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:49.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:49.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:49.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:49.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:49.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:49.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:50.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:50.075 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:50 smithi016 bash[75179]: audit 2023-10-09T09:08:49.239804+0000 mon.a (mon.0) 1254 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:50.075 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:50 smithi016 bash[75179]: cluster 2023-10-09T09:08:49.626623+0000 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 6003/253866 objects degraded (2.365%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:50.076 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:50 smithi016 bash[77892]: audit 2023-10-09T09:08:49.239804+0000 mon.a (mon.0) 1254 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:50.076 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:50 smithi016 bash[77892]: cluster 2023-10-09T09:08:49.626623+0000 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 6003/253866 objects degraded (2.365%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:50.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:50.157 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:50 smithi094 bash[59677]: audit 2023-10-09T09:08:49.239804+0000 mon.a (mon.0) 1254 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:50.157 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:50 smithi094 bash[59677]: cluster 2023-10-09T09:08:49.626623+0000 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 6003/253866 objects degraded (2.365%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:50.174 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:50.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:50.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:50.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:50.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:50.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:50.716 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:50.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:50.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:50.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:50.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:50.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:50.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:50.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:51.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:51.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:51.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:51.218 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:51.226 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:51 smithi094 bash[59677]: cluster 2023-10-09T09:08:49.960652+0000 mgr.y (mgr.25421) 944 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 6003/252126 objects degraded (2.381%); 53174/252126 objects misplaced (21.090%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:51.241 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:51.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:51.302 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:51 smithi016 bash[75179]: cluster 2023-10-09T09:08:49.960652+0000 mgr.y (mgr.25421) 944 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 6003/252126 objects degraded (2.381%); 53174/252126 objects misplaced (21.090%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:51.303 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:51 smithi016 bash[77892]: cluster 2023-10-09T09:08:49.960652+0000 mgr.y (mgr.25421) 944 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 6003/252126 objects degraded (2.381%); 53174/252126 objects misplaced (21.090%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:51.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:51.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:51.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:51.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:51.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:51.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:51.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:51.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:51.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:51.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:51.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:51.749 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:51.802 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:51.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:51.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:51.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:51.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:51.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:52.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:52.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:52.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:52.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:52.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:52.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:52.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:52.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:52.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:52.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:52.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:52.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:52.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:52.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:52.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:52.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:52.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:52.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:52.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:52.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:52.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:52.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:52.891 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:52.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:52.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:52.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:53.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:53.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:53.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:53.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:53.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:53.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:53 smithi094 bash[59677]: cluster 2023-10-09T09:08:51.961360+0000 mgr.y (mgr.25421) 945 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 6003/247449 objects degraded (2.426%); 52750/247449 objects misplaced (21.318%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:53.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:53.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:53.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:53.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:53.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:53 smithi016 bash[75179]: cluster 2023-10-09T09:08:51.961360+0000 mgr.y (mgr.25421) 945 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 6003/247449 objects degraded (2.426%); 52750/247449 objects misplaced (21.318%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:53.384 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:53 smithi016 bash[77892]: cluster 2023-10-09T09:08:51.961360+0000 mgr.y (mgr.25421) 945 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 5.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 6003/247449 objects degraded (2.426%); 52750/247449 objects misplaced (21.318%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:53.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:53.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:53.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:53.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:53.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:53.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:53.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:53.819 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:53.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:53.946 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:53.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:53.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:54.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:54.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:54.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:54.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:54.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:54.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:54.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:54.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:54.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:54.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:54.472 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:54.495 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:54.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:54.598 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:54.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:54.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:54.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:54.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:54.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:54.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:54.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:54.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:55.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:55.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:55.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.179 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:55.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:55.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:55 smithi016 bash[75179]: cluster 2023-10-09T09:08:53.962036+0000 mgr.y (mgr.25421) 946 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 6003/242631 objects degraded (2.474%); 51425/242631 objects misplaced (21.195%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:08:55.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:55 smithi016 bash[75179]: cluster 2023-10-09T09:08:54.627652+0000 mon.a (mon.0) 1256 : cluster 3 Health check update: Degraded data redundancy: 6003/242631 objects degraded (2.474%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:55.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:55 smithi016 bash[77892]: cluster 2023-10-09T09:08:53.962036+0000 mgr.y (mgr.25421) 946 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 6003/242631 objects degraded (2.474%); 51425/242631 objects misplaced (21.195%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:08:55.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:55 smithi016 bash[77892]: cluster 2023-10-09T09:08:54.627652+0000 mon.a (mon.0) 1256 : cluster 3 Health check update: Degraded data redundancy: 6003/242631 objects degraded (2.474%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:55.288 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.296 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:55 smithi094 bash[59677]: cluster 2023-10-09T09:08:53.962036+0000 mgr.y (mgr.25421) 946 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 6003/242631 objects degraded (2.474%); 51425/242631 objects misplaced (21.195%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:08:55.296 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:55 smithi094 bash[59677]: cluster 2023-10-09T09:08:54.627652+0000 mon.a (mon.0) 1256 : cluster 3 Health check update: Degraded data redundancy: 6003/242631 objects degraded (2.474%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:08:55.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:55.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:55.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:55.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:55.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:55.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:55.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:55.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:55.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:55.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:55.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:55.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:55.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:55.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:55.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:55.986 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:08:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:08:55] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.33.4" 2023-10-09T09:08:56.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:56.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:56.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:56.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:56.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:56.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:56.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:56.373 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:56.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:56.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:56.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:56.532 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:56.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:56.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:56.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:56.797 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:56.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:56.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:56.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:56.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:56.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:57.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:57.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:57.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:57.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:57.177 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:57 smithi016 bash[75179]: cluster 2023-10-09T09:08:55.962678+0000 mgr.y (mgr.25421) 947 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 6003/241389 objects degraded (2.487%); 51425/241389 objects misplaced (21.304%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:57.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:57 smithi016 bash[77892]: cluster 2023-10-09T09:08:55.962678+0000 mgr.y (mgr.25421) 947 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 6003/241389 objects degraded (2.487%); 51425/241389 objects misplaced (21.304%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:57.183 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:57.286 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:57.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:57.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:57.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:57 smithi094 bash[59677]: cluster 2023-10-09T09:08:55.962678+0000 mgr.y (mgr.25421) 947 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 6003/241389 objects degraded (2.487%); 51425/241389 objects misplaced (21.304%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:57.454 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:57.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:57.461 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:08:57.462 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:08:57.462 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T09:08:57.462 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T09:08:57.462 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:08:57.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:57.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:57.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:57.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:57.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:57.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:57.853 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:08:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:08:57.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:57.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:08:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:57.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:08:58.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:58.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:58.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:58.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:58.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:58.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:58.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:58.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:58.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:58.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:58.492 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:58.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:58.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:58.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:58.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:08:58.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:58.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:08:58.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:58.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:58.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:58.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:58.939 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:58.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:59.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:59.069 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:08:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:59.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:08:59.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:59.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:59.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:59.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:59.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:59.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:59.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:59.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:59.437 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:08:59 smithi016 bash[75179]: cluster 2023-10-09T09:08:57.966476+0000 mgr.y (mgr.25421) 948 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 6003/237918 objects degraded (2.523%); 51048/237918 objects misplaced (21.456%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:59.437 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:08:59 smithi016 bash[77892]: cluster 2023-10-09T09:08:57.966476+0000 mgr.y (mgr.25421) 948 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 6003/237918 objects degraded (2.523%); 51048/237918 objects misplaced (21.456%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:59.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:08:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:59.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:08:59.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:08:59 smithi094 bash[59677]: cluster 2023-10-09T09:08:57.966476+0000 mgr.y (mgr.25421) 948 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 6003/237918 objects degraded (2.523%); 51048/237918 objects misplaced (21.456%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:08:59.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:59.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:08:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:59.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:08:59.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:59.715 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:08:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:59.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:08:59.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:59.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:08:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:08:59.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:08:59.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:00.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:00.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:00.096 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:00 smithi016 bash[75179]: cluster 2023-10-09T09:08:59.628590+0000 mon.a (mon.0) 1257 : cluster 3 Health check update: Degraded data redundancy: 6003/237918 objects degraded (2.523%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:00.107 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:00.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:00.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:00 smithi094 bash[59677]: cluster 2023-10-09T09:08:59.628590+0000 mon.a (mon.0) 1257 : cluster 3 Health check update: Degraded data redundancy: 6003/237918 objects degraded (2.523%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:00.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:00.270 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:00.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:00.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:00.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:00.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:00 smithi016 bash[77892]: cluster 2023-10-09T09:08:59.628590+0000 mon.a (mon.0) 1257 : cluster 3 Health check update: Degraded data redundancy: 6003/237918 objects degraded (2.523%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:00.500 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:00.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:00.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:00.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:00.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:00.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:00.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:00.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:00.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:00.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:00.950 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:00.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:00.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:01.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:01.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:01.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:01.096 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:01 smithi094 bash[59677]: cluster 2023-10-09T09:08:59.967176+0000 mgr.y (mgr.25421) 949 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 6003/232944 objects degraded (2.577%); 49696/232944 objects misplaced (21.334%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:01.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:01.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:01.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:01.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:01.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:01.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:01.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:01.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:01.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:01.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:01.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:01.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:01.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:01 smithi016 bash[75179]: cluster 2023-10-09T09:08:59.967176+0000 mgr.y (mgr.25421) 949 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 6003/232944 objects degraded (2.577%); 49696/232944 objects misplaced (21.334%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:01.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:01 smithi016 bash[77892]: cluster 2023-10-09T09:08:59.967176+0000 mgr.y (mgr.25421) 949 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 6003/232944 objects degraded (2.577%); 49696/232944 objects misplaced (21.334%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:01.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:01.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:01.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:01.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:01.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:01.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:01.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:01.867 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:01.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:01.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:02.001 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:02.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:02.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:02.163 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:02.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:02.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:02.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:02.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:02.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:02.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:02.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:02.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:02.668 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:02.702 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:02.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:02.826 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:02.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:02.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:02.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:02.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:03.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:03.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:03.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:03.198 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:03.224 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:03.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:03.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:03.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:03 smithi016 bash[75179]: cluster 2023-10-09T09:09:01.968181+0000 mgr.y (mgr.25421) 950 : cluster 0 pgmap v999: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 6003/227889 objects degraded (2.634%); 49278/227889 objects misplaced (21.624%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:03.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:03 smithi016 bash[77892]: cluster 2023-10-09T09:09:01.968181+0000 mgr.y (mgr.25421) 950 : cluster 0 pgmap v999: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 6003/227889 objects degraded (2.634%); 49278/227889 objects misplaced (21.624%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:03.329 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:03.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:03.364 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:03 smithi094 bash[59677]: cluster 2023-10-09T09:09:01.968181+0000 mgr.y (mgr.25421) 950 : cluster 0 pgmap v999: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 6003/227889 objects degraded (2.634%); 49278/227889 objects misplaced (21.624%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:03.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:03.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:03.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:09:03.466 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:03.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:03.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:03.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:03.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:03.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:03.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:03.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:03.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:03.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:03.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:03.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:03.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:03.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:04.012 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:04.039 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:04.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:04.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:04.278 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:04.311 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:04.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:04.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:04.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:04.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:04.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:04.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:04.611 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:09:04.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:04.680 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:09:04.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:04.734 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:09:04.785 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:09:04.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:04.826 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:09:04.851 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:04.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:04.864 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:09:04.901 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:09:04.942 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:09:04.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.948 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.949 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:09:04.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:04.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:04.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:05.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:05.106 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:05 smithi094 bash[59677]: cluster 2023-10-09T09:09:03.968857+0000 mgr.y (mgr.25421) 951 : cluster 0 pgmap v1000: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 6003/223173 objects degraded (2.690%); 47989/223173 objects misplaced (21.503%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:05.107 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:05 smithi094 bash[59677]: audit 2023-10-09T09:09:04.239873+0000 mon.a (mon.0) 1258 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:05.107 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:05 smithi094 bash[59677]: audit 2023-10-09T09:09:04.558839+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.16:0/703594732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:09:05.107 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:05 smithi094 bash[59677]: cluster 2023-10-09T09:09:04.629549+0000 mon.a (mon.0) 1259 : cluster 3 Health check update: Degraded data redundancy: 6003/223173 objects degraded (2.690%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:05.107 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:05 smithi094 bash[59677]: audit 2023-10-09T09:09:04.641932+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.16:0/3432159849' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T09:09:05.107 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:05 smithi094 bash[59677]: audit 2023-10-09T09:09:04.642693+0000 mon.a (mon.0) 1260 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T09:09:05.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:05.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:05.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:05.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:05.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[75179]: cluster 2023-10-09T09:09:03.968857+0000 mgr.y (mgr.25421) 951 : cluster 0 pgmap v1000: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 6003/223173 objects degraded (2.690%); 47989/223173 objects misplaced (21.503%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:05.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[75179]: audit 2023-10-09T09:09:04.239873+0000 mon.a (mon.0) 1258 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:05.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[75179]: audit 2023-10-09T09:09:04.558839+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.16:0/703594732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[75179]: cluster 2023-10-09T09:09:04.629549+0000 mon.a (mon.0) 1259 : cluster 3 Health check update: Degraded data redundancy: 6003/223173 objects degraded (2.690%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[75179]: audit 2023-10-09T09:09:04.641932+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.16:0/3432159849' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[75179]: audit 2023-10-09T09:09:04.642693+0000 mon.a (mon.0) 1260 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[77892]: cluster 2023-10-09T09:09:03.968857+0000 mgr.y (mgr.25421) 951 : cluster 0 pgmap v1000: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 6003/223173 objects degraded (2.690%); 47989/223173 objects misplaced (21.503%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[77892]: audit 2023-10-09T09:09:04.239873+0000 mon.a (mon.0) 1258 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[77892]: audit 2023-10-09T09:09:04.558839+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.16:0/703594732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[77892]: cluster 2023-10-09T09:09:04.629549+0000 mon.a (mon.0) 1259 : cluster 3 Health check update: Degraded data redundancy: 6003/223173 objects degraded (2.690%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[77892]: audit 2023-10-09T09:09:04.641932+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.16:0/3432159849' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T09:09:05.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[77892]: audit 2023-10-09T09:09:04.642693+0000 mon.a (mon.0) 1260 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T09:09:05.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:05.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:05.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:05.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:05.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:05.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:05.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:05.531 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:05.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:05.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:05.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:05.647 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:09:05.653 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.3 primary-affinity to 0 (802) 2023-10-09T09:09:05.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:05.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:05.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:05.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:05.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:05.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:05.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:05.916 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:05.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:05.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:05.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:09:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:09:05] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-09T09:09:06.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:06.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:06.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:06.180 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:06.192 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:06 smithi094 bash[59677]: audit 2023-10-09T09:09:05.103297+0000 mon.a (mon.0) 1261 : audit 1 from='client.? 172.21.15.16:0/3180101931' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:09:06.192 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:06 smithi094 bash[59677]: cluster 2023-10-09T09:09:05.631711+0000 mon.a (mon.0) 1262 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:09:06.192 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:06 smithi094 bash[59677]: cluster 2023-10-09T09:09:05.633670+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:06.192 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:06 smithi094 bash[59677]: audit 2023-10-09T09:09:05.637586+0000 mon.a (mon.0) 1263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T09:09:06.192 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:06 smithi094 bash[59677]: audit 2023-10-09T09:09:05.637776+0000 mon.a (mon.0) 1264 : audit 1 from='client.? 172.21.15.16:0/3180101931' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:09:06.192 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:06 smithi094 bash[59677]: cluster 2023-10-09T09:09:05.637877+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e225: 8 total, 8 up, 6 in 2023-10-09T09:09:06.192 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:06 smithi094 bash[59677]: cluster 2023-10-09T09:09:05.643376+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:06.209 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:06.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:06.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:06.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:06.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:06.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:06.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[75179]: audit 2023-10-09T09:09:05.103297+0000 mon.a (mon.0) 1261 : audit 1 from='client.? 172.21.15.16:0/3180101931' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:09:06.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[75179]: cluster 2023-10-09T09:09:05.631711+0000 mon.a (mon.0) 1262 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:09:06.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[75179]: cluster 2023-10-09T09:09:05.633670+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[75179]: audit 2023-10-09T09:09:05.637586+0000 mon.a (mon.0) 1263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[75179]: audit 2023-10-09T09:09:05.637776+0000 mon.a (mon.0) 1264 : audit 1 from='client.? 172.21.15.16:0/3180101931' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[75179]: cluster 2023-10-09T09:09:05.637877+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e225: 8 total, 8 up, 6 in 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[75179]: cluster 2023-10-09T09:09:05.643376+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[77892]: audit 2023-10-09T09:09:05.103297+0000 mon.a (mon.0) 1261 : audit 1 from='client.? 172.21.15.16:0/3180101931' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[77892]: cluster 2023-10-09T09:09:05.631711+0000 mon.a (mon.0) 1262 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[77892]: cluster 2023-10-09T09:09:05.633670+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[77892]: audit 2023-10-09T09:09:05.637586+0000 mon.a (mon.0) 1263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[77892]: audit 2023-10-09T09:09:05.637776+0000 mon.a (mon.0) 1264 : audit 1 from='client.? 172.21.15.16:0/3180101931' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[77892]: cluster 2023-10-09T09:09:05.637877+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e225: 8 total, 8 up, 6 in 2023-10-09T09:09:06.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:06 smithi016 bash[77892]: cluster 2023-10-09T09:09:05.643376+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:06.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:06.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:06.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:06.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:06.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:06.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:06.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:06.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:06.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:06.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:06.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:06.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:07.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:07.108 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:07.111 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:05.969375+0000 mgr.y (mgr.25421) 952 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 6003/222006 objects degraded (2.704%); 47989/222006 objects misplaced (21.616%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:09:07.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:07.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:07.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:07.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:07.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:05.969375+0000 mgr.y (mgr.25421) 952 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 6003/222006 objects degraded (2.704%); 47989/222006 objects misplaced (21.616%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.062583+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.642731+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.645751+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.645803+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.645914+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.646598+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e226: 8 total, 8 up, 6 in 2023-10-09T09:09:07.377 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.647908+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.648989+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.649910+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:05.969375+0000 mgr.y (mgr.25421) 952 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 6003/222006 objects degraded (2.704%); 47989/222006 objects misplaced (21.616%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.062583+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.642731+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.645751+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.645803+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.645914+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.378 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.646598+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e226: 8 total, 8 up, 6 in 2023-10-09T09:09:07.379 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.647908+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.379 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.648989+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.379 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:07 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.649910+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:07.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:07.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:07.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.062583+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.642731+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.645751+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.645803+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.645914+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.646598+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e226: 8 total, 8 up, 6 in 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.647908+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.648989+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:07 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.649910+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T09:09:07.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:07.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:07.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:07.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:07.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:07.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:07.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:07.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:07.876 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:07.907 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:07.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:07.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:08.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:08.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:08.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:08.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:08.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:08.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:08.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:08 smithi094 bash[59677]: cluster 2023-10-09T09:09:06.644967+0000 osd.3 (osd.3) 6 : cluster 1 9.3 continuing backfill to osd.5 from (218'2100,224'7691] MIN to 224'7691 2023-10-09T09:09:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:08 smithi094 bash[59677]: cluster 2023-10-09T09:09:07.644248+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:08 smithi094 bash[59677]: cluster 2023-10-09T09:09:07.644437+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:08 smithi094 bash[59677]: cluster 2023-10-09T09:09:07.647620+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e227: 8 total, 8 up, 6 in 2023-10-09T09:09:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:08 smithi094 bash[59677]: cluster 2023-10-09T09:09:07.649406+0000 osd.0 (osd.0) 13 : cluster 1 9.8 continuing backfill to osd.5 from (218'2100,224'7734] MIN to 224'7734 2023-10-09T09:09:08.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:08 smithi094 bash[59677]: cluster 2023-10-09T09:09:07.685797+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[75179]: cluster 2023-10-09T09:09:06.644967+0000 osd.3 (osd.3) 6 : cluster 1 9.3 continuing backfill to osd.5 from (218'2100,224'7691] MIN to 224'7691 2023-10-09T09:09:08.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[75179]: cluster 2023-10-09T09:09:07.644248+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[75179]: cluster 2023-10-09T09:09:07.644437+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[75179]: cluster 2023-10-09T09:09:07.647620+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e227: 8 total, 8 up, 6 in 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[75179]: cluster 2023-10-09T09:09:07.649406+0000 osd.0 (osd.0) 13 : cluster 1 9.8 continuing backfill to osd.5 from (218'2100,224'7734] MIN to 224'7734 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[75179]: cluster 2023-10-09T09:09:07.685797+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[77892]: cluster 2023-10-09T09:09:06.644967+0000 osd.3 (osd.3) 6 : cluster 1 9.3 continuing backfill to osd.5 from (218'2100,224'7691] MIN to 224'7691 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[77892]: cluster 2023-10-09T09:09:07.644248+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[77892]: cluster 2023-10-09T09:09:07.644437+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[77892]: cluster 2023-10-09T09:09:07.647620+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e227: 8 total, 8 up, 6 in 2023-10-09T09:09:08.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[77892]: cluster 2023-10-09T09:09:07.649406+0000 osd.0 (osd.0) 13 : cluster 1 9.8 continuing backfill to osd.5 from (218'2100,224'7734] MIN to 224'7734 2023-10-09T09:09:08.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:08 smithi016 bash[77892]: cluster 2023-10-09T09:09:07.685797+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T09:09:08.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:08.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:08.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:08.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:08.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:08.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:08.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:08.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:08.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:08.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:08.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:08.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:08.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:08.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:08.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:08.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:08.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:09.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:09.106 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:09.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:09.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:09.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:09 smithi094 bash[59677]: cluster 2023-10-09T09:09:07.969873+0000 mgr.y (mgr.25421) 953 : cluster 0 pgmap v1005: 89 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 74 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 6003/220017 objects degraded (2.728%); 38545/220017 objects misplaced (17.519%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:09:09.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:09 smithi094 bash[59677]: cluster 2023-10-09T09:09:08.643691+0000 mon.a (mon.0) 1268 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:09:09.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:09.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:09.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:09.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:09 smithi016 bash[75179]: cluster 2023-10-09T09:09:07.969873+0000 mgr.y (mgr.25421) 953 : cluster 0 pgmap v1005: 89 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 74 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 6003/220017 objects degraded (2.728%); 38545/220017 objects misplaced (17.519%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:09:09.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:09 smithi016 bash[75179]: cluster 2023-10-09T09:09:08.643691+0000 mon.a (mon.0) 1268 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:09:09.375 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:09 smithi016 bash[77892]: cluster 2023-10-09T09:09:07.969873+0000 mgr.y (mgr.25421) 953 : cluster 0 pgmap v1005: 89 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 74 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 6003/220017 objects degraded (2.728%); 38545/220017 objects misplaced (17.519%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:09:09.375 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:09 smithi016 bash[77892]: cluster 2023-10-09T09:09:08.643691+0000 mon.a (mon.0) 1268 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:09:09.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:09.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:09.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:09.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:09.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:09.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:09.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:09.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:09.785 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:09.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:09.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:09.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:09.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:09:10.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:10.070 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:10.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:10.179 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:10.204 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:10 smithi016 bash[75179]: cluster 2023-10-09T09:09:09.630606+0000 mon.a (mon.0) 1269 : cluster 3 Health check update: Degraded data redundancy: 6003/220017 objects degraded (2.728%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:10.204 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:10.206 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:10 smithi016 bash[77892]: cluster 2023-10-09T09:09:09.630606+0000 mon.a (mon.0) 1269 : cluster 3 Health check update: Degraded data redundancy: 6003/220017 objects degraded (2.728%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:10.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:10.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:10.322 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:10 smithi094 bash[59677]: cluster 2023-10-09T09:09:09.630606+0000 mon.a (mon.0) 1269 : cluster 3 Health check update: Degraded data redundancy: 6003/220017 objects degraded (2.728%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:10.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:10.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:10.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:10.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:10.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:10.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:10.589 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:10.613 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:10.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:10.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:10.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:10.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:10.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:10.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:10.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:10.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:10.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:11.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:11.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:11.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:11.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:11.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:11.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:11.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:11 smithi094 bash[59677]: cluster 2023-10-09T09:09:09.972524+0000 mgr.y (mgr.25421) 954 : cluster 0 pgmap v1006: 89 pgs: 1 remapped+peering, 4 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 74 active+clean; 4.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 397 op/s; 6003/216012 objects degraded (2.779%); 42078/216012 objects misplaced (19.479%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:11.285 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:11.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:11.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:11.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:11.445 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:11.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:11.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:11 smithi016 bash[75179]: cluster 2023-10-09T09:09:09.972524+0000 mgr.y (mgr.25421) 954 : cluster 0 pgmap v1006: 89 pgs: 1 remapped+peering, 4 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 74 active+clean; 4.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 397 op/s; 6003/216012 objects degraded (2.779%); 42078/216012 objects misplaced (19.479%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:11.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:11 smithi016 bash[77892]: cluster 2023-10-09T09:09:09.972524+0000 mgr.y (mgr.25421) 954 : cluster 0 pgmap v1006: 89 pgs: 1 remapped+peering, 4 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 74 active+clean; 4.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 397 op/s; 6003/216012 objects degraded (2.779%); 42078/216012 objects misplaced (19.479%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:11.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:11.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:11.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:11.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:11.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:11.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:11.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:11.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:11.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:11.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:11.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:12.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:12.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:12.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:12.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:09:12.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:12 smithi016 bash[75179]: cluster 2023-10-09T09:09:12.116394+0000 mon.a (mon.0) 1270 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T09:09:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:12 smithi016 bash[77892]: cluster 2023-10-09T09:09:12.116394+0000 mon.a (mon.0) 1270 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T09:09:12.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:12.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:12.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:12.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:12.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:12.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:12 smithi094 bash[59677]: cluster 2023-10-09T09:09:12.116394+0000 mon.a (mon.0) 1270 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T09:09:12.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:12.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:12.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:12.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:12.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:12.752 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:09:12.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:12.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:12.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:12.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:12.954 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:12.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:12.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:13.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:13.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:13.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:13.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:13.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:13 smithi094 bash[59677]: cluster 2023-10-09T09:09:11.973345+0000 mgr.y (mgr.25421) 955 : cluster 0 pgmap v1007: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 6003/212598 objects degraded (2.824%); 46441/212598 objects misplaced (21.845%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:09:13.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:13.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:13.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:13.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:13 smithi016 bash[75179]: cluster 2023-10-09T09:09:11.973345+0000 mgr.y (mgr.25421) 955 : cluster 0 pgmap v1007: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 6003/212598 objects degraded (2.824%); 46441/212598 objects misplaced (21.845%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:09:13.352 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:13 smithi016 bash[77892]: cluster 2023-10-09T09:09:11.973345+0000 mgr.y (mgr.25421) 955 : cluster 0 pgmap v1007: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 6003/212598 objects degraded (2.824%); 46441/212598 objects misplaced (21.845%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:09:13.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:13.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:13.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:13.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:13.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:13.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:13.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:13.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:13.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:13.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:13.788 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:13.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:13.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:13.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:13.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:13.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:14.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:14.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:14.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:14.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:14.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:14.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:14.300 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:14.326 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:14.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:14.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:14.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:14.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:14.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:14.565 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:14.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:14.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:14.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:14.692 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:14.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:14.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:14.798 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:09:14.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:14.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:14.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:14.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:15.004 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:15.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:15.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:15.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:15.248 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5] out_osds: [7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:09:15.248 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:09:15.248 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T09:09:15.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T09:09:15.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:15.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:15.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:15.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:15 smithi094 bash[59677]: cluster 2023-10-09T09:09:13.974020+0000 mgr.y (mgr.25421) 956 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 6003/206205 objects degraded (2.911%); 44832/206205 objects misplaced (21.741%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T09:09:15.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:15 smithi094 bash[59677]: cluster 2023-10-09T09:09:14.631765+0000 mon.a (mon.0) 1271 : cluster 3 Health check update: Degraded data redundancy: 6003/206205 objects degraded (2.911%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:15.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:15 smithi094 bash[59677]: audit 2023-10-09T09:09:14.983754+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:15.404 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:15.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:15 smithi016 bash[77892]: cluster 2023-10-09T09:09:13.974020+0000 mgr.y (mgr.25421) 956 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 6003/206205 objects degraded (2.911%); 44832/206205 objects misplaced (21.741%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T09:09:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:15 smithi016 bash[77892]: cluster 2023-10-09T09:09:14.631765+0000 mon.a (mon.0) 1271 : cluster 3 Health check update: Degraded data redundancy: 6003/206205 objects degraded (2.911%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:15 smithi016 bash[77892]: audit 2023-10-09T09:09:14.983754+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:15 smithi016 bash[75179]: cluster 2023-10-09T09:09:13.974020+0000 mgr.y (mgr.25421) 956 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 6003/206205 objects degraded (2.911%); 44832/206205 objects misplaced (21.741%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T09:09:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:15 smithi016 bash[75179]: cluster 2023-10-09T09:09:14.631765+0000 mon.a (mon.0) 1271 : cluster 3 Health check update: Degraded data redundancy: 6003/206205 objects degraded (2.911%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:15.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:15 smithi016 bash[75179]: audit 2023-10-09T09:09:14.983754+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:15.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:15.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:15.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:15.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:15.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:15.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:15.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:09:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:09:15] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-09T09:09:15.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:15.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:15.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:15.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:16.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:16.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:16.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:16.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:16.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:16.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:16.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:16.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:16.366 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:16.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:16.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:16.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:16.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:16.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:16.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:16.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:16.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:16.734 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:16.757 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:16.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:16.830 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:09:16.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:16.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:16.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:16.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:17.016 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:17.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:17.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:17.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:17.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:17.373 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:17.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:17 smithi094 bash[59677]: cluster 2023-10-09T09:09:15.974655+0000 mgr.y (mgr.25421) 957 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 6003/204288 objects degraded (2.938%); 44832/204288 objects misplaced (21.945%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T09:09:17.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:17 smithi016 bash[75179]: cluster 2023-10-09T09:09:15.974655+0000 mgr.y (mgr.25421) 957 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 6003/204288 objects degraded (2.938%); 44832/204288 objects misplaced (21.945%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T09:09:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:17 smithi016 bash[77892]: cluster 2023-10-09T09:09:15.974655+0000 mgr.y (mgr.25421) 957 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 6003/204288 objects degraded (2.938%); 44832/204288 objects misplaced (21.945%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T09:09:17.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:17.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:17.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:17.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:17.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:17.813 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:17.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:17.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:17.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:18.061 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:18.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:18.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:18.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:18.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:18.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:18.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:18.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:18.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:18.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:18.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:18.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:18.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:18.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:18.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:18.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:18.586 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:18.612 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:18.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:18.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:18.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:18.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:18.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:18.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:18.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:18.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:18.973 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:19.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:19.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:19.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:19.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:19 smithi094 bash[59677]: cluster 2023-10-09T09:09:17.975208+0000 mgr.y (mgr.25421) 958 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 6003/202518 objects degraded (2.964%); 44635/202518 objects misplaced (22.040%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:09:19.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:19.280 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:19.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:19.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.414 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:19.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.526 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:19 smithi016 bash[75179]: cluster 2023-10-09T09:09:17.975208+0000 mgr.y (mgr.25421) 958 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 6003/202518 objects degraded (2.964%); 44635/202518 objects misplaced (22.040%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:09:19.527 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:19 smithi016 bash[77892]: cluster 2023-10-09T09:09:17.975208+0000 mgr.y (mgr.25421) 958 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 6003/202518 objects degraded (2.964%); 44635/202518 objects misplaced (22.040%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:09:19.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:19.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:19.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:19.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:19.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:19.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:19.888 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:09:19.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:19.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:19.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:20.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:20.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:20.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:20.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:20.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:20.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:20.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:20.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:20.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:20.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:20 smithi094 bash[59677]: audit 2023-10-09T09:09:19.242625+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:20 smithi094 bash[59677]: audit 2023-10-09T09:09:19.243355+0000 mon.a (mon.0) 1274 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:20 smithi094 bash[59677]: cluster 2023-10-09T09:09:19.632888+0000 mon.a (mon.0) 1275 : cluster 3 Health check update: Degraded data redundancy: 6003/202518 objects degraded (2.964%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:20.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:20 smithi016 bash[75179]: audit 2023-10-09T09:09:19.242625+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:20 smithi016 bash[75179]: audit 2023-10-09T09:09:19.243355+0000 mon.a (mon.0) 1274 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:20 smithi016 bash[75179]: cluster 2023-10-09T09:09:19.632888+0000 mon.a (mon.0) 1275 : cluster 3 Health check update: Degraded data redundancy: 6003/202518 objects degraded (2.964%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:20.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:20 smithi016 bash[77892]: audit 2023-10-09T09:09:19.242625+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:20 smithi016 bash[77892]: audit 2023-10-09T09:09:19.243355+0000 mon.a (mon.0) 1274 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:20 smithi016 bash[77892]: cluster 2023-10-09T09:09:19.632888+0000 mon.a (mon.0) 1275 : cluster 3 Health check update: Degraded data redundancy: 6003/202518 objects degraded (2.964%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:20.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:20.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:20.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:20.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:20.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:20.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:20.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:20.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:20.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:20.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:20.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:20.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:20.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:20.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:21.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:21.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:21.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:21.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:21.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:21.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:21.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:21.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:21.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:21.357 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:21.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:21.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:21.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:21.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:21.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:21.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:21.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:21 smithi016 bash[75179]: cluster 2023-10-09T09:09:19.975862+0000 mgr.y (mgr.25421) 959 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 6003/196569 objects degraded (3.054%); 43123/196569 objects misplaced (21.938%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T09:09:21.556 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:21 smithi016 bash[77892]: cluster 2023-10-09T09:09:19.975862+0000 mgr.y (mgr.25421) 959 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 6003/196569 objects degraded (3.054%); 43123/196569 objects misplaced (21.938%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T09:09:21.595 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:21.603 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:21 smithi094 bash[59677]: cluster 2023-10-09T09:09:19.975862+0000 mgr.y (mgr.25421) 959 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 4.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 6003/196569 objects degraded (3.054%); 43123/196569 objects misplaced (21.938%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T09:09:21.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:21.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:21.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:21.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:21.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:21.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:21.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:21.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:22.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:22.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:22.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:22.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:22.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:22.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:22.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:22.323 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:22.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:22.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:22.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:22.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:22.549 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:22.597 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:22.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:22.702 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:22.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:22.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:22.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:22.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:22.867 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:22.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:22.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:23.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:23.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:23.110 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:23.145 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:23.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:23.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:23.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:23.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:23.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:23 smithi016 bash[75179]: cluster 2023-10-09T09:09:21.976551+0000 mgr.y (mgr.25421) 960 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 6003/192810 objects degraded (3.113%); 42929/192810 objects misplaced (22.265%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T09:09:23.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:23 smithi016 bash[75179]: audit 2023-10-09T09:09:23.058856+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:23.485 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:23 smithi016 bash[77892]: cluster 2023-10-09T09:09:21.976551+0000 mgr.y (mgr.25421) 960 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 6003/192810 objects degraded (3.113%); 42929/192810 objects misplaced (22.265%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T09:09:23.485 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:23 smithi016 bash[77892]: audit 2023-10-09T09:09:23.058856+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:23 smithi094 bash[59677]: cluster 2023-10-09T09:09:21.976551+0000 mgr.y (mgr.25421) 960 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 6003/192810 objects degraded (3.113%); 42929/192810 objects misplaced (22.265%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T09:09:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:23 smithi094 bash[59677]: audit 2023-10-09T09:09:23.058856+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:23.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:23.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:23.656 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:23.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:23.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:23.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:23.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:23.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:23.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:23.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:23.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:24.061 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:24.089 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:24.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:24.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:24.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:24.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:24.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:24.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:24.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:24.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:24.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:24.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:24.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:24.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:24.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:24.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:24.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:24.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:24.735 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:24.765 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:24.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:24.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:24.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:24.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:24.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:25.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:25.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:25.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:25.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:25.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:25.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:25.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:25.400 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:25.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:25.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:25 smithi016 bash[75179]: cluster 2023-10-09T09:09:23.977223+0000 mgr.y (mgr.25421) 961 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 6003/187230 objects degraded (3.206%); 41545/187230 objects misplaced (22.189%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:09:25.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:25 smithi016 bash[75179]: cluster 2023-10-09T09:09:24.633998+0000 mon.a (mon.0) 1277 : cluster 3 Health check update: Degraded data redundancy: 6003/187230 objects degraded (3.206%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:25.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:25 smithi016 bash[77892]: cluster 2023-10-09T09:09:23.977223+0000 mgr.y (mgr.25421) 961 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 6003/187230 objects degraded (3.206%); 41545/187230 objects misplaced (22.189%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:09:25.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:25 smithi016 bash[77892]: cluster 2023-10-09T09:09:24.633998+0000 mon.a (mon.0) 1277 : cluster 3 Health check update: Degraded data redundancy: 6003/187230 objects degraded (3.206%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:25.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:25.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:25.678 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:25.687 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:25 smithi094 bash[59677]: cluster 2023-10-09T09:09:23.977223+0000 mgr.y (mgr.25421) 961 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 6003/187230 objects degraded (3.206%); 41545/187230 objects misplaced (22.189%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:09:25.688 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:25 smithi094 bash[59677]: cluster 2023-10-09T09:09:24.633998+0000 mon.a (mon.0) 1277 : cluster 3 Health check update: Degraded data redundancy: 6003/187230 objects degraded (3.206%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:25.712 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:25.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:25.856 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:25.950 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:09:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:09:25] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-09T09:09:25.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:25.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:25.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:26.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:26.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:26.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:26.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:26.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:26.267 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:26.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:26.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:26.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:26.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:26.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:26.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:26.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:26.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:26.641 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:26.669 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:26.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:26.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:26.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:26.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:26.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:26.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:26.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:27.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:27.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:27.158 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:27.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:27.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:27.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:27.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:27.430 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:27 smithi094 bash[59677]: cluster 2023-10-09T09:09:25.977868+0000 mgr.y (mgr.25421) 962 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 6003/185610 objects degraded (3.234%); 41545/185610 objects misplaced (22.383%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:27.447 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:27.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:27.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:27.682 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:27.696 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:27 smithi016 bash[75179]: cluster 2023-10-09T09:09:25.977868+0000 mgr.y (mgr.25421) 962 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 6003/185610 objects degraded (3.234%); 41545/185610 objects misplaced (22.383%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:27.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:27 smithi016 bash[77892]: cluster 2023-10-09T09:09:25.977868+0000 mgr.y (mgr.25421) 962 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 6003/185610 objects degraded (3.234%); 41545/185610 objects misplaced (22.383%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:27.713 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:27.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:27.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:27.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:27.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:27.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:27.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:27.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:27.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:28.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:28.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:28.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:28.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:28.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:28.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:28.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:28.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:28.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:28.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:28.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:28.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:28.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:28.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:28.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:28.629 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:28.654 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:28.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:28.762 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:28.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:28.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:28.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:28.916 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:28.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:28.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:29.018 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:29.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:29.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:29.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:29.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:29.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:29.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:29.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:29.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:29.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:29.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:29.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:29.493 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:29.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:29 smithi016 bash[75179]: cluster 2023-10-09T09:09:27.978720+0000 mgr.y (mgr.25421) 963 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 6003/183948 objects degraded (3.263%); 41363/183948 objects misplaced (22.486%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:29.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:29 smithi016 bash[77892]: cluster 2023-10-09T09:09:27.978720+0000 mgr.y (mgr.25421) 963 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 6003/183948 objects degraded (3.263%); 41363/183948 objects misplaced (22.486%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:29.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:29.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:29.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:29.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:29 smithi094 bash[59677]: cluster 2023-10-09T09:09:27.978720+0000 mgr.y (mgr.25421) 963 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 6003/183948 objects degraded (3.263%); 41363/183948 objects misplaced (22.486%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:29.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:29.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:29.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:29.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:29.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:29.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:30.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:30.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:30.123 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:30.156 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:30.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:30.260 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:30.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:30.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:30.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:30.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:30.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:30 smithi016 bash[75179]: cluster 2023-10-09T09:09:29.634987+0000 mon.a (mon.0) 1278 : cluster 3 Health check update: Degraded data redundancy: 6003/183948 objects degraded (3.263%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:30.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:30 smithi016 bash[75179]: audit 2023-10-09T09:09:29.653353+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:30.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:30 smithi016 bash[75179]: audit 2023-10-09T09:09:29.661695+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:30.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:30 smithi016 bash[77892]: cluster 2023-10-09T09:09:29.634987+0000 mon.a (mon.0) 1278 : cluster 3 Health check update: Degraded data redundancy: 6003/183948 objects degraded (3.263%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:30.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:30 smithi016 bash[77892]: audit 2023-10-09T09:09:29.653353+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:30.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:30 smithi016 bash[77892]: audit 2023-10-09T09:09:29.661695+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:30.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:30 smithi094 bash[59677]: cluster 2023-10-09T09:09:29.634987+0000 mon.a (mon.0) 1278 : cluster 3 Health check update: Degraded data redundancy: 6003/183948 objects degraded (3.263%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:30 smithi094 bash[59677]: audit 2023-10-09T09:09:29.653353+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:30.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:30 smithi094 bash[59677]: audit 2023-10-09T09:09:29.661695+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:30.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:30.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:30.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:30.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:30.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:30.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:30.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:30.914 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:30.939 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:30.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:30.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:31.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:31.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:31.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:31.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:31.204 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:31.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:31.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:31.329 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:31.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:31.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:31.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:31.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:31.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:31 smithi094 bash[59677]: cluster 2023-10-09T09:09:29.979471+0000 mgr.y (mgr.25421) 964 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 681 op/s; 6003/177960 objects degraded (3.373%); 39863/177960 objects misplaced (22.400%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:31.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:31.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:31.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:31.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:31.671 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:31 smithi016 bash[75179]: cluster 2023-10-09T09:09:29.979471+0000 mgr.y (mgr.25421) 964 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 681 op/s; 6003/177960 objects degraded (3.373%); 39863/177960 objects misplaced (22.400%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:31.672 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:31 smithi016 bash[77892]: cluster 2023-10-09T09:09:29.979471+0000 mgr.y (mgr.25421) 964 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 681 op/s; 6003/177960 objects degraded (3.373%); 39863/177960 objects misplaced (22.400%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:09:31.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:31.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:31.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:31.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:31.841 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:31.875 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:31.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:31.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:31.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:32.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:32.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:32.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:32.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:32.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:32.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:32.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:32.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:32.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:32.538 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:32.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:32.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:32.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:32.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:32.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:32.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:32.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:32.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:32.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:32.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:33.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:33.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:33.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:33.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:33.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:33.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:33.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:33.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:33.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:33.350 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:33 smithi016 bash[75179]: cluster 2023-10-09T09:09:31.980185+0000 mgr.y (mgr.25421) 965 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 6003/174435 objects degraded (3.441%); 39683/174435 objects misplaced (22.749%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:33.351 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:33 smithi016 bash[77892]: cluster 2023-10-09T09:09:31.980185+0000 mgr.y (mgr.25421) 965 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 6003/174435 objects degraded (3.441%); 39683/174435 objects misplaced (22.749%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:33.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:33.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:33.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:33.482 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:33.506 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:33.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:33.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:33.620 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:33 smithi094 bash[59677]: cluster 2023-10-09T09:09:31.980185+0000 mgr.y (mgr.25421) 965 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 6003/174435 objects degraded (3.441%); 39683/174435 objects misplaced (22.749%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T09:09:33.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:33.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:33.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:33.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:33.793 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:33.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:33.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:33.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:33.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:34.026 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:34.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:34.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:34.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:34.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:34.297 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:34.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:34.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:34.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:34.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:34 smithi016 bash[75179]: audit 2023-10-09T09:09:34.240436+0000 mon.a (mon.0) 1281 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:34.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:34 smithi016 bash[77892]: audit 2023-10-09T09:09:34.240436+0000 mon.a (mon.0) 1281 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:34.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:34.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:34.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:34.631 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:09:34.632 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:09:34.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:34.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:34.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:34.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:34 smithi094 bash[59677]: audit 2023-10-09T09:09:34.240436+0000 mon.a (mon.0) 1281 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:34.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:34.869 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:34.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:34.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:34.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:35.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:35.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:35.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:35.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:35.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:35.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:35.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:35.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:35.384 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:35 smithi094 bash[59677]: cluster 2023-10-09T09:09:33.980873+0000 mgr.y (mgr.25421) 966 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 669 op/s; 6003/168717 objects degraded (3.558%); 38216/168717 objects misplaced (22.651%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:09:35.384 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:35 smithi094 bash[59677]: cluster 2023-10-09T09:09:34.637210+0000 mon.a (mon.0) 1282 : cluster 3 Health check update: Degraded data redundancy: 6003/168717 objects degraded (3.558%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:35.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:35.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:35.516 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:35.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:35.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:35.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:35.675 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:35.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:35.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:35 smithi016 bash[75179]: cluster 2023-10-09T09:09:33.980873+0000 mgr.y (mgr.25421) 966 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 669 op/s; 6003/168717 objects degraded (3.558%); 38216/168717 objects misplaced (22.651%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:09:35.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:35 smithi016 bash[75179]: cluster 2023-10-09T09:09:34.637210+0000 mon.a (mon.0) 1282 : cluster 3 Health check update: Degraded data redundancy: 6003/168717 objects degraded (3.558%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:35.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:35 smithi016 bash[77892]: cluster 2023-10-09T09:09:33.980873+0000 mgr.y (mgr.25421) 966 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 669 op/s; 6003/168717 objects degraded (3.558%); 38216/168717 objects misplaced (22.651%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:09:35.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:35 smithi016 bash[77892]: cluster 2023-10-09T09:09:34.637210+0000 mon.a (mon.0) 1282 : cluster 3 Health check update: Degraded data redundancy: 6003/168717 objects degraded (3.558%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:35.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:35.808 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:35.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:35.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:35.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:35.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:35.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:35.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:09:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:09:35] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.33.4" 2023-10-09T09:09:36.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:36.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:36.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:36.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:36.196 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:36.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:36.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:36.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:36.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:36.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:36.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:36.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:36.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:36.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:36.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:36.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:36.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:36.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:36.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:36.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:36.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:36.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:36.879 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:36.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:36.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:37.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:37.048 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:37.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:37.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:37.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:37.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:37.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:37.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:37.285 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:37 smithi094 bash[59677]: cluster 2023-10-09T09:09:35.981521+0000 mgr.y (mgr.25421) 967 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 6003/166800 objects degraded (3.599%); 38216/166800 objects misplaced (22.911%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:09:37.288 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:37.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:37.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:37.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:37.419 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:37.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:37.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:37.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:37.521 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:37 smithi016 bash[75179]: cluster 2023-10-09T09:09:35.981521+0000 mgr.y (mgr.25421) 967 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 6003/166800 objects degraded (3.599%); 38216/166800 objects misplaced (22.911%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:09:37.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:37 smithi016 bash[77892]: cluster 2023-10-09T09:09:35.981521+0000 mgr.y (mgr.25421) 967 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 6003/166800 objects degraded (3.599%); 38216/166800 objects misplaced (22.911%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:09:37.570 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:37.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:37.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:37.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:37.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:37.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:37.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:37.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:37.869 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:37.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:37.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:38.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:38.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:38.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:38.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:38.250 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:38.276 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:38.379 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:38.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:38.519 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:38.553 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:38.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:38.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:38.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:38.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:38.832 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:38.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:38.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:38.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:38.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:39.065 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:39.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:39.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.239 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:39.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:39.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.355 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:39 smithi094 bash[59677]: cluster 2023-10-09T09:09:37.982139+0000 mgr.y (mgr.25421) 968 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 6003/165021 objects degraded (3.638%); 38007/165021 objects misplaced (23.032%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:09:39.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:39.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:39.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:39.508 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:39.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:39.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:39.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:39.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:39.741 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:39.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:39 smithi016 bash[75179]: cluster 2023-10-09T09:09:37.982139+0000 mgr.y (mgr.25421) 968 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 6003/165021 objects degraded (3.638%); 38007/165021 objects misplaced (23.032%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:09:39.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:39 smithi016 bash[77892]: cluster 2023-10-09T09:09:37.982139+0000 mgr.y (mgr.25421) 968 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 6003/165021 objects degraded (3.638%); 38007/165021 objects misplaced (23.032%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:09:39.768 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:39.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:39.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:39.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:39.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:39.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:39.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:40.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:40.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:40.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:40.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:40.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:40.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:40.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.420 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:40 smithi094 bash[59677]: cluster 2023-10-09T09:09:39.638285+0000 mon.a (mon.0) 1283 : cluster 3 Health check update: Degraded data redundancy: 6003/165021 objects degraded (3.638%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:40.445 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:40.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.577 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:40.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:40.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:40 smithi016 bash[75179]: cluster 2023-10-09T09:09:39.638285+0000 mon.a (mon.0) 1283 : cluster 3 Health check update: Degraded data redundancy: 6003/165021 objects degraded (3.638%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:40.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:40 smithi016 bash[77892]: cluster 2023-10-09T09:09:39.638285+0000 mon.a (mon.0) 1283 : cluster 3 Health check update: Degraded data redundancy: 6003/165021 objects degraded (3.638%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:40.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:40.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.847 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:40.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:40.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:40.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:41.064 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:09:41.097 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:41.136 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:41.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:41.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:41.240 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:41.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:41.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:41.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:41.389 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:41.397 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:41 smithi094 bash[59677]: cluster 2023-10-09T09:09:39.982823+0000 mgr.y (mgr.25421) 969 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 688 op/s; 6003/159153 objects degraded (3.772%); 36464/159153 objects misplaced (22.911%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T09:09:41.420 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:41.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:41.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:41.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:41 smithi016 bash[75179]: cluster 2023-10-09T09:09:39.982823+0000 mgr.y (mgr.25421) 969 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 688 op/s; 6003/159153 objects degraded (3.772%); 36464/159153 objects misplaced (22.911%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T09:09:41.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:41 smithi016 bash[77892]: cluster 2023-10-09T09:09:39.982823+0000 mgr.y (mgr.25421) 969 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 688 op/s; 6003/159153 objects degraded (3.772%); 36464/159153 objects misplaced (22.911%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T09:09:41.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:41.548 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:41.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:41.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:41.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:41.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:41.806 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:09:41.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:41.871 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:09:41.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:41.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:41.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:41.944 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:09:41.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:42.015 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:09:42.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:42.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:42.063 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:09:42.113 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:09:42.122 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:42.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:42.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:42.174 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:09:42.234 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.246 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:09:42.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:42.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:42.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:42.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:42.320 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:42 smithi094 bash[59677]: audit 2023-10-09T09:09:41.739563+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.16:0/551314568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:09:42.320 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:42 smithi094 bash[59677]: audit 2023-10-09T09:09:42.222186+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.16:0/861202458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:09:42.321 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:42 smithi094 bash[59677]: audit 2023-10-09T09:09:42.226599+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:09:42.337 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:09:42.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:42.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:42.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:42.547 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:42.570 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:42.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:42.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:42.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:42 smithi016 bash[75179]: audit 2023-10-09T09:09:41.739563+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.16:0/551314568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:09:42.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:42 smithi016 bash[75179]: audit 2023-10-09T09:09:42.222186+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.16:0/861202458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:09:42.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:42 smithi016 bash[75179]: audit 2023-10-09T09:09:42.226599+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:09:42.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:42 smithi016 bash[75179]: debug 2023-10-09T09:09:42.650+0000 7fa046a4d700 -1 mon.a@0(leader).osd e228 definitely_dead 0 2023-10-09T09:09:42.656 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:42 smithi016 bash[77892]: audit 2023-10-09T09:09:41.739563+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.16:0/551314568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:09:42.656 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:42 smithi016 bash[77892]: audit 2023-10-09T09:09:42.222186+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.16:0/861202458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:09:42.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:42 smithi016 bash[77892]: audit 2023-10-09T09:09:42.226599+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:09:42.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:42.696 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:42.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:42.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:42.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:42.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:42.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:43.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:43.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:43.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:43.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:43.334 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.7. 2023-10-09T09:09:43.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:43.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:43.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:43.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:43.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:43.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: cluster 2023-10-09T09:09:41.984619+0000 mgr.y (mgr.25421) 970 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 6003/155763 objects degraded (3.854%); 36277/155763 objects misplaced (23.290%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:09:43.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: cluster 2023-10-09T09:09:42.300328+0000 mon.a (mon.0) 1285 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:09:43.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: cluster 2023-10-09T09:09:42.318272+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: cluster 2023-10-09T09:09:42.321306+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: audit 2023-10-09T09:09:42.324277+0000 mon.a (mon.0) 1286 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: cluster 2023-10-09T09:09:42.324508+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e228: 8 total, 8 up, 6 in 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: audit 2023-10-09T09:09:42.654091+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.16:0/3025189797' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: audit 2023-10-09T09:09:42.654722+0000 mon.a (mon.0) 1288 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: audit 2023-10-09T09:09:43.146426+0000 mon.a (mon.0) 1289 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[75179]: audit 2023-10-09T09:09:43.153880+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: cluster 2023-10-09T09:09:41.984619+0000 mgr.y (mgr.25421) 970 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 6003/155763 objects degraded (3.854%); 36277/155763 objects misplaced (23.290%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:09:43.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: cluster 2023-10-09T09:09:42.300328+0000 mon.a (mon.0) 1285 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: cluster 2023-10-09T09:09:42.318272+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: cluster 2023-10-09T09:09:42.321306+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: audit 2023-10-09T09:09:42.324277+0000 mon.a (mon.0) 1286 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: cluster 2023-10-09T09:09:42.324508+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e228: 8 total, 8 up, 6 in 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: audit 2023-10-09T09:09:42.654091+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.16:0/3025189797' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: audit 2023-10-09T09:09:42.654722+0000 mon.a (mon.0) 1288 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: audit 2023-10-09T09:09:43.146426+0000 mon.a (mon.0) 1289 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:43.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:43 smithi016 bash[77892]: audit 2023-10-09T09:09:43.153880+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:43.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:43.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: cluster 2023-10-09T09:09:41.984619+0000 mgr.y (mgr.25421) 970 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 78 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 6003/155763 objects degraded (3.854%); 36277/155763 objects misplaced (23.290%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: cluster 2023-10-09T09:09:42.300328+0000 mon.a (mon.0) 1285 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: cluster 2023-10-09T09:09:42.318272+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: cluster 2023-10-09T09:09:42.321306+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: audit 2023-10-09T09:09:42.324277+0000 mon.a (mon.0) 1286 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: cluster 2023-10-09T09:09:42.324508+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e228: 8 total, 8 up, 6 in 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: audit 2023-10-09T09:09:42.654091+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.16:0/3025189797' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: audit 2023-10-09T09:09:42.654722+0000 mon.a (mon.0) 1288 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:09:43.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: audit 2023-10-09T09:09:43.146426+0000 mon.a (mon.0) 1289 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:43.743 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:43 smithi094 bash[59677]: audit 2023-10-09T09:09:43.153880+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:43.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:43.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:43.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:43.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:43.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:43.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:44.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:44.058 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:44.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:44.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:44.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:44.290 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:44.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:44.421 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:44.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:44.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[77892]: cluster 2023-10-09T09:09:43.321640+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T09:09:44.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[77892]: audit 2023-10-09T09:09:43.323271+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T09:09:44.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[77892]: cluster 2023-10-09T09:09:43.323351+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-09T09:09:44.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[77892]: audit 2023-10-09T09:09:43.675930+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:44.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[77892]: audit 2023-10-09T09:09:43.676895+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:44.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[77892]: audit 2023-10-09T09:09:43.683258+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:44.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[75179]: cluster 2023-10-09T09:09:43.321640+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T09:09:44.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[75179]: audit 2023-10-09T09:09:43.323271+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T09:09:44.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[75179]: cluster 2023-10-09T09:09:43.323351+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-09T09:09:44.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[75179]: audit 2023-10-09T09:09:43.675930+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:44.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[75179]: audit 2023-10-09T09:09:43.676895+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:44.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:44 smithi016 bash[75179]: audit 2023-10-09T09:09:43.683258+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:44.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:44.606 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:44 smithi094 bash[59677]: cluster 2023-10-09T09:09:43.321640+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T09:09:44.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:44 smithi094 bash[59677]: audit 2023-10-09T09:09:43.323271+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T09:09:44.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:44 smithi094 bash[59677]: cluster 2023-10-09T09:09:43.323351+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-09T09:09:44.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:44 smithi094 bash[59677]: audit 2023-10-09T09:09:43.675930+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:44.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:44 smithi094 bash[59677]: audit 2023-10-09T09:09:43.676895+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:44.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:44 smithi094 bash[59677]: audit 2023-10-09T09:09:43.683258+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:44.623 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:44.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:44.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:44.746 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:44.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:44.849 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:44.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:44.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:45.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:45.109 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:45.139 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:45.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:45.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:45.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:45.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:45.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:45.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:43.985410+0000 mgr.y (mgr.25421) 971 : cluster 0 pgmap v1025: 89 pgs: 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 680 op/s; 6003/150456 objects degraded (3.990%); 28723/150456 objects misplaced (19.091%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:09:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.330569+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.331089+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.333174+0000 mon.a (mon.0) 1296 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-09T09:09:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.333179+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.333305+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.335040+0000 osd.2 (osd.2) 5 : cluster 1 9.a continuing backfill to osd.5 from (218'1200,228'8915] 9:501cfc93:::benchmark_data_smithi016_172771_object69144:head to 228'8915 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.335097+0000 osd.2 (osd.2) 6 : cluster 0 9.a starting backfill to osd.7 from (0'0,0'0] MAX to 228'8915 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.335198+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.336141+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.336548+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.336652+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.337222+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.338295+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.341681+0000 osd.2 (osd.2) 7 : cluster 1 9.7 continuing backfill to osd.5 from (218'1100,228'8761] MIN to 228'8761 2023-10-09T09:09:45.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.341740+0000 osd.2 (osd.2) 8 : cluster 0 9.7 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8761 2023-10-09T09:09:45.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.353497+0000 osd.0 (osd.0) 14 : cluster 0 9.9 starting backfill to osd.5 from (218'2900,228'8946] MIN to 228'8948 2023-10-09T09:09:45.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.353707+0000 osd.3 (osd.3) 7 : cluster 0 9.3 starting backfill to osd.5 from (219'3500,228'9083] MIN to 228'9084 2023-10-09T09:09:45.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.384676+0000 osd.3 (osd.3) 8 : cluster 0 9.3 starting backfill to osd.7 from (0'0,0'0] MAX to 228'9084 2023-10-09T09:09:45.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.401188+0000 osd.0 (osd.0) 15 : cluster 0 9.9 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8948 2023-10-09T09:09:45.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:44.639543+0000 mon.a (mon.0) 1297 : cluster 3 Health check update: Degraded data redundancy: 6003/155763 objects degraded (3.854%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:45.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:45 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.337968+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-09T09:09:45.501 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:45.531 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:45.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:45.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:45.661 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:45.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:45.741 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:09:45] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.33.4" 2023-10-09T09:09:45.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:43.985410+0000 mgr.y (mgr.25421) 971 : cluster 0 pgmap v1025: 89 pgs: 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 680 op/s; 6003/150456 objects degraded (3.990%); 28723/150456 objects misplaced (19.091%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:09:45.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.330569+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.331089+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.333174+0000 mon.a (mon.0) 1296 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-09T09:09:45.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.333179+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.333305+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.335040+0000 osd.2 (osd.2) 5 : cluster 1 9.a continuing backfill to osd.5 from (218'1200,228'8915] 9:501cfc93:::benchmark_data_smithi016_172771_object69144:head to 228'8915 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.335097+0000 osd.2 (osd.2) 6 : cluster 0 9.a starting backfill to osd.7 from (0'0,0'0] MAX to 228'8915 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.335198+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.336141+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.336548+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.336652+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.337222+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.338295+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.341681+0000 osd.2 (osd.2) 7 : cluster 1 9.7 continuing backfill to osd.5 from (218'1100,228'8761] MIN to 228'8761 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.341740+0000 osd.2 (osd.2) 8 : cluster 0 9.7 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8761 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.353497+0000 osd.0 (osd.0) 14 : cluster 0 9.9 starting backfill to osd.5 from (218'2900,228'8946] MIN to 228'8948 2023-10-09T09:09:45.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.353707+0000 osd.3 (osd.3) 7 : cluster 0 9.3 starting backfill to osd.5 from (219'3500,228'9083] MIN to 228'9084 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.384676+0000 osd.3 (osd.3) 8 : cluster 0 9.3 starting backfill to osd.7 from (0'0,0'0] MAX to 228'9084 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.401188+0000 osd.0 (osd.0) 15 : cluster 0 9.9 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8948 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:44.639543+0000 mon.a (mon.0) 1297 : cluster 3 Health check update: Degraded data redundancy: 6003/155763 objects degraded (3.854%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.337968+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:43.985410+0000 mgr.y (mgr.25421) 971 : cluster 0 pgmap v1025: 89 pgs: 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 680 op/s; 6003/150456 objects degraded (3.990%); 28723/150456 objects misplaced (19.091%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.330569+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.331089+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.333174+0000 mon.a (mon.0) 1296 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.333179+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.333305+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.335040+0000 osd.2 (osd.2) 5 : cluster 1 9.a continuing backfill to osd.5 from (218'1200,228'8915] 9:501cfc93:::benchmark_data_smithi016_172771_object69144:head to 228'8915 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.335097+0000 osd.2 (osd.2) 6 : cluster 0 9.a starting backfill to osd.7 from (0'0,0'0] MAX to 228'8915 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.335198+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.336141+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.336548+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.336652+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.337222+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.338295+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.341681+0000 osd.2 (osd.2) 7 : cluster 1 9.7 continuing backfill to osd.5 from (218'1100,228'8761] MIN to 228'8761 2023-10-09T09:09:45.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.341740+0000 osd.2 (osd.2) 8 : cluster 0 9.7 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8761 2023-10-09T09:09:45.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.353497+0000 osd.0 (osd.0) 14 : cluster 0 9.9 starting backfill to osd.5 from (218'2900,228'8946] MIN to 228'8948 2023-10-09T09:09:45.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.353707+0000 osd.3 (osd.3) 7 : cluster 0 9.3 starting backfill to osd.5 from (219'3500,228'9083] MIN to 228'9084 2023-10-09T09:09:45.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.384676+0000 osd.3 (osd.3) 8 : cluster 0 9.3 starting backfill to osd.7 from (0'0,0'0] MAX to 228'9084 2023-10-09T09:09:45.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.401188+0000 osd.0 (osd.0) 15 : cluster 0 9.9 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8948 2023-10-09T09:09:45.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:44.639543+0000 mon.a (mon.0) 1297 : cluster 3 Health check update: Degraded data redundancy: 6003/155763 objects degraded (3.854%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:09:45.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:45 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.337968+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-09T09:09:45.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:45.796 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:45.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:45.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:45.908 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:09:45.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:45.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:46.030 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:46.072 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:46.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:46.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:46.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:46.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:46.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:46.317 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:46.338 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:46.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:46.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:46.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:46.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:46.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:46.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.333557+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.333660+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.337777+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.339851+0000 osd.2 (osd.2) 9 : cluster 0 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8916 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.339979+0000 osd.6 (osd.6) 168 : cluster 0 9.1 starting backfill to osd.7 from (0'0,0'0] MAX to 228'9082 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.343420+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.344095+0000 osd.0 (osd.0) 16 : cluster 0 9.d starting backfill to osd.7 from (0'0,0'0] MAX to 228'8795 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.349081+0000 osd.4 (osd.4) 188 : cluster 0 9.c starting backfill to osd.7 from (0'0,0'0] MAX to 228'8752 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.350229+0000 osd.4 (osd.4) 189 : cluster 0 9.e starting backfill to osd.7 from (0'0,0'0] MAX to 228'9024 2023-10-09T09:09:46.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.368962+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.333557+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.333660+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.337777+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.339851+0000 osd.2 (osd.2) 9 : cluster 0 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8916 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.339979+0000 osd.6 (osd.6) 168 : cluster 0 9.1 starting backfill to osd.7 from (0'0,0'0] MAX to 228'9082 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.343420+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.344095+0000 osd.0 (osd.0) 16 : cluster 0 9.d starting backfill to osd.7 from (0'0,0'0] MAX to 228'8795 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.349081+0000 osd.4 (osd.4) 188 : cluster 0 9.c starting backfill to osd.7 from (0'0,0'0] MAX to 228'8752 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.350229+0000 osd.4 (osd.4) 189 : cluster 0 9.e starting backfill to osd.7 from (0'0,0'0] MAX to 228'9024 2023-10-09T09:09:46.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:46 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.368962+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:46.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:46.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:46.685 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T09:09:46.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:46.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:46.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:46.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:46.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:46.871 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.333557+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.871 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.333660+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.871 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.337777+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.871 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.339851+0000 osd.2 (osd.2) 9 : cluster 0 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 228'8916 2023-10-09T09:09:46.871 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.339979+0000 osd.6 (osd.6) 168 : cluster 0 9.1 starting backfill to osd.7 from (0'0,0'0] MAX to 228'9082 2023-10-09T09:09:46.871 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.343420+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.872 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.344095+0000 osd.0 (osd.0) 16 : cluster 0 9.d starting backfill to osd.7 from (0'0,0'0] MAX to 228'8795 2023-10-09T09:09:46.872 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.349081+0000 osd.4 (osd.4) 188 : cluster 0 9.c starting backfill to osd.7 from (0'0,0'0] MAX to 228'8752 2023-10-09T09:09:46.872 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.350229+0000 osd.4 (osd.4) 189 : cluster 0 9.e starting backfill to osd.7 from (0'0,0'0] MAX to 228'9024 2023-10-09T09:09:46.872 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:46 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.368962+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T09:09:46.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:46.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:46.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:47.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:47.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:47.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:47.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:47.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:47.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:47.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:47.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:47.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:47.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:47 smithi094 bash[59677]: cluster 2023-10-09T09:09:45.986009+0000 mgr.y (mgr.25421) 972 : cluster 0 pgmap v1028: 89 pgs: 1 activating+degraded+remapped, 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+remapped+backfill_wait, 75 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 3084/149634 objects degraded (2.061%); 31829/149634 objects misplaced (21.271%); 1.7 MiB/s, 2 keys/s, 26 objects/s recovering 2023-10-09T09:09:47.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:47.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:47.677 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:47.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:47.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:47 smithi016 bash[75179]: cluster 2023-10-09T09:09:45.986009+0000 mgr.y (mgr.25421) 972 : cluster 0 pgmap v1028: 89 pgs: 1 activating+degraded+remapped, 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+remapped+backfill_wait, 75 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 3084/149634 objects degraded (2.061%); 31829/149634 objects misplaced (21.271%); 1.7 MiB/s, 2 keys/s, 26 objects/s recovering 2023-10-09T09:09:47.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:47 smithi016 bash[77892]: cluster 2023-10-09T09:09:45.986009+0000 mgr.y (mgr.25421) 972 : cluster 0 pgmap v1028: 89 pgs: 1 activating+degraded+remapped, 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+remapped+backfill_wait, 75 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 3084/149634 objects degraded (2.061%); 31829/149634 objects misplaced (21.271%); 1.7 MiB/s, 2 keys/s, 26 objects/s recovering 2023-10-09T09:09:47.813 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:47.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:47.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:47.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:47.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:47.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:48.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:48.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:48.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:48.232 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:48.261 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:48.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:48.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:48.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:48.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:48.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:48.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:48.504 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:48.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:48.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:48.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:09:48.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:48.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:09:48.670 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:48.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:48.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:48.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:48.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:48.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:48.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:48.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:48.951 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:48.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:49.056 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:49.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:49.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:49.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:49.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:49.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:49.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:49.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:49.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:49.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:49.420 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:49 smithi094 bash[59677]: cluster 2023-10-09T09:09:47.986623+0000 mgr.y (mgr.25421) 973 : cluster 0 pgmap v1029: 89 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 3084/148830 objects degraded (2.072%); 37911/148830 objects misplaced (25.473%); 2.3 MiB/s, 1 keys/s, 35 objects/s recovering 2023-10-09T09:09:49.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:49 smithi094 bash[59677]: audit 2023-10-09T09:09:49.245490+0000 mon.a (mon.0) 1299 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:49.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:49 smithi094 bash[59677]: audit 2023-10-09T09:09:49.247243+0000 mon.a (mon.0) 1300 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:49.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:49.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:49.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:49.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:49.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:49.597 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:49 smithi016 bash[75179]: cluster 2023-10-09T09:09:47.986623+0000 mgr.y (mgr.25421) 973 : cluster 0 pgmap v1029: 89 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 3084/148830 objects degraded (2.072%); 37911/148830 objects misplaced (25.473%); 2.3 MiB/s, 1 keys/s, 35 objects/s recovering 2023-10-09T09:09:49.597 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:49 smithi016 bash[75179]: audit 2023-10-09T09:09:49.245490+0000 mon.a (mon.0) 1299 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:49.597 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:49 smithi016 bash[75179]: audit 2023-10-09T09:09:49.247243+0000 mon.a (mon.0) 1300 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:49.597 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:49 smithi016 bash[77892]: cluster 2023-10-09T09:09:47.986623+0000 mgr.y (mgr.25421) 973 : cluster 0 pgmap v1029: 89 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 2 remapped+peering, 2 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 3084/148830 objects degraded (2.072%); 37911/148830 objects misplaced (25.473%); 2.3 MiB/s, 1 keys/s, 35 objects/s recovering 2023-10-09T09:09:49.597 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:49 smithi016 bash[77892]: audit 2023-10-09T09:09:49.245490+0000 mon.a (mon.0) 1299 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:49.597 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:49 smithi016 bash[77892]: audit 2023-10-09T09:09:49.247243+0000 mon.a (mon.0) 1300 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:49.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:49.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:49.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:49.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:49.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:49.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:49.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:49.900 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:49.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:50.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:50.030 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:50.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:50.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:50.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:50.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:50.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:50.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:50.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:50.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:50.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:50 smithi094 bash[59677]: cluster 2023-10-09T09:09:49.641373+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 3084/148830 objects degraded (2.072%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:50.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:50 smithi094 bash[59677]: audit 2023-10-09T09:09:50.014173+0000 mon.a (mon.0) 1302 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:50.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:50.573 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:50.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:50.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:50.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:50 smithi016 bash[75179]: cluster 2023-10-09T09:09:49.641373+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 3084/148830 objects degraded (2.072%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:50.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:50 smithi016 bash[75179]: audit 2023-10-09T09:09:50.014173+0000 mon.a (mon.0) 1302 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:50.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:50 smithi016 bash[77892]: cluster 2023-10-09T09:09:49.641373+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 3084/148830 objects degraded (2.072%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:09:50.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:50 smithi016 bash[77892]: audit 2023-10-09T09:09:50.014173+0000 mon.a (mon.0) 1302 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:09:50.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:50.809 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:50.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:50.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:50.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:50.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:50.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:51.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:51.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:51.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:51.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:51.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:51.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:51.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:51.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:51.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:51.498 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:51.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:51 smithi016 bash[75179]: cluster 2023-10-09T09:09:49.987354+0000 mgr.y (mgr.25421) 974 : cluster 0 pgmap v1030: 89 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped, 74 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 5980/146004 objects degraded (4.096%); 55175/146004 objects misplaced (37.790%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T09:09:51.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:51 smithi016 bash[77892]: cluster 2023-10-09T09:09:49.987354+0000 mgr.y (mgr.25421) 974 : cluster 0 pgmap v1030: 89 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped, 74 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 5980/146004 objects degraded (4.096%); 55175/146004 objects misplaced (37.790%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T09:09:51.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:51.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:51.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:51.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:51.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:51.686 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:09:51.686 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:09:51.686 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T09:09:51.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:09:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:51 smithi094 bash[59677]: cluster 2023-10-09T09:09:49.987354+0000 mgr.y (mgr.25421) 974 : cluster 0 pgmap v1030: 89 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped, 74 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 5980/146004 objects degraded (4.096%); 55175/146004 objects misplaced (37.790%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T09:09:51.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:51.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:51.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:51.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:51.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:51.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:52.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:52.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:52.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:52.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:52.140 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:09:52.140 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:09:52.140 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.140 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop_reply(client.45290.0:146828 9.e e231/230)", 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.108407+0000", 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.026337220000000001, 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.026355982999999999, 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.141 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.108407+0000", 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.108407+0000", 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.142 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.108406+0000", 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.4350000000000003e-06 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.108415+0000", 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6954999999999999e-05 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.108432+0000", 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.0649999999999999e-06 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.143 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.108440+0000", 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3056000000000003e-05 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop_reply(client.45290.0:146849 9.e e231/230)", 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.116051+0000", 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.018693075999999999, 2023-10-09T09:09:52.144 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.018746434999999999, 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.116051+0000", 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.116051+0000", 2023-10-09T09:09:52.145 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.116050+0000", 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.507e-06 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.116056+0000", 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4566000000000001e-05 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.146 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.116070+0000", 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.173e-06 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.116077+0000", 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5295000000000001e-05 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.147 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:146849 9.e 9:70f2580f:::benchmark_data_smithi016_172771_object50263:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.062902+0000", 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.071841996000000005, 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.071929646999999999, 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 146849 2023-10-09T09:09:52.148 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.062902+0000", 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.062902+0000", 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.149 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.062900+0000", 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0217999999999999e-05 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.062910+0000", 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.507e-05 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.150 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.062925+0000", 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.6170000000000007e-06 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.062934+0000", 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00085805799999999998 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.063792+0000", 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0034062889999999998 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.151 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.067198+0000", 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.043394480999999999 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.110593+0000", 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000107414 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:09:52.152 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.110700+0000", 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000183819 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.110884+0000", 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.020694565000000002 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131579+0000", 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.068676329999999994 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.153 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop_reply(client.45290.0:146849 9.e e231/230)", 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.118260+0000", 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.016484137999999999, 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.016627376999999999, 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.154 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.118260+0000", 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.118260+0000", 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.118260+0000", 2023-10-09T09:09:52.155 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.2309999999999998e-06 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.118264+0000", 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1009999999999992e-06 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.118272+0000", 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5790000000000003e-06 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.156 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.118278+0000", 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7377e-05 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:146872 9.e 9.faa71cbe (undecoded) ondisk+write+known_if_redirected e231)", 2023-10-09T09:09:52.157 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.121866+0000", 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.012878608999999999, 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.013051775, 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 146872 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.158 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121866+0000", 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121866+0000", 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121864+0000", 2023-10-09T09:09:52.159 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5449999999999998e-06 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121872+0000", 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2055999999999999e-05 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121884+0000", 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.6980000000000001e-06 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.160 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121891+0000", 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.4879e-05 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop_reply(client.45290.0:146850 9.e e231/230)", 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.125879+0000", 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.008865688, 2023-10-09T09:09:52.161 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0090757419999999995, 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125879+0000", 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125879+0000", 2023-10-09T09:09:52.162 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125878+0000", 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0650000000000001e-06 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125885+0000", 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0026e-05 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.163 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125895+0000", 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7359999999999996e-06 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125905+0000", 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5789999999999999e-05 2023-10-09T09:09:52.165 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:146875 9.e 9.8f93712e (undecoded) ondisk+write+known_if_redirected e231)", 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.129669+0000", 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0050757689999999999, 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0053138869999999998, 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:09:52.166 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 146875 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.129669+0000", 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.167 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.168 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.129669+0000", 2023-10-09T09:09:52.168 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:09:52.168 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.168 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.129667+0000", 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7929999999999992e-06 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.129677+0000", 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1584e-05 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.169 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.129688+0000", 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.3159999999999997e-06 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.129697+0000", 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.8138999999999999e-05 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.170 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:146850 9.e 9:703179f0:::benchmark_data_smithi016_172771_object50264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.068507+0000", 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.066237780999999996, 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.066510214999999998, 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:09:52.171 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 146850 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.068507+0000", 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.172 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.068507+0000", 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.068504+0000", 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7680000000000003e-06 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.173 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.174 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.068514+0000", 2023-10-09T09:09:52.174 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8610999999999999e-05 2023-10-09T09:09:52.174 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.174 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.174 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.174 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.068533+0000", 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5989999999999996e-06 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.068540+0000", 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00056609 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.069106+0000", 2023-10-09T09:09:52.175 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00050358300000000005 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.069610+0000", 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.051864607 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121475+0000", 2023-10-09T09:09:52.176 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010296799999999999 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121577+0000", 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015489799999999999 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.177 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.121732+0000", 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0098563009999999996 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131589+0000", 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.063082077 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.178 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:146873 9.1 e231/230 9:8ec47164:::benchmark_data_smithi016_172771_object50287:head v 231'9282, mlcod=231'9278)", 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.131526+0000", 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0032182830000000002, 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0035464419999999999, 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.179 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131526+0000", 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131526+0000", 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.180 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131525+0000", 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2264e-05 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131537+0000", 2023-10-09T09:09:52.181 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3396e-05 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131550+0000", 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.2439999999999993e-06 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.182 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131559+0000", 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000144294 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131703+0000", 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012993999999999999 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.183 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131833+0000", 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00030673899999999999 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:146876 9.f e231/222 9:f9473cd3:::benchmark_data_smithi016_172771_object50290:head v 231'9333, mlcod=231'9331)", 2023-10-09T09:09:52.184 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.132914+0000", 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0018303600000000001, 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0021936920000000001, 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.185 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.132914+0000", 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.132914+0000", 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.186 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.132913+0000", 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.2700000000000004e-06 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.132921+0000", 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5038000000000001e-05 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.187 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.132936+0000", 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.2710000000000003e-06 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.132944+0000", 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5697000000000005e-05 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.188 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.133009+0000", 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.2017999999999997e-05 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.133041+0000", 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000127298 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.189 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:146856 9.e 9:7ea11ef2:::benchmark_data_smithi016_172771_object50270:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.078268+0000", 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.056476180000000001, 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.056875583, 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.190 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 146856 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.078268+0000", 2023-10-09T09:09:52.191 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.078268+0000", 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.192 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.078266+0000", 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.3499999999999997e-06 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.078275+0000", 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7288000000000001e-05 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.193 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.078292+0000", 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.584e-06 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.078299+0000", 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.8068000000000001e-05 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.194 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.078347+0000", 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0047701949999999996 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.083117+0000", 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.048648607000000003 2023-10-09T09:09:52.195 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131766+0000", 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000171003 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.131937+0000", 2023-10-09T09:09:52.196 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00027944899999999997 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.132216+0000", 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.053947790000000002 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.197 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:146828 9.e 9:7c05ca45:::benchmark_data_smithi016_172771_object50242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.026961+0000", 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.107782976, 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.108244753, 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:09:52.198 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 146828 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.026961+0000", 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.199 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.026961+0000", 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.026960+0000", 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.02e-06 2023-10-09T09:09:52.200 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.026969+0000", 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6348999999999999e-05 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.026985+0000", 2023-10-09T09:09:52.201 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.6929999999999996e-06 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.026994+0000", 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.036096862 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.202 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.063090+0000", 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00060539600000000002 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.063696+0000", 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024682098999999999 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.203 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.088378+0000", 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000288413 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.088666+0000", 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00078981799999999997 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.204 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.089456+0000", 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.036115603000000003 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125572+0000", 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010733799999999999 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.205 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.125679+0000", 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.098717732000000002 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop_reply(client.45290.0:146828 9.e e231/230)", 2023-10-09T09:09:52.206 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.096247+0000", 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.038497165999999999, 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.039018679000000001, 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.096247+0000", 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:09:52.207 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.096247+0000", 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.096246+0000", 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.1790000000000002e-06 2023-10-09T09:09:52.208 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.096253+0000", 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3638000000000001e-05 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.096267+0000", 2023-10-09T09:09:52.209 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6229999999999997e-06 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.096275+0000", 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7480000000000001e-05 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.210 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:09:52.211 INFO:teuthology.orchestra.run.smithi094.stdout: ], 2023-10-09T09:09:52.211 INFO:teuthology.orchestra.run.smithi094.stdout: "num_ops": 13 2023-10-09T09:09:52.211 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:09:52.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:52.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:52.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:52.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:52.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:52.381 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:52.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:52.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:52.486 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:52.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:52.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:52.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:52.658 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:52.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:52.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:52.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:52.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:52.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:52.902 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:52.931 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:52.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:52.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:53.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:53.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:53.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:53.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:53.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:53.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:53.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:09:53.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:53.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:53.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:53.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:53.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:53.467 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:09:53.468 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:09:53.470 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:53.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:53.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:53.584 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:53 smithi094 bash[59677]: cluster 2023-10-09T09:09:51.988157+0000 mgr.y (mgr.25421) 975 : cluster 0 pgmap v1031: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 5837/143319 objects degraded (4.073%); 53878/143319 objects misplaced (37.593%); 9.8 MiB/s, 1 keys/s, 157 objects/s recovering 2023-10-09T09:09:53.612 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:53.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:53.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:53.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:53.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:53 smithi016 bash[75179]: cluster 2023-10-09T09:09:51.988157+0000 mgr.y (mgr.25421) 975 : cluster 0 pgmap v1031: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 5837/143319 objects degraded (4.073%); 53878/143319 objects misplaced (37.593%); 9.8 MiB/s, 1 keys/s, 157 objects/s recovering 2023-10-09T09:09:53.724 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:53 smithi016 bash[77892]: cluster 2023-10-09T09:09:51.988157+0000 mgr.y (mgr.25421) 975 : cluster 0 pgmap v1031: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 5837/143319 objects degraded (4.073%); 53878/143319 objects misplaced (37.593%); 9.8 MiB/s, 1 keys/s, 157 objects/s recovering 2023-10-09T09:09:53.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:53.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:53.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:53.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:53.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:53.982 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:54.012 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:54.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:54.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:54.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:54.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:54.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:54.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:54.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:54.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:54.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:54.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:54.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:54.679 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:54.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:54.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:54.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:54.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:54.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:54.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:54.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:54.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:55.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:55.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:55.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:55.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:55.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:55.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:55.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:55.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:55.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:55.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:55.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:55.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:55 smithi016 bash[75179]: cluster 2023-10-09T09:09:53.988954+0000 mgr.y (mgr.25421) 976 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 5692/138456 objects degraded (4.111%); 51839/138456 objects misplaced (37.441%); 9.6 MiB/s, 1 keys/s, 153 objects/s recovering 2023-10-09T09:09:55.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:55 smithi016 bash[75179]: cluster 2023-10-09T09:09:54.642660+0000 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 5692/138456 objects degraded (4.111%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:09:55.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:55 smithi016 bash[77892]: cluster 2023-10-09T09:09:53.988954+0000 mgr.y (mgr.25421) 976 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 5692/138456 objects degraded (4.111%); 51839/138456 objects misplaced (37.441%); 9.6 MiB/s, 1 keys/s, 153 objects/s recovering 2023-10-09T09:09:55.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:55 smithi016 bash[77892]: cluster 2023-10-09T09:09:54.642660+0000 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 5692/138456 objects degraded (4.111%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:09:55.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:55.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:55.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:55.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:55.649 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:55 smithi094 bash[59677]: cluster 2023-10-09T09:09:53.988954+0000 mgr.y (mgr.25421) 976 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 5692/138456 objects degraded (4.111%); 51839/138456 objects misplaced (37.441%); 9.6 MiB/s, 1 keys/s, 153 objects/s recovering 2023-10-09T09:09:55.649 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:55 smithi094 bash[59677]: cluster 2023-10-09T09:09:54.642660+0000 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 5692/138456 objects degraded (4.111%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:09:55.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:55.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:55.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:55.801 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:09:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:09:55] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.33.4" 2023-10-09T09:09:55.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:55.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:55.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:55.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:55.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:56.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:56.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:56.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:56.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:56.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:56.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:56.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:56.340 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:56.377 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:56.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:56.480 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:56.515 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:09:56.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:56.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:56.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:56.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:56.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:56.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:56.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:56.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:56.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:56.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:56.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:56.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:56.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:56.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:57.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:57.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:57.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:57.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:57.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:57.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:57.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:57.298 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:57.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:09:57.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:57.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:09:57.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:57 smithi016 bash[75179]: cluster 2023-10-09T09:09:55.989615+0000 mgr.y (mgr.25421) 977 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 5540/137001 objects degraded (4.044%); 51673/137001 objects misplaced (37.717%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T09:09:57.374 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:57 smithi016 bash[77892]: cluster 2023-10-09T09:09:55.989615+0000 mgr.y (mgr.25421) 977 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 5540/137001 objects degraded (4.044%); 51673/137001 objects misplaced (37.717%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T09:09:57.430 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:57.461 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:57.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:57.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:57.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:57.573 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:57 smithi094 bash[59677]: cluster 2023-10-09T09:09:55.989615+0000 mgr.y (mgr.25421) 977 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 5540/137001 objects degraded (4.044%); 51673/137001 objects misplaced (37.717%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T09:09:57.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:09:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:57.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:09:57.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:57.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:57.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:57.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:57.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:57.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:57.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:57.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:57.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:58.016 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:58.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:58.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:58.127 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:09:58.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:58.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:58.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:58.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:58.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:58.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:09:58.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:09:58.511 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:58.556 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:58.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:58.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:58.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:58.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:58.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:58.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:58.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:58.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:58.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:59.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:59.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:59.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:59.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:59.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:09:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:59.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:09:59.311 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:59.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:09:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:59.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:09:59.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:59.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:09:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:59.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:09:59.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:09:59 smithi016 bash[75179]: cluster 2023-10-09T09:09:57.990238+0000 mgr.y (mgr.25421) 978 : cluster 0 pgmap v1034: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 5540/135519 objects degraded (4.088%); 50582/135519 objects misplaced (37.325%); 12 MiB/s, 191 objects/s recovering 2023-10-09T09:09:59.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:09:59 smithi016 bash[77892]: cluster 2023-10-09T09:09:57.990238+0000 mgr.y (mgr.25421) 978 : cluster 0 pgmap v1034: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 5540/135519 objects degraded (4.088%); 50582/135519 objects misplaced (37.325%); 12 MiB/s, 191 objects/s recovering 2023-10-09T09:09:59.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:59.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:59.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:59.620 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:09:59 smithi094 bash[59677]: cluster 2023-10-09T09:09:57.990238+0000 mgr.y (mgr.25421) 978 : cluster 0 pgmap v1034: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 5540/135519 objects degraded (4.088%); 50582/135519 objects misplaced (37.325%); 12 MiB/s, 191 objects/s recovering 2023-10-09T09:09:59.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:59.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:09:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:59.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:09:59.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:59.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:09:59.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:09:59.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:09:59.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:00.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:00.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:00.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:00.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:00.300 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:00.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:00.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:00.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[75179]: cluster 2023-10-09T09:09:59.643523+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:00.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[75179]: cluster 2023-10-09T09:10:00.000118+0000 mon.a (mon.0) 1305 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded 2023-10-09T09:10:00.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[75179]: cluster 2023-10-09T09:10:00.000156+0000 mon.a (mon.0) 1306 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T09:10:00.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[75179]: cluster 2023-10-09T09:10:00.000170+0000 mon.a (mon.0) 1307 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded 2023-10-09T09:10:00.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[75179]: cluster 2023-10-09T09:10:00.000183+0000 mon.a (mon.0) 1308 : cluster 3 pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T09:10:00.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[75179]: cluster 2023-10-09T09:10:00.000196+0000 mon.a (mon.0) 1309 : cluster 3 pg 9.d is active+recovery_wait+degraded+remapped, acting [0,5,4] 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[75179]: cluster 2023-10-09T09:10:00.000209+0000 mon.a (mon.0) 1310 : cluster 3 pg 9.e is active+recovery_wait+degraded+remapped, acting [4,0,5] 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[77892]: cluster 2023-10-09T09:09:59.643523+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[77892]: cluster 2023-10-09T09:10:00.000118+0000 mon.a (mon.0) 1305 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[77892]: cluster 2023-10-09T09:10:00.000156+0000 mon.a (mon.0) 1306 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[77892]: cluster 2023-10-09T09:10:00.000170+0000 mon.a (mon.0) 1307 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[77892]: cluster 2023-10-09T09:10:00.000183+0000 mon.a (mon.0) 1308 : cluster 3 pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[77892]: cluster 2023-10-09T09:10:00.000196+0000 mon.a (mon.0) 1309 : cluster 3 pg 9.d is active+recovery_wait+degraded+remapped, acting [0,5,4] 2023-10-09T09:10:00.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:00 smithi016 bash[77892]: cluster 2023-10-09T09:10:00.000209+0000 mon.a (mon.0) 1310 : cluster 3 pg 9.e is active+recovery_wait+degraded+remapped, acting [4,0,5] 2023-10-09T09:10:00.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:00.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:00.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:00 smithi094 bash[59677]: cluster 2023-10-09T09:09:59.643523+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:00.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:00 smithi094 bash[59677]: cluster 2023-10-09T09:10:00.000118+0000 mon.a (mon.0) 1305 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded 2023-10-09T09:10:00.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:00 smithi094 bash[59677]: cluster 2023-10-09T09:10:00.000156+0000 mon.a (mon.0) 1306 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T09:10:00.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:00 smithi094 bash[59677]: cluster 2023-10-09T09:10:00.000170+0000 mon.a (mon.0) 1307 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 5540/135519 objects degraded (4.088%), 3 pgs degraded 2023-10-09T09:10:00.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:00 smithi094 bash[59677]: cluster 2023-10-09T09:10:00.000183+0000 mon.a (mon.0) 1308 : cluster 3 pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T09:10:00.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:00 smithi094 bash[59677]: cluster 2023-10-09T09:10:00.000196+0000 mon.a (mon.0) 1309 : cluster 3 pg 9.d is active+recovery_wait+degraded+remapped, acting [0,5,4] 2023-10-09T09:10:00.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:00 smithi094 bash[59677]: cluster 2023-10-09T09:10:00.000209+0000 mon.a (mon.0) 1310 : cluster 3 pg 9.e is active+recovery_wait+degraded+remapped, acting [4,0,5] 2023-10-09T09:10:00.573 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:00.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:00.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:00.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:00.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:00.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:00.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:00.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:00.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:00.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:00.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:00.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:01.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:01.134 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:01.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:01.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:01.270 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:01.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:01.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:01.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:01.385 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:01 smithi016 bash[75179]: cluster 2023-10-09T09:09:59.990937+0000 mgr.y (mgr.25421) 979 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 5388/130887 objects degraded (4.117%); 48696/130887 objects misplaced (37.205%); 13 MiB/s, 203 objects/s recovering 2023-10-09T09:10:01.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:01.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:01.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:01.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:01.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:01.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:01.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:01.653 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:01 smithi094 bash[59677]: cluster 2023-10-09T09:09:59.990937+0000 mgr.y (mgr.25421) 979 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 5388/130887 objects degraded (4.117%); 48696/130887 objects misplaced (37.205%); 13 MiB/s, 203 objects/s recovering 2023-10-09T09:10:01.675 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:01.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:01.747 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:01 smithi016 bash[77892]: cluster 2023-10-09T09:09:59.990937+0000 mgr.y (mgr.25421) 979 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 5388/130887 objects degraded (4.117%); 48696/130887 objects misplaced (37.205%); 13 MiB/s, 203 objects/s recovering 2023-10-09T09:10:01.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:01.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:01.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:01.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:01.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:01.938 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:01.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:01.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:02.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:02.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:02.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:02.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:02.219 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:02.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:02.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:02.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:02.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:02.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:02.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:02.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:02.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:02.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:02.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:02.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:02.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:02.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:02.735 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:02.761 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:02.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:02.867 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:02.899 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:02.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:03.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:03.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:03.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:03.139 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:03.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:03.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:03.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:03.310 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:03.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:03.391 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:03 smithi016 bash[75179]: cluster 2023-10-09T09:10:01.991747+0000 mgr.y (mgr.25421) 980 : cluster 0 pgmap v1036: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 5200/128067 objects degraded (4.060%); 47472/128067 objects misplaced (37.068%); 16 MiB/s, 253 objects/s recovering 2023-10-09T09:10:03.416 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:03.424 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:03 smithi094 bash[59677]: cluster 2023-10-09T09:10:01.991747+0000 mgr.y (mgr.25421) 980 : cluster 0 pgmap v1036: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 5200/128067 objects degraded (4.060%); 47472/128067 objects misplaced (37.068%); 16 MiB/s, 253 objects/s recovering 2023-10-09T09:10:03.457 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:03.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:03.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:03.561 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:03.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:03.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:03.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:03.705 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:03 smithi016 bash[77892]: cluster 2023-10-09T09:10:01.991747+0000 mgr.y (mgr.25421) 980 : cluster 0 pgmap v1036: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 5200/128067 objects degraded (4.060%); 47472/128067 objects misplaced (37.068%); 16 MiB/s, 253 objects/s recovering 2023-10-09T09:10:03.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:03.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:03.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:03.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:03.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:03.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:04.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:04.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:04.075 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:10:04.076 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:10:04.076 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:10:04.076 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:10:04.076 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:10:04.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:04.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:04.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:04.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:04.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:04.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:04.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:04.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:04.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:04.559 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:04.561 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:04 smithi094 bash[59677]: audit 2023-10-09T09:10:04.242090+0000 mon.a (mon.0) 1311 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:04.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:04.671 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:04 smithi016 bash[77892]: audit 2023-10-09T09:10:04.242090+0000 mon.a (mon.0) 1311 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:04.672 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:04 smithi016 bash[75179]: audit 2023-10-09T09:10:04.242090+0000 mon.a (mon.0) 1311 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:04.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:04.797 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:04.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:04.936 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:04.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:04.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:04.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:05.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:05.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:05.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:05.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:05.267 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:05.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:05.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:05.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:10:05.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:05.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:05.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:05.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:05 smithi094 bash[59677]: cluster 2023-10-09T09:10:03.992459+0000 mgr.y (mgr.25421) 981 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 5040/123381 objects degraded (4.085%); 45529/123381 objects misplaced (36.901%); 13 MiB/s, 201 objects/s recovering 2023-10-09T09:10:05.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:05 smithi094 bash[59677]: cluster 2023-10-09T09:10:04.644731+0000 mon.a (mon.0) 1312 : cluster 3 Health check update: Degraded data redundancy: 5040/123381 objects degraded (4.085%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:05.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:05.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:05.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:05.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:05.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:05.668 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:05 smithi016 bash[75179]: cluster 2023-10-09T09:10:03.992459+0000 mgr.y (mgr.25421) 981 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 5040/123381 objects degraded (4.085%); 45529/123381 objects misplaced (36.901%); 13 MiB/s, 201 objects/s recovering 2023-10-09T09:10:05.669 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:05 smithi016 bash[75179]: cluster 2023-10-09T09:10:04.644731+0000 mon.a (mon.0) 1312 : cluster 3 Health check update: Degraded data redundancy: 5040/123381 objects degraded (4.085%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:05.669 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:05 smithi016 bash[77892]: cluster 2023-10-09T09:10:03.992459+0000 mgr.y (mgr.25421) 981 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 5040/123381 objects degraded (4.085%); 45529/123381 objects misplaced (36.901%); 13 MiB/s, 201 objects/s recovering 2023-10-09T09:10:05.669 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:05 smithi016 bash[77892]: cluster 2023-10-09T09:10:04.644731+0000 mon.a (mon.0) 1312 : cluster 3 Health check update: Degraded data redundancy: 5040/123381 objects degraded (4.085%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:05.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:05.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:05.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:05.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:05.821 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:05.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:05.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:05.933 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:10:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:10:05] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.33.4" 2023-10-09T09:10:05.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:05.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:05.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:06.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:06.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:06.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:06.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:06.227 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:06.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:06.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:06.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:06.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:06.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:06.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:06.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:06.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:06.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:06.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:06.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:06.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:06.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:06.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:06.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:06.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:06.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:06.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:06.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:07.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:07.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:07.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:07.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:07.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:07.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:07.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:07.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:07.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:07.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:07.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:07.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:07.549 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:07.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:07.653 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:07.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:07 smithi094 bash[59677]: cluster 2023-10-09T09:10:05.993176+0000 mgr.y (mgr.25421) 982 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 4886/121902 objects degraded (4.008%); 45358/121902 objects misplaced (37.209%); 12 MiB/s, 195 objects/s recovering 2023-10-09T09:10:07.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:07 smithi094 bash[59677]: cluster 2023-10-09T09:10:06.414284+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:07 smithi094 bash[59677]: cluster 2023-10-09T09:10:06.416315+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T09:10:07.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:07 smithi094 bash[59677]: cluster 2023-10-09T09:10:06.417407+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:07 smithi094 bash[59677]: cluster 2023-10-09T09:10:06.424796+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.691 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:07.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:07.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[75179]: cluster 2023-10-09T09:10:05.993176+0000 mgr.y (mgr.25421) 982 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 4886/121902 objects degraded (4.008%); 45358/121902 objects misplaced (37.209%); 12 MiB/s, 195 objects/s recovering 2023-10-09T09:10:07.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[75179]: cluster 2023-10-09T09:10:06.414284+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[75179]: cluster 2023-10-09T09:10:06.416315+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T09:10:07.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[75179]: cluster 2023-10-09T09:10:06.417407+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[75179]: cluster 2023-10-09T09:10:06.424796+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[77892]: cluster 2023-10-09T09:10:05.993176+0000 mgr.y (mgr.25421) 982 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 4886/121902 objects degraded (4.008%); 45358/121902 objects misplaced (37.209%); 12 MiB/s, 195 objects/s recovering 2023-10-09T09:10:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[77892]: cluster 2023-10-09T09:10:06.414284+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[77892]: cluster 2023-10-09T09:10:06.416315+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T09:10:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[77892]: cluster 2023-10-09T09:10:06.417407+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:07 smithi016 bash[77892]: cluster 2023-10-09T09:10:06.424796+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T09:10:07.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:07.823 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:07.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:07.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:07.951 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:07.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:08.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:08.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:08.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:08.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:08.190 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:08.220 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:08.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:08.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:08.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:08.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:08.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:08.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:08.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:08.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:08 smithi094 bash[59677]: cluster 2023-10-09T09:10:07.414795+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T09:10:08.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:08 smithi094 bash[59677]: cluster 2023-10-09T09:10:07.417920+0000 mon.a (mon.0) 1314 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T09:10:08.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:08 smithi094 bash[59677]: cluster 2023-10-09T09:10:07.420090+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T09:10:08.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:08.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:08.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:08.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:08.635 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:08.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:08.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:08.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:08.747 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:08 smithi016 bash[77892]: cluster 2023-10-09T09:10:07.414795+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T09:10:08.748 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:08 smithi016 bash[77892]: cluster 2023-10-09T09:10:07.417920+0000 mon.a (mon.0) 1314 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T09:10:08.748 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:08 smithi016 bash[77892]: cluster 2023-10-09T09:10:07.420090+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T09:10:08.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:08 smithi016 bash[75179]: cluster 2023-10-09T09:10:07.414795+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T09:10:08.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:08 smithi016 bash[75179]: cluster 2023-10-09T09:10:07.417920+0000 mon.a (mon.0) 1314 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T09:10:08.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:08 smithi016 bash[75179]: cluster 2023-10-09T09:10:07.420090+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T09:10:08.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:08.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:08.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:08.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:08.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:08.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:09.023 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:09.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:09.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:09.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:09.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:09.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:09.232 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:10:09.270 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:10:09.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:09.314 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:10:09.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:09.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:09.351 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:10:09.389 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:10:09.430 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:10:09.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:09.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:09.476 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:10:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:09.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:09.522 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.528 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:10:09.579 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:09.588 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:09 smithi094 bash[59677]: cluster 2023-10-09T09:10:07.993710+0000 mgr.y (mgr.25421) 983 : cluster 0 pgmap v1041: 89 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 4886/120390 objects degraded (4.058%); 44409/120390 objects misplaced (36.888%); 15 MiB/s, 237 objects/s recovering 2023-10-09T09:10:09.589 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:09 smithi094 bash[59677]: audit 2023-10-09T09:10:09.189277+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.16:0/3879825069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:09.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:09.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:09.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:09.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:09.724 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:09 smithi016 bash[77892]: cluster 2023-10-09T09:10:07.993710+0000 mgr.y (mgr.25421) 983 : cluster 0 pgmap v1041: 89 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 4886/120390 objects degraded (4.058%); 44409/120390 objects misplaced (36.888%); 15 MiB/s, 237 objects/s recovering 2023-10-09T09:10:09.724 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:09 smithi016 bash[77892]: audit 2023-10-09T09:10:09.189277+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.16:0/3879825069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:09.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:09 smithi016 bash[75179]: cluster 2023-10-09T09:10:07.993710+0000 mgr.y (mgr.25421) 983 : cluster 0 pgmap v1041: 89 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 4886/120390 objects degraded (4.058%); 44409/120390 objects misplaced (36.888%); 15 MiB/s, 237 objects/s recovering 2023-10-09T09:10:09.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:09 smithi016 bash[75179]: audit 2023-10-09T09:10:09.189277+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.16:0/3879825069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:09.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:09.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:09.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:09.901 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:09.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:09.957 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:10:09.970 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:10:10.005 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:10.037 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:10.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:10.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:10.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:10.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:10.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:10.181 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:10:10.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:10.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:10.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:10.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:10.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:10.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:10.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:10.576 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:10 smithi016 bash[75179]: cluster 2023-10-09T09:10:09.650130+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: Degraded data redundancy: 4886/120390 objects degraded (4.058%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:10.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:10 smithi016 bash[75179]: audit 2023-10-09T09:10:10.142871+0000 mon.a (mon.0) 1316 : audit 1 from='client.? 172.21.15.16:0/588166208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:10.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:10 smithi016 bash[77892]: cluster 2023-10-09T09:10:09.650130+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: Degraded data redundancy: 4886/120390 objects degraded (4.058%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:10.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:10 smithi016 bash[77892]: audit 2023-10-09T09:10:10.142871+0000 mon.a (mon.0) 1316 : audit 1 from='client.? 172.21.15.16:0/588166208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:10.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:10.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:10.665 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:10:10.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:10.712 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:10 smithi094 bash[59677]: cluster 2023-10-09T09:10:09.650130+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: Degraded data redundancy: 4886/120390 objects degraded (4.058%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:10.713 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:10 smithi094 bash[59677]: audit 2023-10-09T09:10:10.142871+0000 mon.a (mon.0) 1316 : audit 1 from='client.? 172.21.15.16:0/588166208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:10.734 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:10.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:10.834 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:10.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:10.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:10.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:10.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:11.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:11.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:11.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:11.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:11.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:11.294 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:11.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:11.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:11.527 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:11.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:11 smithi094 bash[59677]: audit 2023-10-09T09:10:09.938991+0000 mgr.y (mgr.25421) 984 : audit 0 from='client.45392 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:10:11.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:11 smithi094 bash[59677]: cluster 2023-10-09T09:10:09.994435+0000 mgr.y (mgr.25421) 985 : cluster 0 pgmap v1042: 89 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 4756/116145 objects degraded (4.095%); 42603/116145 objects misplaced (36.681%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T09:10:11.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:11 smithi094 bash[59677]: cluster 2023-10-09T09:10:10.652617+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:10:11.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:11 smithi094 bash[59677]: cluster 2023-10-09T09:10:10.654521+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T09:10:11.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:11 smithi094 bash[59677]: audit 2023-10-09T09:10:10.658723+0000 mon.a (mon.0) 1318 : audit 1 from='client.? 172.21.15.16:0/588166208' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:10:11.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:11 smithi094 bash[59677]: cluster 2023-10-09T09:10:10.658943+0000 mon.a (mon.0) 1319 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T09:10:11.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:11.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:11.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[75179]: audit 2023-10-09T09:10:09.938991+0000 mgr.y (mgr.25421) 984 : audit 0 from='client.45392 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:10:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[75179]: cluster 2023-10-09T09:10:09.994435+0000 mgr.y (mgr.25421) 985 : cluster 0 pgmap v1042: 89 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 4756/116145 objects degraded (4.095%); 42603/116145 objects misplaced (36.681%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T09:10:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[75179]: cluster 2023-10-09T09:10:10.652617+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:10:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[75179]: cluster 2023-10-09T09:10:10.654521+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T09:10:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[75179]: audit 2023-10-09T09:10:10.658723+0000 mon.a (mon.0) 1318 : audit 1 from='client.? 172.21.15.16:0/588166208' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:10:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[75179]: cluster 2023-10-09T09:10:10.658943+0000 mon.a (mon.0) 1319 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T09:10:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[77892]: audit 2023-10-09T09:10:09.938991+0000 mgr.y (mgr.25421) 984 : audit 0 from='client.45392 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:10:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[77892]: cluster 2023-10-09T09:10:09.994435+0000 mgr.y (mgr.25421) 985 : cluster 0 pgmap v1042: 89 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 4756/116145 objects degraded (4.095%); 42603/116145 objects misplaced (36.681%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T09:10:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[77892]: cluster 2023-10-09T09:10:10.652617+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:10:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[77892]: cluster 2023-10-09T09:10:10.654521+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T09:10:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[77892]: audit 2023-10-09T09:10:10.658723+0000 mon.a (mon.0) 1318 : audit 1 from='client.? 172.21.15.16:0/588166208' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:10:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:11 smithi016 bash[77892]: cluster 2023-10-09T09:10:10.658943+0000 mon.a (mon.0) 1319 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T09:10:11.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:11.826 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:11.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:11.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:11.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:11.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:12.073 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:12.098 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:12.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:12.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:12.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:12.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:12.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:12.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:12.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:12.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:12.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:12.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:12.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:12.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:12.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:12.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:12.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:12.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:12.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:12.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:12.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:12.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:13.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:13.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:13.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:13.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:13.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:13.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:13.385 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:13.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:13.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:13.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:13.619 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:13.627 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:13 smithi094 bash[59677]: cluster 2023-10-09T09:10:11.995015+0000 mgr.y (mgr.25421) 986 : cluster 0 pgmap v1044: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 4639/113763 objects degraded (4.078%); 41298/113763 objects misplaced (36.302%); 20 MiB/s, 315 objects/s recovering 2023-10-09T09:10:13.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:13.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:13 smithi016 bash[77892]: cluster 2023-10-09T09:10:11.995015+0000 mgr.y (mgr.25421) 986 : cluster 0 pgmap v1044: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 4639/113763 objects degraded (4.078%); 41298/113763 objects misplaced (36.302%); 20 MiB/s, 315 objects/s recovering 2023-10-09T09:10:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:13 smithi016 bash[75179]: cluster 2023-10-09T09:10:11.995015+0000 mgr.y (mgr.25421) 986 : cluster 0 pgmap v1044: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 4639/113763 objects degraded (4.078%); 41298/113763 objects misplaced (36.302%); 20 MiB/s, 315 objects/s recovering 2023-10-09T09:10:13.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:13.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:13.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:13.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:13.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:13.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:13.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:14.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:14.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:14.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:14.140 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1041,"stamp":"2023-10-09T09:10:07.993378+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2628917176,"num_objects":40130,"num_object_clones":0,"num_object_copies":120390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4886,"num_objects_misplaced":44409,"num_objects_unfound":0,"num_objects_dirty":40130,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":153322,"num_write_kb":6183758,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6388,"num_bytes_recovered":415453266,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114556,"ondisk_log_size":114556,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":288,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12420884,"kb_used_data":8132700,"kb_used_omap":117,"kb_used_meta":4287882,"kb_avail":643795180,"statfs":{"total":671965249536,"available":659246264320,"internally_reserved":0,"allocated":8327884800,"data_stored":8323181631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":4390791826},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,4,5,3,4,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-229310464,"num_objects":-3499,"num_object_clones":0,"num_object_copies":-10497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-502,"num_objects_misplaced":-4287,"num_objects_unfound":0,"num_objects_dirty":-3499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3499,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1903,"num_bytes_recovered":124715008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002936"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":93,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.339791+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:09:44.339791+0000","last_peered":"2023-10-09T09:09:44.339791+0000","last_clean":"2023-10-09T09:09:44.339791+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:09:44.339791+0000","last_undegraded":"2023-10-09T09:09:44.339791+0000","last_fullsized":"2023-10-09T09:09:44.339791+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":199,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.342053+0000","last_change":"2023-10-09T09:09:06.663045+0000","last_active":"2023-10-09T09:09:44.342053+0000","last_peered":"2023-10-09T09:09:44.342053+0000","last_clean":"2023-10-09T09:09:44.342053+0000","last_became_active":"2023-10-09T09:09:06.662681+0000","last_became_peered":"2023-10-09T09:09:06.662681+0000","last_unstale":"2023-10-09T09:09:44.342053+0000","last_undegraded":"2023-10-09T09:09:44.342053+0000","last_fullsized":"2023-10-09T09:09:44.342053+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":152,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.339809+0000","last_change":"2023-10-09T09:09:06.662154+0000","last_active":"2023-10-09T09:09:44.339809+0000","last_peered":"2023-10-09T09:09:44.339809+0000","last_clean":"2023-10-09T09:09:44.339809+0000","last_became_active":"2023-10-09T09:09:06.661824+0000","last_became_peered":"2023-10-09T09:09:06.661824+0000","last_unstale":"2023-10-09T09:09:44.339809+0000","last_undegraded":"2023-10-09T09:09:44.339809+0000","last_fullsized":"2023-10-09T09:09:44.339809+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":263,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T09:09:45.353017+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:09:45.353017+0000","last_peered":"2023-10-09T09:09:45.353017+0000","last_clean":"2023-10-09T09:09:45.353017+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:09:45.353017+0000","last_undegraded":"2023-10-09T09:09:45.353017+0000","last_fullsized":"2023-10-09T09:09:45.353017+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.337416+0000","last_change":"2023-10-09T09:09:06.657019+0000","last_active":"2023-10-09T09:09:44.337416+0000","last_peered":"2023-10-09T09:09:44.337416+0000","last_clean":"2023-10-09T09:09:44.337416+0000","last_became_active":"2023-10-09T09:09:06.656605+0000","last_became_peered":"2023-10-09T09:09:06.656605+0000","last_unstale":"2023-10-09T09:09:44.337416+0000","last_undegraded":"2023-10-09T09:09:44.337416+0000","last_fullsized":"2023-10-09T09:09:44.337416+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:08.279835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":116,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.349610+0000","last_change":"2023-10-09T09:08:24.652386+0000","last_active":"2023-10-09T09:09:44.349610+0000","last_peered":"2023-10-09T09:09:44.349610+0000","last_clean":"2023-10-09T09:09:44.349610+0000","last_became_active":"2023-10-09T09:08:24.651982+0000","last_became_peered":"2023-10-09T09:08:24.651982+0000","last_unstale":"2023-10-09T09:09:44.349610+0000","last_undegraded":"2023-10-09T09:09:44.349610+0000","last_fullsized":"2023-10-09T09:09:44.349610+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":216,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T09:09:45.344295+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:09:45.344295+0000","last_peered":"2023-10-09T09:09:45.344295+0000","last_clean":"2023-10-09T09:09:45.344295+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:09:45.344295+0000","last_undegraded":"2023-10-09T09:09:45.344295+0000","last_fullsized":"2023-10-09T09:09:45.344295+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":200,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.344369+0000","last_change":"2023-10-09T09:09:06.662705+0000","last_active":"2023-10-09T09:09:44.344369+0000","last_peered":"2023-10-09T09:09:44.344369+0000","last_clean":"2023-10-09T09:09:44.344369+0000","last_became_active":"2023-10-09T09:09:06.662304+0000","last_became_peered":"2023-10-09T09:09:06.662304+0000","last_unstale":"2023-10-09T09:09:44.344369+0000","last_undegraded":"2023-10-09T09:09:44.344369+0000","last_fullsized":"2023-10-09T09:09:44.344369+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.366365+0000","last_change":"2023-10-09T09:09:44.365998+0000","last_active":"2023-10-09T09:09:44.366365+0000","last_peered":"2023-10-09T09:09:44.366365+0000","last_clean":"2023-10-09T09:09:44.366365+0000","last_became_active":"2023-10-09T09:09:44.365450+0000","last_became_peered":"2023-10-09T09:09:44.365450+0000","last_unstale":"2023-10-09T09:09:44.366365+0000","last_undegraded":"2023-10-09T09:09:44.366365+0000","last_fullsized":"2023-10-09T09:09:44.366365+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.364589+0000","last_change":"2023-10-09T09:09:44.364273+0000","last_active":"2023-10-09T09:09:44.364589+0000","last_peered":"2023-10-09T09:09:44.364589+0000","last_clean":"2023-10-09T09:09:44.364589+0000","last_became_active":"2023-10-09T09:09:44.363921+0000","last_became_peered":"2023-10-09T09:09:44.363921+0000","last_unstale":"2023-10-09T09:09:44.364589+0000","last_undegraded":"2023-10-09T09:09:44.364589+0000","last_fullsized":"2023-10-09T09:09:44.364589+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.784888+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:09:44.784888+0000","last_peered":"2023-10-09T09:09:44.784888+0000","last_clean":"2023-10-09T09:09:44.784888+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:09:44.784888+0000","last_undegraded":"2023-10-09T09:09:44.784888+0000","last_fullsized":"2023-10-09T09:09:44.784888+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":190,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.344778+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:09:44.344778+0000","last_peered":"2023-10-09T09:09:44.344778+0000","last_clean":"2023-10-09T09:09:44.344778+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:09:44.344778+0000","last_undegraded":"2023-10-09T09:09:44.344778+0000","last_fullsized":"2023-10-09T09:09:44.344778+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":132,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.378542+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:09:44.378542+0000","last_peered":"2023-10-09T09:09:44.378542+0000","last_clean":"2023-10-09T09:09:44.378542+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:09:44.378542+0000","last_undegraded":"2023-10-09T09:09:44.378542+0000","last_fullsized":"2023-10-09T09:09:44.378542+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":111,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.340151+0000","last_change":"2023-10-09T09:08:24.651923+0000","last_active":"2023-10-09T09:09:44.340151+0000","last_peered":"2023-10-09T09:09:44.340151+0000","last_clean":"2023-10-09T09:09:44.340151+0000","last_became_active":"2023-10-09T09:08:24.650836+0000","last_became_peered":"2023-10-09T09:08:24.650836+0000","last_unstale":"2023-10-09T09:09:44.340151+0000","last_undegraded":"2023-10-09T09:09:44.340151+0000","last_fullsized":"2023-10-09T09:09:44.340151+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":17,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.587670+0000","last_change":"2023-10-09T09:09:44.587384+0000","last_active":"2023-10-09T09:09:44.587670+0000","last_peered":"2023-10-09T09:09:44.587670+0000","last_clean":"2023-10-09T09:09:44.587670+0000","last_became_active":"2023-10-09T09:09:44.355040+0000","last_became_peered":"2023-10-09T09:09:44.355040+0000","last_unstale":"2023-10-09T09:09:44.587670+0000","last_undegraded":"2023-10-09T09:09:44.587670+0000","last_fullsized":"2023-10-09T09:09:44.587670+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.368696+0000","last_change":"2023-10-09T09:09:44.365312+0000","last_active":"2023-10-09T09:09:44.368696+0000","last_peered":"2023-10-09T09:09:44.368696+0000","last_clean":"2023-10-09T09:09:44.368696+0000","last_became_active":"2023-10-09T09:09:44.364936+0000","last_became_peered":"2023-10-09T09:09:44.364936+0000","last_unstale":"2023-10-09T09:09:44.368696+0000","last_undegraded":"2023-10-09T09:09:44.368696+0000","last_fullsized":"2023-10-09T09:09:44.368696+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":243,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.381017+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:09:44.381017+0000","last_peered":"2023-10-09T09:09:44.381017+0000","last_clean":"2023-10-09T09:09:44.381017+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:09:44.381017+0000","last_undegraded":"2023-10-09T09:09:44.381017+0000","last_fullsized":"2023-10-09T09:09:44.381017+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.364560+0000","last_change":"2023-10-09T09:09:44.363249+0000","last_active":"2023-10-09T09:09:44.364560+0000","last_peered":"2023-10-09T09:09:44.364560+0000","last_clean":"2023-10-09T09:09:44.364560+0000","last_became_active":"2023-10-09T09:09:44.362838+0000","last_became_peered":"2023-10-09T09:09:44.362838+0000","last_unstale":"2023-10-09T09:09:44.364560+0000","last_undegraded":"2023-10-09T09:09:44.364560+0000","last_fullsized":"2023-10-09T09:09:44.364560+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.353365+0000","last_change":"2023-10-09T09:09:44.352696+0000","last_active":"2023-10-09T09:09:44.353365+0000","last_peered":"2023-10-09T09:09:44.353365+0000","last_clean":"2023-10-09T09:09:44.353365+0000","last_became_active":"2023-10-09T09:09:44.351809+0000","last_became_peered":"2023-10-09T09:09:44.351809+0000","last_unstale":"2023-10-09T09:09:44.353365+0000","last_undegraded":"2023-10-09T09:09:44.353365+0000","last_fullsized":"2023-10-09T09:09:44.353365+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":216,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T09:09:45.353132+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:09:45.353132+0000","last_peered":"2023-10-09T09:09:45.353132+0000","last_clean":"2023-10-09T09:09:45.353132+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:09:45.353132+0000","last_undegraded":"2023-10-09T09:09:45.353132+0000","last_fullsized":"2023-10-09T09:09:45.353132+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":128,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.346655+0000","last_change":"2023-10-09T09:08:24.741108+0000","last_active":"2023-10-09T09:09:44.346655+0000","last_peered":"2023-10-09T09:09:44.346655+0000","last_clean":"2023-10-09T09:09:44.346655+0000","last_became_active":"2023-10-09T09:08:24.651865+0000","last_became_peered":"2023-10-09T09:08:24.651865+0000","last_unstale":"2023-10-09T09:09:44.346655+0000","last_undegraded":"2023-10-09T09:09:44.346655+0000","last_fullsized":"2023-10-09T09:09:44.346655+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.571281+0000","last_change":"2023-10-09T09:09:44.460257+0000","last_active":"2023-10-09T09:09:44.571281+0000","last_peered":"2023-10-09T09:09:44.571281+0000","last_clean":"2023-10-09T09:09:44.571281+0000","last_became_active":"2023-10-09T09:09:44.459713+0000","last_became_peered":"2023-10-09T09:09:44.459713+0000","last_unstale":"2023-10-09T09:09:44.571281+0000","last_undegraded":"2023-10-09T09:09:44.571281+0000","last_fullsized":"2023-10-09T09:09:44.571281+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"231'9387","reported_seq":9428,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.223334+0000","last_change":"2023-10-09T09:09:44.592056+0000","last_active":"2023-10-09T09:10:03.223334+0000","last_peered":"2023-10-09T09:10:03.223334+0000","last_clean":"2023-10-09T09:08:23.960450+0000","last_became_active":"2023-10-09T09:08:26.199042+0000","last_became_peered":"2023-10-09T09:08:26.199042+0000","last_unstale":"2023-10-09T09:10:03.223334+0000","last_undegraded":"2023-10-09T09:10:03.223334+0000","last_fullsized":"2023-10-09T09:10:03.223334+0000","mapping_epoch":222,"log_start":"218'2800","ondisk_log_start":"218'2800","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:30.450356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162463744,"num_objects":2479,"num_object_clones":0,"num_object_copies":7437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2479,"num_objects_unfound":0,"num_objects_dirty":2479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9387,"num_write_kb":379712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,3,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":2479}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:45.088922+0000","last_change":"2023-10-09T09:09:45.088513+0000","last_active":"2023-10-09T09:09:45.088922+0000","last_peered":"2023-10-09T09:09:45.088922+0000","last_clean":"2023-10-09T09:09:45.088922+0000","last_became_active":"2023-10-09T09:09:44.459170+0000","last_became_peered":"2023-10-09T09:09:44.459170+0000","last_unstale":"2023-10-09T09:09:45.088922+0000","last_undegraded":"2023-10-09T09:09:45.088922+0000","last_fullsized":"2023-10-09T09:09:45.088922+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.357045+0000","last_change":"2023-10-09T09:09:44.355469+0000","last_active":"2023-10-09T09:09:44.357045+0000","last_peered":"2023-10-09T09:09:44.357045+0000","last_clean":"2023-10-09T09:09:44.357045+0000","last_became_active":"2023-10-09T09:09:44.354880+0000","last_became_peered":"2023-10-09T09:09:44.354880+0000","last_unstale":"2023-10-09T09:09:44.357045+0000","last_undegraded":"2023-10-09T09:09:44.357045+0000","last_fullsized":"2023-10-09T09:09:44.357045+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"231'9772","reported_seq":9798,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T09:10:05.650203+0000","last_change":"2023-10-09T09:06:27.616114+0000","last_active":"2023-10-09T09:10:05.650203+0000","last_peered":"2023-10-09T09:10:05.650203+0000","last_clean":"2023-10-09T09:10:05.650203+0000","last_became_active":"2023-10-09T09:06:27.615815+0000","last_became_peered":"2023-10-09T09:06:27.615815+0000","last_unstale":"2023-10-09T09:10:05.650203+0000","last_undegraded":"2023-10-09T09:10:05.650203+0000","last_fullsized":"2023-10-09T09:10:05.650203+0000","mapping_epoch":212,"log_start":"218'2200","ondisk_log_start":"218'2200","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165937152,"num_objects":2532,"num_object_clones":0,"num_object_copies":7596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9772,"num_write_kb":393728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":15,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:45.001235+0000","last_change":"2023-10-09T09:09:45.000846+0000","last_active":"2023-10-09T09:09:45.001235+0000","last_peered":"2023-10-09T09:09:45.001235+0000","last_clean":"2023-10-09T09:09:45.001235+0000","last_became_active":"2023-10-09T09:09:44.404839+0000","last_became_peered":"2023-10-09T09:09:44.404839+0000","last_unstale":"2023-10-09T09:09:45.001235+0000","last_undegraded":"2023-10-09T09:09:45.001235+0000","last_fullsized":"2023-10-09T09:09:45.001235+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":111,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.339992+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:09:44.339992+0000","last_peered":"2023-10-09T09:09:44.339992+0000","last_clean":"2023-10-09T09:09:44.339992+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:09:44.339992+0000","last_undegraded":"2023-10-09T09:09:44.339992+0000","last_fullsized":"2023-10-09T09:09:44.339992+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"231'9320","reported_seq":9363,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.363146+0000","last_change":"2023-10-09T09:09:44.602165+0000","last_active":"2023-10-09T09:10:03.363146+0000","last_peered":"2023-10-09T09:10:03.363146+0000","last_clean":"2023-10-09T09:08:23.906250+0000","last_became_active":"2023-10-09T09:09:44.572919+0000","last_became_peered":"2023-10-09T09:09:44.572919+0000","last_unstale":"2023-10-09T09:10:03.363146+0000","last_undegraded":"2023-10-09T09:10:03.363146+0000","last_fullsized":"2023-10-09T09:10:03.363146+0000","mapping_epoch":229,"log_start":"218'1100","ondisk_log_start":"218'1100","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":8220,"log_dups_size":0,"ondisk_log_size":8220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:40.258646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164888576,"num_objects":2516,"num_object_clones":0,"num_object_copies":7548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5032,"num_objects_unfound":0,"num_objects_dirty":2516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9320,"num_write_kb":378752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[2,3,6],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":2516}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.913781+0000","last_change":"2023-10-09T09:09:44.913374+0000","last_active":"2023-10-09T09:09:44.913781+0000","last_peered":"2023-10-09T09:09:44.913781+0000","last_clean":"2023-10-09T09:09:44.913781+0000","last_became_active":"2023-10-09T09:09:44.369058+0000","last_became_peered":"2023-10-09T09:09:44.369058+0000","last_unstale":"2023-10-09T09:09:44.913781+0000","last_undegraded":"2023-10-09T09:09:44.913781+0000","last_fullsized":"2023-10-09T09:09:44.913781+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":125,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.360699+0000","last_change":"2023-10-09T09:09:44.359930+0000","last_active":"2023-10-09T09:09:44.360699+0000","last_peered":"2023-10-09T09:09:44.360699+0000","last_clean":"2023-10-09T09:09:44.360699+0000","last_became_active":"2023-10-09T09:09:44.359475+0000","last_became_peered":"2023-10-09T09:09:44.359475+0000","last_unstale":"2023-10-09T09:09:44.360699+0000","last_undegraded":"2023-10-09T09:09:44.360699+0000","last_fullsized":"2023-10-09T09:09:44.360699+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"231'9418","reported_seq":10263,"reported_epoch":231,"state":"active+recovering+remapped","last_fresh":"2023-10-09T09:10:03.226205+0000","last_change":"2023-10-09T09:09:45.092930+0000","last_active":"2023-10-09T09:10:03.226205+0000","last_peered":"2023-10-09T09:10:03.226205+0000","last_clean":"2023-10-09T09:08:23.959307+0000","last_became_active":"2023-10-09T09:08:26.300908+0000","last_became_peered":"2023-10-09T09:08:26.300908+0000","last_unstale":"2023-10-09T09:10:03.226205+0000","last_undegraded":"2023-10-09T09:10:03.226205+0000","last_fullsized":"2023-10-09T09:10:03.226205+0000","mapping_epoch":222,"log_start":"218'2800","ondisk_log_start":"218'2800","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6618,"log_dups_size":0,"ondisk_log_size":6618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:06.715524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163971072,"num_objects":2502,"num_object_clones":0,"num_object_copies":7506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5507,"num_objects_unfound":0,"num_objects_dirty":2502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9418,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":400,"num_bytes_recovered":26214400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[0,2,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":-3005}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":19,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.354902+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:09:44.354902+0000","last_peered":"2023-10-09T09:09:44.354902+0000","last_clean":"2023-10-09T09:09:44.354902+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:09:44.354902+0000","last_undegraded":"2023-10-09T09:09:44.354902+0000","last_fullsized":"2023-10-09T09:09:44.354902+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":225,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.338472+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:09:44.338472+0000","last_peered":"2023-10-09T09:09:44.338472+0000","last_clean":"2023-10-09T09:09:44.338472+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:09:44.338472+0000","last_undegraded":"2023-10-09T09:09:44.338472+0000","last_fullsized":"2023-10-09T09:09:44.338472+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"231'9758","reported_seq":15395,"reported_epoch":231,"state":"active+clean+remapped","last_fresh":"2023-10-09T09:10:06.240524+0000","last_change":"2023-10-09T09:10:05.788264+0000","last_active":"2023-10-09T09:10:06.240524+0000","last_peered":"2023-10-09T09:10:06.240524+0000","last_clean":"2023-10-09T09:10:06.240524+0000","last_became_active":"2023-10-09T09:09:45.496687+0000","last_became_peered":"2023-10-09T09:09:45.496687+0000","last_unstale":"2023-10-09T09:10:06.240524+0000","last_undegraded":"2023-10-09T09:10:06.240524+0000","last_fullsized":"2023-10-09T09:10:06.240524+0000","mapping_epoch":230,"log_start":"218'2900","ondisk_log_start":"218'2900","created":212,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161611776,"num_objects":2466,"num_object_clones":0,"num_object_copies":7398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9758,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2793,"num_bytes_recovered":183042048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,4],"avail_no_missing":["6","3","4","7"],"object_location_counts":[{"shards":"3,4,6","objects":2466}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":37,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.338619+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:09:44.338619+0000","last_peered":"2023-10-09T09:09:44.338619+0000","last_clean":"2023-10-09T09:09:44.338619+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:09:44.338619+0000","last_undegraded":"2023-10-09T09:09:44.338619+0000","last_fullsized":"2023-10-09T09:09:44.338619+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":93,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.341872+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:09:44.341872+0000","last_peered":"2023-10-09T09:09:44.341872+0000","last_clean":"2023-10-09T09:09:44.341872+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:09:44.341872+0000","last_undegraded":"2023-10-09T09:09:44.341872+0000","last_fullsized":"2023-10-09T09:09:44.341872+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"231'9656","reported_seq":9683,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T09:10:06.216651+0000","last_change":"2023-10-09T09:06:27.621010+0000","last_active":"2023-10-09T09:10:06.216651+0000","last_peered":"2023-10-09T09:10:06.216651+0000","last_clean":"2023-10-09T09:10:06.216651+0000","last_became_active":"2023-10-09T09:06:27.620675+0000","last_became_peered":"2023-10-09T09:06:27.620675+0000","last_unstale":"2023-10-09T09:10:06.216651+0000","last_undegraded":"2023-10-09T09:10:06.216651+0000","last_fullsized":"2023-10-09T09:10:06.216651+0000","mapping_epoch":212,"log_start":"218'2800","ondisk_log_start":"218'2800","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6856,"log_dups_size":0,"ondisk_log_size":6856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156499968,"num_objects":2388,"num_object_clones":0,"num_object_copies":7164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9656,"num_write_kb":385408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":89,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.344878+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:09:44.344878+0000","last_peered":"2023-10-09T09:09:44.344878+0000","last_clean":"2023-10-09T09:09:44.344878+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:09:44.344878+0000","last_undegraded":"2023-10-09T09:09:44.344878+0000","last_fullsized":"2023-10-09T09:09:44.344878+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":93,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.342173+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:09:44.342173+0000","last_peered":"2023-10-09T09:09:44.342173+0000","last_clean":"2023-10-09T09:09:44.342173+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:09:44.342173+0000","last_undegraded":"2023-10-09T09:09:44.342173+0000","last_fullsized":"2023-10-09T09:09:44.342173+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"231'9741","reported_seq":9804,"reported_epoch":232,"state":"active+remapped","last_fresh":"2023-10-09T09:10:06.440319+0000","last_change":"2023-10-09T09:09:44.619093+0000","last_active":"2023-10-09T09:10:06.440319+0000","last_peered":"2023-10-09T09:10:06.440319+0000","last_clean":"2023-10-09T09:08:23.948158+0000","last_became_active":"2023-10-09T09:09:44.544110+0000","last_became_peered":"2023-10-09T09:09:44.544110+0000","last_unstale":"2023-10-09T09:10:06.440319+0000","last_undegraded":"2023-10-09T09:10:06.440319+0000","last_fullsized":"2023-10-09T09:10:06.440319+0000","mapping_epoch":229,"log_start":"219'3600","ondisk_log_start":"219'3600","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6141,"log_dups_size":0,"ondisk_log_size":6141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:22.040106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164691968,"num_objects":2513,"num_object_clones":0,"num_object_copies":7539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5026,"num_objects_unfound":0,"num_objects_dirty":2513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9741,"num_write_kb":392128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[3,6,0],"avail_no_missing":["3","0","6"],"object_location_counts":[{"shards":"0,3,6","objects":2513}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.353348+0000","last_change":"2023-10-09T09:09:44.352729+0000","last_active":"2023-10-09T09:09:44.353348+0000","last_peered":"2023-10-09T09:09:44.353348+0000","last_clean":"2023-10-09T09:09:44.353348+0000","last_became_active":"2023-10-09T09:09:44.352124+0000","last_became_peered":"2023-10-09T09:09:44.352124+0000","last_unstale":"2023-10-09T09:09:44.353348+0000","last_undegraded":"2023-10-09T09:09:44.353348+0000","last_fullsized":"2023-10-09T09:09:44.353348+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":102,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.342243+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:09:44.342243+0000","last_peered":"2023-10-09T09:09:44.342243+0000","last_clean":"2023-10-09T09:09:44.342243+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:09:44.342243+0000","last_undegraded":"2023-10-09T09:09:44.342243+0000","last_fullsized":"2023-10-09T09:09:44.342243+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"231'9422","reported_seq":9468,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.323165+0000","last_change":"2023-10-09T09:09:45.561410+0000","last_active":"2023-10-09T09:10:03.323165+0000","last_peered":"2023-10-09T09:10:03.323165+0000","last_clean":"2023-10-09T09:09:43.316467+0000","last_became_active":"2023-10-09T09:09:45.510708+0000","last_became_peered":"2023-10-09T09:09:45.510708+0000","last_unstale":"2023-10-09T09:10:03.323165+0000","last_undegraded":"2023-10-09T09:10:03.323165+0000","last_fullsized":"2023-10-09T09:10:03.323165+0000","mapping_epoch":230,"log_start":"218'1900","ondisk_log_start":"218'1900","created":212,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":7522,"log_dups_size":0,"ondisk_log_size":7522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:15.844025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167116800,"num_objects":2550,"num_object_clones":0,"num_object_copies":7650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2550,"num_objects_unfound":0,"num_objects_dirty":2550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9422,"num_write_kb":383104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,6],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":2550}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.363591+0000","last_change":"2023-10-09T09:09:44.362102+0000","last_active":"2023-10-09T09:09:44.363591+0000","last_peered":"2023-10-09T09:09:44.363591+0000","last_clean":"2023-10-09T09:09:44.363591+0000","last_became_active":"2023-10-09T09:09:44.361694+0000","last_became_peered":"2023-10-09T09:09:44.361694+0000","last_unstale":"2023-10-09T09:09:44.363591+0000","last_undegraded":"2023-10-09T09:09:44.363591+0000","last_fullsized":"2023-10-09T09:09:44.363591+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":204,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.354114+0000","last_change":"2023-10-09T09:09:44.353425+0000","last_active":"2023-10-09T09:09:44.354114+0000","last_peered":"2023-10-09T09:09:44.354114+0000","last_clean":"2023-10-09T09:09:44.354114+0000","last_became_active":"2023-10-09T09:09:44.352821+0000","last_became_peered":"2023-10-09T09:09:44.352821+0000","last_unstale":"2023-10-09T09:09:44.354114+0000","last_undegraded":"2023-10-09T09:09:44.354114+0000","last_fullsized":"2023-10-09T09:09:44.354114+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"231'9699","reported_seq":9858,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.415921+0000","last_change":"2023-10-09T09:09:45.093965+0000","last_active":"2023-10-09T09:10:03.415921+0000","last_peered":"2023-10-09T09:10:03.415921+0000","last_clean":"2023-10-09T09:08:23.947723+0000","last_became_active":"2023-10-09T09:08:26.115150+0000","last_became_peered":"2023-10-09T09:08:26.115150+0000","last_unstale":"2023-10-09T09:10:03.415921+0000","last_undegraded":"2023-10-09T09:10:03.415921+0000","last_fullsized":"2023-10-09T09:10:03.415921+0000","mapping_epoch":222,"log_start":"218'1400","ondisk_log_start":"218'1400","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":8299,"log_dups_size":0,"ondisk_log_size":8299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:54.311905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164298752,"num_objects":2507,"num_object_clones":0,"num_object_copies":7521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2460,"num_objects_unfound":0,"num_objects_dirty":2507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9699,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":3932160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":2507}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":184,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.380709+0000","last_change":"2023-10-09T09:09:44.376837+0000","last_active":"2023-10-09T09:09:44.380709+0000","last_peered":"2023-10-09T09:09:44.380709+0000","last_clean":"2023-10-09T09:09:44.380709+0000","last_became_active":"2023-10-09T09:09:44.376542+0000","last_became_peered":"2023-10-09T09:09:44.376542+0000","last_unstale":"2023-10-09T09:09:44.380709+0000","last_undegraded":"2023-10-09T09:09:44.380709+0000","last_fullsized":"2023-10-09T09:09:44.380709+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:53.092218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":19,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.357218+0000","last_change":"2023-10-09T09:08:24.730554+0000","last_active":"2023-10-09T09:09:44.357218+0000","last_peered":"2023-10-09T09:09:44.357218+0000","last_clean":"2023-10-09T09:09:44.357218+0000","last_became_active":"2023-10-09T09:08:24.729359+0000","last_became_peered":"2023-10-09T09:08:24.729359+0000","last_unstale":"2023-10-09T09:09:44.357218+0000","last_undegraded":"2023-10-09T09:09:44.357218+0000","last_fullsized":"2023-10-09T09:09:44.357218+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":19,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.353647+0000","last_change":"2023-10-09T09:08:24.658172+0000","last_active":"2023-10-09T09:09:44.353647+0000","last_peered":"2023-10-09T09:09:44.353647+0000","last_clean":"2023-10-09T09:09:44.353647+0000","last_became_active":"2023-10-09T09:08:24.657766+0000","last_became_peered":"2023-10-09T09:08:24.657766+0000","last_unstale":"2023-10-09T09:09:44.353647+0000","last_undegraded":"2023-10-09T09:09:44.353647+0000","last_fullsized":"2023-10-09T09:09:44.353647+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"231'9479","reported_seq":9531,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.158807+0000","last_change":"2023-10-09T09:09:44.618377+0000","last_active":"2023-10-09T09:10:03.158807+0000","last_peered":"2023-10-09T09:10:03.158807+0000","last_clean":"2023-10-09T09:08:23.964566+0000","last_became_active":"2023-10-09T09:09:44.556693+0000","last_became_peered":"2023-10-09T09:09:44.556693+0000","last_unstale":"2023-10-09T09:10:03.158807+0000","last_undegraded":"2023-10-09T09:10:03.158807+0000","last_fullsized":"2023-10-09T09:10:03.158807+0000","mapping_epoch":229,"log_start":"218'2900","ondisk_log_start":"218'2900","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:01.672798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167706624,"num_objects":2559,"num_object_clones":0,"num_object_copies":7677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5118,"num_objects_unfound":0,"num_objects_dirty":2559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9479,"num_write_kb":385216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,3,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":2559}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.354387+0000","last_change":"2023-10-09T09:08:24.658358+0000","last_active":"2023-10-09T09:09:44.354387+0000","last_peered":"2023-10-09T09:09:44.354387+0000","last_clean":"2023-10-09T09:09:44.354387+0000","last_became_active":"2023-10-09T09:08:24.657360+0000","last_became_peered":"2023-10-09T09:08:24.657360+0000","last_unstale":"2023-10-09T09:09:44.354387+0000","last_undegraded":"2023-10-09T09:09:44.354387+0000","last_fullsized":"2023-10-09T09:09:44.354387+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.340589+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:09:44.340589+0000","last_peered":"2023-10-09T09:09:44.340589+0000","last_clean":"2023-10-09T09:09:44.340589+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:09:44.340589+0000","last_undegraded":"2023-10-09T09:09:44.340589+0000","last_fullsized":"2023-10-09T09:09:44.340589+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":199,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.335402+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:09:44.335402+0000","last_peered":"2023-10-09T09:09:44.335402+0000","last_clean":"2023-10-09T09:09:44.335402+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:09:44.335402+0000","last_undegraded":"2023-10-09T09:09:44.335402+0000","last_fullsized":"2023-10-09T09:09:44.335402+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"231'9509","reported_seq":9651,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.402901+0000","last_change":"2023-10-09T09:09:44.608398+0000","last_active":"2023-10-09T09:10:03.402901+0000","last_peered":"2023-10-09T09:10:03.402901+0000","last_clean":"2023-10-09T09:08:23.939502+0000","last_became_active":"2023-10-09T09:09:44.581577+0000","last_became_peered":"2023-10-09T09:09:44.581577+0000","last_unstale":"2023-10-09T09:10:03.402901+0000","last_undegraded":"2023-10-09T09:10:03.402901+0000","last_fullsized":"2023-10-09T09:10:03.402901+0000","mapping_epoch":229,"log_start":"218'1200","ondisk_log_start":"218'1200","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":8309,"log_dups_size":0,"ondisk_log_size":8309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:53.708871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168755200,"num_objects":2575,"num_object_clones":0,"num_object_copies":7725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5130,"num_objects_unfound":0,"num_objects_dirty":2575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9509,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[2,3,6],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":2575}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":258,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.334733+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:09:44.334733+0000","last_peered":"2023-10-09T09:09:44.334733+0000","last_clean":"2023-10-09T09:09:44.334733+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:09:44.334733+0000","last_undegraded":"2023-10-09T09:09:44.334733+0000","last_fullsized":"2023-10-09T09:09:44.334733+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":191,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.344307+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:09:44.344307+0000","last_peered":"2023-10-09T09:09:44.344307+0000","last_clean":"2023-10-09T09:09:44.344307+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:09:44.344307+0000","last_undegraded":"2023-10-09T09:09:44.344307+0000","last_fullsized":"2023-10-09T09:09:44.344307+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":28,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.444955+0000","last_change":"2023-10-09T09:09:44.441355+0000","last_active":"2023-10-09T09:09:44.444955+0000","last_peered":"2023-10-09T09:09:44.444955+0000","last_clean":"2023-10-09T09:09:44.444955+0000","last_became_active":"2023-10-09T09:09:44.408597+0000","last_became_peered":"2023-10-09T09:09:44.408597+0000","last_unstale":"2023-10-09T09:09:44.444955+0000","last_undegraded":"2023-10-09T09:09:44.444955+0000","last_fullsized":"2023-10-09T09:09:44.444955+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"231'9679","reported_seq":9724,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.222961+0000","last_change":"2023-10-09T09:09:07.675454+0000","last_active":"2023-10-09T09:10:03.222961+0000","last_peered":"2023-10-09T09:10:03.222961+0000","last_clean":"2023-10-09T09:08:23.924163+0000","last_became_active":"2023-10-09T09:09:07.656229+0000","last_became_peered":"2023-10-09T09:09:07.656229+0000","last_unstale":"2023-10-09T09:10:03.222961+0000","last_undegraded":"2023-10-09T09:10:03.222961+0000","last_fullsized":"2023-10-09T09:10:03.222961+0000","mapping_epoch":226,"log_start":"219'3100","ondisk_log_start":"219'3100","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:43.972555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171507712,"num_objects":2617,"num_object_clones":0,"num_object_copies":7851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2617,"num_objects_unfound":0,"num_objects_dirty":2617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9679,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,2],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2617}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":28,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.357181+0000","last_change":"2023-10-09T09:09:44.356239+0000","last_active":"2023-10-09T09:09:44.357181+0000","last_peered":"2023-10-09T09:09:44.357181+0000","last_clean":"2023-10-09T09:09:44.357181+0000","last_became_active":"2023-10-09T09:09:44.355854+0000","last_became_peered":"2023-10-09T09:09:44.355854+0000","last_unstale":"2023-10-09T09:09:44.357181+0000","last_undegraded":"2023-10-09T09:09:44.357181+0000","last_fullsized":"2023-10-09T09:09:44.357181+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":477,"reported_epoch":231,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:09:45.354744+0000","last_change":"2023-10-09T09:09:45.355026+0000","last_active":"2023-10-09T09:09:45.354744+0000","last_peered":"2023-10-09T09:09:45.354744+0000","last_clean":"2023-10-09T09:09:42.333501+0000","last_became_active":"2023-10-09T09:09:45.354572+0000","last_became_peered":"2023-10-09T09:09:45.354572+0000","last_unstale":"2023-10-09T09:09:45.354744+0000","last_undegraded":"2023-10-09T09:09:45.342537+0000","last_fullsized":"2023-10-09T09:09:45.341447+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:53:21.907088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":58,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.353765+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:09:44.353765+0000","last_peered":"2023-10-09T09:09:44.353765+0000","last_clean":"2023-10-09T09:09:44.353765+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:09:44.353765+0000","last_undegraded":"2023-10-09T09:09:44.353765+0000","last_fullsized":"2023-10-09T09:09:44.353765+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":89,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.361892+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:09:44.361892+0000","last_peered":"2023-10-09T09:09:44.361892+0000","last_clean":"2023-10-09T09:09:44.361892+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:09:44.361892+0000","last_undegraded":"2023-10-09T09:09:44.361892+0000","last_fullsized":"2023-10-09T09:09:44.361892+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"231'9838","reported_seq":9883,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:03.211736+0000","last_change":"2023-10-09T09:09:43.390773+0000","last_active":"2023-10-09T09:10:03.211736+0000","last_peered":"2023-10-09T09:10:03.211736+0000","last_clean":"2023-10-09T09:08:23.947423+0000","last_became_active":"2023-10-09T09:08:26.089656+0000","last_became_peered":"2023-10-09T09:08:26.089656+0000","last_unstale":"2023-10-09T09:10:03.211736+0000","last_undegraded":"2023-10-09T09:10:03.211736+0000","last_fullsized":"2023-10-09T09:10:03.211736+0000","mapping_epoch":222,"log_start":"219'3300","ondisk_log_start":"219'3300","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.380018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166461440,"num_objects":2540,"num_object_clones":0,"num_object_copies":7620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2539,"num_objects_unfound":0,"num_objects_dirty":2540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9838,"num_write_kb":396096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":2540}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.394093+0000","last_change":"2023-10-09T09:09:44.393807+0000","last_active":"2023-10-09T09:09:44.394093+0000","last_peered":"2023-10-09T09:09:44.394093+0000","last_clean":"2023-10-09T09:09:44.394093+0000","last_became_active":"2023-10-09T09:09:44.393066+0000","last_became_peered":"2023-10-09T09:09:44.393066+0000","last_unstale":"2023-10-09T09:09:44.394093+0000","last_undegraded":"2023-10-09T09:09:44.394093+0000","last_fullsized":"2023-10-09T09:09:44.394093+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":262,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.337479+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:09:44.337479+0000","last_peered":"2023-10-09T09:09:44.337479+0000","last_clean":"2023-10-09T09:09:44.337479+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:09:44.337479+0000","last_undegraded":"2023-10-09T09:09:44.337479+0000","last_fullsized":"2023-10-09T09:09:44.337479+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":241,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.392834+0000","last_change":"2023-10-09T09:09:44.378484+0000","last_active":"2023-10-09T09:09:44.392834+0000","last_peered":"2023-10-09T09:09:44.392834+0000","last_clean":"2023-10-09T09:09:44.392834+0000","last_became_active":"2023-10-09T09:09:44.378153+0000","last_became_peered":"2023-10-09T09:09:44.378153+0000","last_unstale":"2023-10-09T09:09:44.392834+0000","last_undegraded":"2023-10-09T09:09:44.392834+0000","last_fullsized":"2023-10-09T09:09:44.392834+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:18.230306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"231'9661","reported_seq":10980,"reported_epoch":231,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T09:10:05.655770+0000","last_change":"2023-10-09T09:09:45.745196+0000","last_active":"2023-10-09T09:10:05.655770+0000","last_peered":"2023-10-09T09:10:05.655770+0000","last_clean":"2023-10-09T09:08:23.959796+0000","last_became_active":"2023-10-09T09:09:45.623623+0000","last_became_peered":"2023-10-09T09:09:45.623623+0000","last_unstale":"2023-10-09T09:10:05.655770+0000","last_undegraded":"2023-10-09T09:09:45.598566+0000","last_fullsized":"2023-10-09T09:10:05.655770+0000","mapping_epoch":230,"log_start":"218'2100","ondisk_log_start":"218'2100","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":7561,"log_dups_size":0,"ondisk_log_size":7561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160366592,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2447,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9661,"num_write_kb":387456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":639,"num_bytes_recovered":41811968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[4,0,5],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":2395},{"shards":"0,4,5","objects":52}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.439441+0000","last_change":"2023-10-09T09:09:44.439090+0000","last_active":"2023-10-09T09:09:44.439441+0000","last_peered":"2023-10-09T09:09:44.439441+0000","last_clean":"2023-10-09T09:09:44.439441+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:09:44.439441+0000","last_undegraded":"2023-10-09T09:09:44.439441+0000","last_fullsized":"2023-10-09T09:09:44.439441+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":210,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.335222+0000","last_change":"2023-10-09T09:08:24.652222+0000","last_active":"2023-10-09T09:09:44.335222+0000","last_peered":"2023-10-09T09:09:44.335222+0000","last_clean":"2023-10-09T09:09:44.335222+0000","last_became_active":"2023-10-09T09:08:24.651702+0000","last_became_peered":"2023-10-09T09:08:24.651702+0000","last_unstale":"2023-10-09T09:09:44.335222+0000","last_undegraded":"2023-10-09T09:09:44.335222+0000","last_fullsized":"2023-10-09T09:09:44.335222+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:49.138740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":238,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.396529+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:09:44.396529+0000","last_peered":"2023-10-09T09:09:44.396529+0000","last_clean":"2023-10-09T09:09:44.396529+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:09:44.396529+0000","last_undegraded":"2023-10-09T09:09:44.396529+0000","last_fullsized":"2023-10-09T09:09:44.396529+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"231'9339","reported_seq":14108,"reported_epoch":231,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T09:10:03.223556+0000","last_change":"2023-10-09T09:09:45.613583+0000","last_active":"2023-10-09T09:10:03.223556+0000","last_peered":"2023-10-09T09:10:03.223556+0000","last_clean":"2023-10-09T09:08:23.934373+0000","last_became_active":"2023-10-09T09:09:45.576234+0000","last_became_peered":"2023-10-09T09:09:45.576234+0000","last_unstale":"2023-10-09T09:10:03.223556+0000","last_undegraded":"2023-10-09T09:09:45.458487+0000","last_fullsized":"2023-10-09T09:10:03.223556+0000","mapping_epoch":230,"log_start":"218'2800","ondisk_log_start":"218'2800","created":212,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":6539,"log_dups_size":0,"ondisk_log_size":6539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:19.144620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159645720,"num_objects":2437,"num_object_clones":0,"num_object_copies":7311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2437,"num_objects_misplaced":3473,"num_objects_unfound":0,"num_objects_dirty":2437,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9339,"num_write_kb":376769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2364,"num_bytes_recovered":154796032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5","objects":51},{"shards":"0,4,6","objects":2386}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":60,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.616706+0000","last_change":"2023-10-09T09:09:44.616450+0000","last_active":"2023-10-09T09:09:44.616706+0000","last_peered":"2023-10-09T09:09:44.616706+0000","last_clean":"2023-10-09T09:09:44.616706+0000","last_became_active":"2023-10-09T09:09:44.393002+0000","last_became_peered":"2023-10-09T09:09:44.393002+0000","last_unstale":"2023-10-09T09:09:44.616706+0000","last_undegraded":"2023-10-09T09:09:44.616706+0000","last_fullsized":"2023-10-09T09:09:44.616706+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:53:56.650152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":193,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.343645+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:09:44.343645+0000","last_peered":"2023-10-09T09:09:44.343645+0000","last_clean":"2023-10-09T09:09:44.343645+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:09:44.343645+0000","last_undegraded":"2023-10-09T09:09:44.343645+0000","last_fullsized":"2023-10-09T09:09:44.343645+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.394909+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:09:44.394909+0000","last_peered":"2023-10-09T09:09:44.394909+0000","last_clean":"2023-10-09T09:09:44.394909+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:09:44.394909+0000","last_undegraded":"2023-10-09T09:09:44.394909+0000","last_fullsized":"2023-10-09T09:09:44.394909+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"231'9384","reported_seq":9431,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:10:05.579918+0000","last_change":"2023-10-09T09:09:45.560307+0000","last_active":"2023-10-09T09:10:05.579918+0000","last_peered":"2023-10-09T09:10:05.579918+0000","last_clean":"2023-10-09T09:09:43.330952+0000","last_became_active":"2023-10-09T09:09:45.522912+0000","last_became_peered":"2023-10-09T09:09:45.522912+0000","last_unstale":"2023-10-09T09:10:05.579918+0000","last_undegraded":"2023-10-09T09:10:05.579918+0000","last_fullsized":"2023-10-09T09:10:05.579918+0000","mapping_epoch":230,"log_start":"218'1800","ondisk_log_start":"218'1800","created":212,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:06:26.587297+0000","last_clean_scrub_stamp":"2023-10-09T09:06:26.587297+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162398208,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2478,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9384,"num_write_kb":379584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":2478}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":80,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.349705+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:09:44.349705+0000","last_peered":"2023-10-09T09:09:44.349705+0000","last_clean":"2023-10-09T09:09:44.349705+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:09:44.349705+0000","last_undegraded":"2023-10-09T09:09:44.349705+0000","last_fullsized":"2023-10-09T09:09:44.349705+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":268,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.611069+0000","last_change":"2023-10-09T09:09:44.610749+0000","last_active":"2023-10-09T09:09:44.611069+0000","last_peered":"2023-10-09T09:09:44.611069+0000","last_clean":"2023-10-09T09:09:44.611069+0000","last_became_active":"2023-10-09T09:09:44.391749+0000","last_became_peered":"2023-10-09T09:09:44.391749+0000","last_unstale":"2023-10-09T09:09:44.611069+0000","last_undegraded":"2023-10-09T09:09:44.611069+0000","last_fullsized":"2023-10-09T09:09:44.611069+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:23.536700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":111,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.340081+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:09:44.340081+0000","last_peered":"2023-10-09T09:09:44.340081+0000","last_clean":"2023-10-09T09:09:44.340081+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:09:44.340081+0000","last_undegraded":"2023-10-09T09:09:44.340081+0000","last_fullsized":"2023-10-09T09:09:44.340081+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":249,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.388044+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:09:44.388044+0000","last_peered":"2023-10-09T09:09:44.388044+0000","last_clean":"2023-10-09T09:09:44.388044+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:09:44.388044+0000","last_undegraded":"2023-10-09T09:09:44.388044+0000","last_fullsized":"2023-10-09T09:09:44.388044+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":62,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.335023+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:09:44.335023+0000","last_peered":"2023-10-09T09:09:44.335023+0000","last_clean":"2023-10-09T09:09:44.335023+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:09:44.335023+0000","last_undegraded":"2023-10-09T09:09:44.335023+0000","last_fullsized":"2023-10-09T09:09:44.335023+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":23,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.354504+0000","last_change":"2023-10-09T09:08:24.661552+0000","last_active":"2023-10-09T09:09:44.354504+0000","last_peered":"2023-10-09T09:09:44.354504+0000","last_clean":"2023-10-09T09:09:44.354504+0000","last_became_active":"2023-10-09T09:08:24.655946+0000","last_became_peered":"2023-10-09T09:08:24.655946+0000","last_unstale":"2023-10-09T09:09:44.354504+0000","last_undegraded":"2023-10-09T09:09:44.354504+0000","last_fullsized":"2023-10-09T09:09:44.354504+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":93,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.340270+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:09:44.340270+0000","last_peered":"2023-10-09T09:09:44.340270+0000","last_clean":"2023-10-09T09:09:44.340270+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:09:44.340270+0000","last_undegraded":"2023-10-09T09:09:44.340270+0000","last_fullsized":"2023-10-09T09:09:44.340270+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":23,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.357392+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:09:44.357392+0000","last_peered":"2023-10-09T09:09:44.357392+0000","last_clean":"2023-10-09T09:09:44.357392+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:09:44.357392+0000","last_undegraded":"2023-10-09T09:09:44.357392+0000","last_fullsized":"2023-10-09T09:09:44.357392+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.339305+0000","last_change":"2023-10-09T08:59:53.522794+0000","last_active":"2023-10-09T09:09:44.339305+0000","last_peered":"2023-10-09T09:09:44.339305+0000","last_clean":"2023-10-09T09:09:44.339305+0000","last_became_active":"2023-10-09T08:59:53.522419+0000","last_became_peered":"2023-10-09T08:59:53.522419+0000","last_unstale":"2023-10-09T09:09:44.339305+0000","last_undegraded":"2023-10-09T09:09:44.339305+0000","last_fullsized":"2023-10-09T09:09:44.339305+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":167,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.349292+0000","last_change":"2023-10-09T09:09:06.657370+0000","last_active":"2023-10-09T09:09:44.349292+0000","last_peered":"2023-10-09T09:09:44.349292+0000","last_clean":"2023-10-09T09:09:44.349292+0000","last_became_active":"2023-10-09T09:09:06.656920+0000","last_became_peered":"2023-10-09T09:09:06.656920+0000","last_unstale":"2023-10-09T09:09:44.349292+0000","last_undegraded":"2023-10-09T09:09:44.349292+0000","last_fullsized":"2023-10-09T09:09:44.349292+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":206,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.334634+0000","last_change":"2023-10-09T09:09:06.665214+0000","last_active":"2023-10-09T09:09:44.334634+0000","last_peered":"2023-10-09T09:09:44.334634+0000","last_clean":"2023-10-09T09:09:44.334634+0000","last_became_active":"2023-10-09T09:09:06.664862+0000","last_became_peered":"2023-10-09T09:09:06.664862+0000","last_unstale":"2023-10-09T09:09:44.334634+0000","last_undegraded":"2023-10-09T09:09:44.334634+0000","last_fullsized":"2023-10-09T09:09:44.334634+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:29.278156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":308,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.347796+0000","last_change":"2023-10-09T09:08:24.737079+0000","last_active":"2023-10-09T09:09:44.347796+0000","last_peered":"2023-10-09T09:09:44.347796+0000","last_clean":"2023-10-09T09:09:44.347796+0000","last_became_active":"2023-10-09T09:08:24.656441+0000","last_became_peered":"2023-10-09T09:08:24.656441+0000","last_unstale":"2023-10-09T09:09:44.347796+0000","last_undegraded":"2023-10-09T09:09:44.347796+0000","last_fullsized":"2023-10-09T09:09:44.347796+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:36.720565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":111,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T09:09:44.339449+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:09:44.339449+0000","last_peered":"2023-10-09T09:09:44.339449+0000","last_clean":"2023-10-09T09:09:44.339449+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:09:44.339449+0000","last_undegraded":"2023-10-09T09:09:44.339449+0000","last_fullsized":"2023-10-09T09:09:44.339449+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":18386,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2628321304,"num_objects":40106,"num_object_clones":0,"num_object_copies":120318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4884,"num_objects_misplaced":44409,"num_objects_unfound":0,"num_objects_dirty":40106,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153062,"num_write_kb":6181313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6308,"num_bytes_recovered":413073408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8258273280,"data_stored":8258256992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114362,"ondisk_log_size":114362,"up":48,"acting":48,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430675,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":451984,"kb_used_data":154696,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93293168,"statfs":{"total":95995035648,"available":95532204032,"internally_reserved":0,"allocated":158408704,"data_stored":157899832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":4.3449999999999998,"15min":2.7309999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.6210000000000004,"5min":119.498,"15min":119.498},"last":4.1580000000000004},{"interface":"front","average":{"1min":1.131,"5min":4.3659999999999997,"15min":2.714},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.26400000000000001},"max":{"1min":5.633,"5min":119.532,"15min":119.532},"last":4.1500000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.81999999999999995,"15min":1.27},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.29699999999999999},"max":{"1min":5.3109999999999999,"5min":5.3109999999999999,"15min":25.893999999999998},"last":2.774},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.749,"15min":1.234},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.738,"5min":3.738,"15min":25.908000000000001},"last":3.177}]},{"osd":2,"last update":"Mon Oct 9 09:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":2.6669999999999998,"15min":2.2650000000000001},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.29299999999999998},"max":{"1min":7.8630000000000004,"5min":126.072,"15min":126.072},"last":2.8410000000000002},{"interface":"front","average":{"1min":1.6879999999999999,"5min":2.6320000000000001,"15min":2.3559999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":6.9539999999999997,"5min":120.729,"15min":120.729},"last":2.819}]},{"osd":3,"last update":"Mon Oct 9 09:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":3.1909999999999998,"15min":3.1469999999999998},"min":{"1min":0.42699999999999999,"5min":0.36899999999999999,"15min":0.32700000000000001},"max":{"1min":4.0309999999999997,"5min":63.095999999999997,"15min":191.30799999999999},"last":3.1520000000000001},{"interface":"front","average":{"1min":2.3100000000000001,"5min":3.1360000000000001,"15min":3.1859999999999999},"min":{"1min":0.47599999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.6779999999999999,"5min":62.610999999999997,"15min":191.47900000000001},"last":2.7970000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:09:47 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.0029999999999999,"15min":1.4299999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30199999999999999},"max":{"1min":9.2219999999999995,"5min":9.2219999999999995,"15min":138.34299999999999},"last":2.931},{"interface":"front","average":{"1min":1.008,"5min":0.98399999999999999,"15min":1.5149999999999999},"min":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.33100000000000002},"max":{"1min":4.0129999999999999,"5min":4.0129999999999999,"15min":137.04900000000001},"last":2.8610000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:09:47 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":0.93500000000000005,"15min":0.92000000000000004},"min":{"1min":0.38800000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.8659999999999997,"5min":4.8659999999999997,"15min":7.8120000000000003},"last":3.121},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.91400000000000003,"15min":0.92600000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":3.7360000000000002,"5min":3.7360000000000002,"15min":4.923},"last":2.9089999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:09:47 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.103,"15min":1.0940000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":2.319,"5min":11.268000000000001,"15min":11.268000000000001},"last":2.8809999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.2050000000000001,"15min":1.1819999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.34499999999999997},"max":{"1min":5.1699999999999999,"5min":5.1699999999999999,"15min":10.137},"last":2.742}]}]},{"osd":6,"up_from":42,"seq":180388626908,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2664076,"kb_used_data":1764804,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91081076,"statfs":{"total":95995035648,"available":93267021824,"internally_reserved":0,"allocated":1807159296,"data_stored":1806654892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2229999999999999,"5min":1.423,"15min":2.2930000000000001},"min":{"1min":0.379,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":7.0309999999999997,"5min":19.568999999999999,"15min":153.63800000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.3170000000000002,"5min":1.5109999999999999,"15min":2.4279999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.0499999999999998,"5min":19.498000000000001,"15min":153.72200000000001},"last":0.39100000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:10:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":1.1839999999999999,"15min":1.5369999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.29699999999999999},"max":{"1min":16.879000000000001,"5min":16.879000000000001,"15min":109.453},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.929,"5min":1.133,"15min":1.615},"min":{"1min":0.35799999999999998,"5min":0.315,"15min":0.314},"max":{"1min":6.5220000000000002,"5min":6.5220000000000002,"15min":111.47499999999999},"last":0.41399999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:09:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.5,"15min":1.359},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.1719999999999997,"5min":18.756,"15min":18.756},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.0960000000000001,"5min":1.472,"15min":1.52},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":6.6050000000000004,"5min":18.841999999999999,"15min":18.841999999999999},"last":6.2590000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:10:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9430000000000001,"5min":1.556,"15min":1.569},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":16.977,"5min":16.977,"15min":41.161000000000001},"last":2.8090000000000002},{"interface":"front","average":{"1min":2.851,"5min":1.954,"15min":1.74},"min":{"1min":0.45300000000000001,"5min":0.438,"15min":0.371},"max":{"1min":20.573,"5min":20.573,"15min":40.158000000000001},"last":2.4369999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8759999999999999,"5min":1.1619999999999999,"15min":1.196},"min":{"1min":0.35699999999999998,"5min":0.314,"15min":0.314},"max":{"1min":7.25,"5min":7.25,"15min":32.280999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.556,"5min":1.1539999999999999,"15min":1.1060000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":7.7460000000000004,"5min":7.7460000000000004,"15min":34.689},"last":1.224}]},{"osd":5,"last update":"Mon Oct 9 09:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.415,"15min":1.208},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.0519999999999996,"5min":8.0519999999999996,"15min":10.349},"last":1.234},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.268,"15min":1.0720000000000001},"min":{"1min":0.47999999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.6390000000000002,"5min":7.6390000000000002,"15min":7.6390000000000002},"last":1.1639999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.21,"15min":1.2470000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.8929999999999998,"5min":7.5149999999999997,"15min":11.382},"last":6.298},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.1240000000000001,"15min":1.0600000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.39000000000000001},"max":{"1min":7.718,"5min":7.718,"15min":9.4800000000000004},"last":1.196}]}]},{"osd":1,"up_from":0,"seq":356482285856,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449381,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2417016,"kb_used_data":1810096,"kb_used_omap":28,"kb_used_meta":606819,"kb_avail":91328136,"statfs":{"total":95995035648,"available":93520011264,"internally_reserved":0,"allocated":1853538304,"data_stored":1852643799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":621383504},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:09:01 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.048,"15min":1.484},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":3.798,"5min":5.2599999999999998,"15min":9.9879999999999995},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.179,"5min":0.998,"15min":1.4850000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.21199999999999999},"max":{"1min":4.4329999999999998,"5min":10.102,"15min":18.661000000000001},"last":0.68899999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3999999999999999,"5min":2.391,"15min":2.8210000000000002},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.5340000000000007,"5min":61.75,"15min":71.322999999999993},"last":0.71199999999999997},{"interface":"front","average":{"1min":2.8540000000000001,"5min":2.5030000000000001,"15min":3.73},"min":{"1min":0.377,"5min":0.33200000000000002,"15min":0.24399999999999999},"max":{"1min":13.035,"5min":61.718000000000004,"15min":219.00700000000001},"last":0.79500000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:09:54 2023","interfaces":[{"interface":"back","average":{"1min":3.5649999999999999,"5min":2.1800000000000002,"15min":3.0409999999999999},"min":{"1min":0.39300000000000002,"5min":0.34699999999999998,"15min":0.27700000000000002},"max":{"1min":9.2929999999999993,"5min":9.2929999999999993,"15min":219.041},"last":0.67900000000000005},{"interface":"front","average":{"1min":8.9559999999999995,"5min":3.0230000000000001,"15min":2.5390000000000001},"min":{"1min":0.33600000000000002,"5min":0.29599999999999999,"15min":0.25},"max":{"1min":137.94999999999999,"5min":137.94999999999999,"15min":137.94999999999999},"last":0.83899999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.236,"15min":1.4610000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":3.9470000000000001,"5min":7.0259999999999998,"15min":10.178000000000001},"last":3.5569999999999999},{"interface":"front","average":{"1min":3.0379999999999998,"5min":1.5389999999999999,"15min":1.7},"min":{"1min":0.46200000000000002,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":10.581,"5min":10.581,"15min":11.654},"last":0.35599999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.417,"15min":1.4970000000000001},"min":{"1min":0.42499999999999999,"5min":0.41299999999999998,"15min":0.37},"max":{"1min":8.6020000000000003,"5min":8.6020000000000003,"15min":9.3689999999999998},"last":4.4710000000000001},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.5169999999999999,"15min":1.6120000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.33700000000000002},"max":{"1min":16.364000000000001,"5min":16.364000000000001,"15min":16.364000000000001},"last":5.6479999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:09:46 2023","interfaces":[{"interface":"back","average":{"1min":6.7969999999999997,"5min":2.4990000000000001,"15min":2.1930000000000001},"min":{"1min":0.46200000000000002,"5min":0.39600000000000002,"15min":0.37},"max":{"1min":98.864000000000004,"5min":98.864000000000004,"15min":98.864000000000004},"last":3.6339999999999999},{"interface":"front","average":{"1min":7.008,"5min":2.5630000000000002,"15min":2.4249999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":98.878,"5min":98.878,"15min":98.878},"last":3.6160000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.2849999999999999,"15min":1.5620000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.42499999999999999},"max":{"1min":4.056,"5min":6.9550000000000001,"15min":10.157999999999999},"last":4.2969999999999997},{"interface":"front","average":{"1min":2.7450000000000001,"5min":1.554,"15min":1.8009999999999999},"min":{"1min":0.41799999999999998,"5min":0.40999999999999998,"15min":0.40100000000000002},"max":{"1min":10.619,"5min":10.619,"15min":14.359},"last":4.2809999999999997}]}]},{"osd":2,"up_from":88,"seq":377957122329,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1975004,"kb_used_data":1314708,"kb_used_omap":81,"kb_used_meta":660206,"kb_avail":91770148,"statfs":{"total":95995035648,"available":93972631552,"internally_reserved":0,"allocated":1346260992,"data_stored":1345333538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":676051162},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:09:45 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.8490000000000002,"15min":3.6139999999999999},"min":{"1min":0.377,"5min":0.188,"15min":0.188},"max":{"1min":8.6869999999999994,"5min":68.457999999999998,"15min":134.458},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.7170000000000001,"5min":2.8559999999999999,"15min":3.298},"min":{"1min":0.254,"5min":0.22700000000000001,"15min":0.221},"max":{"1min":9.5790000000000006,"5min":68.162999999999997,"15min":138.417},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:09:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":1.1559999999999999,"15min":2.266},"min":{"1min":0.27800000000000002,"5min":0.214,"15min":0.21299999999999999},"max":{"1min":5.7130000000000001,"5min":5.7130000000000001,"15min":57.939999999999998},"last":1.8680000000000001},{"interface":"front","average":{"1min":1.635,"5min":1.0980000000000001,"15min":2.0600000000000001},"min":{"1min":0.32200000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.4059999999999997,"5min":7.4059999999999997,"15min":57.122999999999998},"last":0.52800000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:09:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9350000000000001,"5min":2.7320000000000002,"15min":3.157},"min":{"1min":0.52700000000000002,"5min":0.34699999999999998,"15min":0.29399999999999998},"max":{"1min":10.861000000000001,"5min":73.221999999999994,"15min":112.47499999999999},"last":9.468},{"interface":"front","average":{"1min":1.7629999999999999,"5min":2.5059999999999998,"15min":3.141},"min":{"1min":0.27100000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.0389999999999997,"5min":75.840999999999994,"15min":112.929},"last":0.61199999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.6120000000000001,"15min":1.8089999999999999},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":6.133,"5min":17.408999999999999,"15min":17.408999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.2949999999999999,"5min":1.468,"15min":1.7949999999999999},"min":{"1min":0.39400000000000002,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":7.734,"5min":7.734,"15min":14.071999999999999},"last":2.2559999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":2.085,"15min":1.6699999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.3070000000000004,"5min":72.311000000000007,"15min":72.311000000000007},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.5609999999999999,"15min":2.1240000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":6.0099999999999998,"5min":73.042000000000002,"15min":73.042000000000002},"last":2.2709999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:09:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":1.5069999999999999,"15min":1.8109999999999999},"min":{"1min":0.64800000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.2859999999999996,"5min":7.2859999999999996,"15min":10.456},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.806,"5min":1.427,"15min":1.5209999999999999},"min":{"1min":0.43099999999999999,"5min":0.40999999999999998,"15min":0.39300000000000002},"max":{"1min":7.3789999999999996,"5min":7.3789999999999996,"15min":12.381},"last":0.82699999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":2.0899999999999999,"15min":1.9930000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.7309999999999999,"5min":67.491,"15min":67.491},"last":0.69699999999999995},{"interface":"front","average":{"1min":2.0409999999999999,"5min":1.6020000000000001,"15min":1.5820000000000001},"min":{"1min":0.495,"5min":0.44400000000000001,"15min":0.433},"max":{"1min":7.4329999999999998,"5min":7.4329999999999998,"15min":12.438000000000001},"last":0.89700000000000002}]}]},{"osd":3,"up_from":93,"seq":399431958804,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2735952,"kb_used_data":2119240,"kb_used_omap":7,"kb_used_meta":616696,"kb_avail":91009200,"statfs":{"total":95995035648,"available":93193420800,"internally_reserved":0,"allocated":2170101760,"data_stored":2169243122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":631497320},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.714,"5min":3.9889999999999999,"15min":2.4420000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.19400000000000001},"max":{"1min":9.2859999999999996,"5min":233.22900000000001,"15min":233.22900000000001},"last":2.444},{"interface":"front","average":{"1min":1.5109999999999999,"5min":4.5899999999999999,"15min":2.7989999999999999},"min":{"1min":0.29399999999999998,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":7.0949999999999998,"5min":237.447,"15min":237.447},"last":1.806}]},{"osd":1,"last update":"Mon Oct 9 09:09:56 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.1459999999999999,"15min":2.6589999999999998},"min":{"1min":0.26300000000000001,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":2.452,"5min":7.1500000000000004,"15min":326.75700000000001},"last":10.662000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":1.0009999999999999,"15min":2.5659999999999998},"min":{"1min":0.35499999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.4889999999999999,"5min":7.194,"15min":327.024},"last":0.41599999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.9390000000000001,"15min":2.5139999999999998},"min":{"1min":0.41999999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":3.2480000000000002,"5min":36.453000000000003,"15min":75.641999999999996},"last":5.4530000000000003},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.1349999999999998,"15min":2.4279999999999999},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.27000000000000002},"max":{"1min":3.98,"5min":36.764000000000003,"15min":75.207999999999998},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 09:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.284,"15min":1.3420000000000001},"min":{"1min":0.42899999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.399,"5min":7.2380000000000004,"15min":9.2940000000000005},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.831,"15min":1.722},"min":{"1min":0.34499999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.9319999999999999,"5min":10.505000000000001,"15min":10.505000000000001},"last":5.7350000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":2.101,"15min":1.774},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.33700000000000002},"max":{"1min":4.8760000000000003,"5min":16.132999999999999,"15min":16.132999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.4099999999999999,"15min":1.532},"min":{"1min":0.49099999999999999,"5min":0.41399999999999998,"15min":0.35399999999999998},"max":{"1min":2.8420000000000001,"5min":7.056,"15min":7.5590000000000002},"last":14.361000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:09:56 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":2.1749999999999998,"15min":1.8919999999999999},"min":{"1min":0.495,"5min":0.375,"15min":0.308},"max":{"1min":5.9299999999999997,"5min":12.105,"15min":12.105},"last":0.69899999999999995},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.696,"15min":1.7689999999999999},"min":{"1min":0.51700000000000002,"5min":0.38500000000000001,"15min":0.35099999999999998},"max":{"1min":5.8529999999999998,"5min":7.0890000000000004,"15min":9.5090000000000003},"last":5.532}]},{"osd":7,"last update":"Mon Oct 9 09:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.603,"15min":1.649},"min":{"1min":0.61699999999999999,"5min":0.41499999999999998,"15min":0.41299999999999998},"max":{"1min":5.431,"5min":7.1219999999999999,"15min":8.9130000000000003},"last":5.7809999999999997},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.4299999999999999,"15min":1.484},"min":{"1min":0.51800000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.4530000000000003,"5min":7.7999999999999998,"15min":9.2509999999999994},"last":0.67700000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019374,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1850896,"kb_used_data":952712,"kb_used_omap":0,"kb_used_meta":898176,"kb_avail":91894256,"statfs":{"total":95995035648,"available":94099718144,"internally_reserved":0,"allocated":975577088,"data_stored":975068413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919732224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:09:27 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.7170000000000001,"15min":2.3399999999999999},"min":{"1min":0.28199999999999997,"5min":0.248,"15min":0.248},"max":{"1min":4.109,"5min":46.463000000000001,"15min":141.07400000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.659,"5min":2.016,"15min":2.4550000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":9.3109999999999999,"5min":46.823999999999998,"15min":141.15600000000001},"last":6.4660000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:09:58 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.80900000000000005,"15min":1.085},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.29899999999999999},"max":{"1min":5.3140000000000001,"5min":5.3140000000000001,"15min":7.5780000000000003},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.79700000000000004,"15min":1.2230000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29199999999999998},"max":{"1min":1.631,"5min":3.9729999999999999,"15min":7.6879999999999997},"last":1.1659999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:09:27 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.121,"15min":1.992},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":9.9190000000000005,"5min":9.9190000000000005,"15min":142.816},"last":1.145},{"interface":"front","average":{"1min":1.55,"5min":1.335,"15min":1.827},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.3949999999999996,"5min":7.8449999999999998,"15min":143.81800000000001},"last":5.0800000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:09:58 2023","interfaces":[{"interface":"back","average":{"1min":6.4699999999999998,"5min":7.4900000000000002,"15min":3.9409999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":104.45,"5min":165.995,"15min":165.995},"last":3.7869999999999999},{"interface":"front","average":{"1min":6.4690000000000003,"5min":7.9050000000000002,"15min":4.2110000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":104.697,"5min":165.53200000000001,"15min":165.53200000000001},"last":1.224}]},{"osd":5,"last update":"Mon Oct 9 09:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.93500000000000005,"15min":1.02},"min":{"1min":0.35699999999999998,"5min":0.35099999999999998,"15min":0.33000000000000002},"max":{"1min":2.2250000000000001,"5min":5.3639999999999999,"15min":7.7439999999999998},"last":2.681},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.97099999999999997,"15min":1.0489999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0510000000000002,"5min":4.1379999999999999,"15min":8.6419999999999995},"last":1.331}]},{"osd":6,"last update":"Mon Oct 9 09:09:50 2023","interfaces":[{"interface":"back","average":{"1min":4.9489999999999998,"5min":1.754,"15min":1.3089999999999999},"min":{"1min":0.376,"5min":0.36799999999999999,"15min":0.32100000000000001},"max":{"1min":74.700999999999993,"5min":74.700999999999993,"15min":74.700999999999993},"last":1.014},{"interface":"front","average":{"1min":4.5739999999999998,"5min":1.6990000000000001,"15min":1.4059999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":73.572999999999993,"5min":73.572999999999993,"15min":73.572999999999993},"last":2.722}]},{"osd":7,"last update":"Mon Oct 9 09:09:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.98699999999999999,"15min":1.1459999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.32800000000000001},"max":{"1min":3.085,"5min":3.9039999999999999,"15min":11.303000000000001},"last":5.0460000000000003},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.96299999999999997,"15min":1.1339999999999999},"min":{"1min":0.46400000000000002,"5min":0.42199999999999999,"15min":0.36199999999999999},"max":{"1min":2.5099999999999998,"5min":4.984,"15min":10.698},"last":0.92700000000000005}]}]},{"osd":5,"up_from":36,"seq":154618823142,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325956,"kb_used_data":16444,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93419196,"statfs":{"total":95995035648,"available":95661256704,"internally_reserved":0,"allocated":16838656,"data_stored":16338035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:09:34 2023","interfaces":[{"interface":"back","average":{"1min":4.5700000000000003,"5min":3.8069999999999999,"15min":3.6579999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27500000000000002},"max":{"1min":56.823999999999998,"5min":156.97800000000001,"15min":192.55799999999999},"last":3.3250000000000002},{"interface":"front","average":{"1min":4.4470000000000001,"5min":3.9220000000000002,"15min":3.7690000000000001},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.29699999999999999},"max":{"1min":60.362000000000002,"5min":157.45699999999999,"15min":194.11000000000001},"last":7.9669999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.83599999999999997,"15min":1.2989999999999999},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.28699999999999998},"max":{"1min":5.7640000000000002,"5min":5.7640000000000002,"15min":36.963999999999999},"last":3.355},{"interface":"front","average":{"1min":1.2410000000000001,"5min":0.86599999999999999,"15min":1.5409999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":6.9359999999999999,"5min":6.9359999999999999,"15min":36.158000000000001},"last":0.56699999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":3.0089999999999999,"15min":2.3999999999999999},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":6.4219999999999997,"5min":156.69499999999999,"15min":156.69499999999999},"last":3.399},{"interface":"front","average":{"1min":1.01,"5min":2.641,"15min":2.2229999999999999},"min":{"1min":0.32100000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.524,"5min":156.87799999999999,"15min":156.87799999999999},"last":3.4380000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:09:49 2023","interfaces":[{"interface":"back","average":{"1min":3.883,"5min":2.633,"15min":3.0680000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":57.963000000000001,"5min":101.121,"15min":161.678},"last":0.54200000000000004},{"interface":"front","average":{"1min":4.5670000000000002,"5min":2.9929999999999999,"15min":3.129},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":57.603000000000002,"5min":101.164,"15min":159.43899999999999},"last":0.79900000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.026,"15min":1.077},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":5.7119999999999997,"5min":5.7119999999999997,"15min":9.0229999999999997},"last":4.6139999999999999},{"interface":"front","average":{"1min":0.875,"5min":1.03,"15min":1.077},"min":{"1min":0.373,"5min":0.373,"15min":0.33900000000000002},"max":{"1min":3.5569999999999999,"5min":5.0209999999999999,"15min":7.5309999999999997},"last":3.0390000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:09:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1429999999999998,"5min":1.4510000000000001,"15min":1.256},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":16.052,"5min":16.052,"15min":16.052},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.47,"5min":1.264,"15min":1.224},"min":{"1min":0.47899999999999998,"5min":0.46100000000000002,"15min":0.38600000000000001},"max":{"1min":4.7320000000000002,"5min":4.883,"15min":10.788},"last":3.6150000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:09:49 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.0129999999999999,"15min":1.012},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":2.6400000000000001,"5min":4.9580000000000002,"15min":11.237},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.8,"5min":1.133,"15min":1.0960000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.41299999999999998},"max":{"1min":6.9790000000000001,"5min":6.9790000000000001,"15min":7.0860000000000003},"last":2.9900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1830883328,"data_stored":1830879256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1329725440,"data_stored":1329725440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2163802112,"data_stored":2163802112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":965545984,"data_stored":965541912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":14618624,"data_stored":14614552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1795362816,"data_stored":1795358744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":158334976,"data_stored":158334976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:10:14.147 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:10:14.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-10-09T09:10:14.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:14.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:14.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:14.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:14.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:14.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:14.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:14.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:14.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:14.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:14.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:14.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:14.606 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:14.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:14.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:14.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:14.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:14.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:14.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:14.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:14.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:14.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:14.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:15.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:15.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:15.125 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:15.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:15.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:15.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:15.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:15.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:15.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:15.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:15.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:15.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:15.532 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:15.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:15.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:15.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:15.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:15.706 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:15 smithi094 bash[59677]: cluster 2023-10-09T09:10:13.995707+0000 mgr.y (mgr.25421) 987 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 4486/109185 objects degraded (4.109%); 39397/109185 objects misplaced (36.083%); 13 MiB/s, 201 objects/s recovering 2023-10-09T09:10:15.706 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:15 smithi094 bash[59677]: cluster 2023-10-09T09:10:14.651356+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 4486/109185 objects degraded (4.109%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:15.706 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:15 smithi094 bash[59677]: audit 2023-10-09T09:10:15.041707+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:15.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:15 smithi016 bash[75179]: cluster 2023-10-09T09:10:13.995707+0000 mgr.y (mgr.25421) 987 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 4486/109185 objects degraded (4.109%); 39397/109185 objects misplaced (36.083%); 13 MiB/s, 201 objects/s recovering 2023-10-09T09:10:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:15 smithi016 bash[75179]: cluster 2023-10-09T09:10:14.651356+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 4486/109185 objects degraded (4.109%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:15.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:15 smithi016 bash[75179]: audit 2023-10-09T09:10:15.041707+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:15.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:15 smithi016 bash[77892]: cluster 2023-10-09T09:10:13.995707+0000 mgr.y (mgr.25421) 987 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 4486/109185 objects degraded (4.109%); 39397/109185 objects misplaced (36.083%); 13 MiB/s, 201 objects/s recovering 2023-10-09T09:10:15.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:15 smithi016 bash[77892]: cluster 2023-10-09T09:10:14.651356+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 4486/109185 objects degraded (4.109%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:15.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:15 smithi016 bash[77892]: audit 2023-10-09T09:10:15.041707+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:15.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:15.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:15.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:15.919 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:10:15.919 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:10:15.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:10:15.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:15.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:15.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:10:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:10:15] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.33.4" 2023-10-09T09:10:16.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:16.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:16.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:16.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:16.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:10:16.237 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:16.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:16.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:16.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:16.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:16.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:16.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:16.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:16.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:16.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:16.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:16.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:16.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:16.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:16.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:16.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:16.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:16.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:16.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:17.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:17.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:10:17.091 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:17.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:17.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:17.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:17.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:17.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:17.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:17.359 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:17.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:17.464 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:17.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:17.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:17.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:17.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:17.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:17.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:17.731 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:17 smithi094 bash[59677]: cluster 2023-10-09T09:10:15.996483+0000 mgr.y (mgr.25421) 988 : cluster 0 pgmap v1046: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 4351/107748 objects degraded (4.038%); 38624/107748 objects misplaced (35.847%); 17 MiB/s, 274 objects/s recovering 2023-10-09T09:10:17.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:17 smithi016 bash[75179]: cluster 2023-10-09T09:10:15.996483+0000 mgr.y (mgr.25421) 988 : cluster 0 pgmap v1046: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 4351/107748 objects degraded (4.038%); 38624/107748 objects misplaced (35.847%); 17 MiB/s, 274 objects/s recovering 2023-10-09T09:10:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:17 smithi016 bash[77892]: cluster 2023-10-09T09:10:15.996483+0000 mgr.y (mgr.25421) 988 : cluster 0 pgmap v1046: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 4351/107748 objects degraded (4.038%); 38624/107748 objects misplaced (35.847%); 17 MiB/s, 274 objects/s recovering 2023-10-09T09:10:17.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:17.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:17.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:17.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:17.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:18.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:18.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:18.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:18.164 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:18.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:18.301 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:18.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:18.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:18.430 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:18.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:18.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:18.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:18.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:18.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:18.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:18.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:18.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:18.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:18.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:18.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:18.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:18.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:19.012 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:10:19.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:19.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:19.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:19.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:19.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:19.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:19.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:19.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:19.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:19.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:19.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:19.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:19.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:19 smithi016 bash[75179]: cluster 2023-10-09T09:10:18.000495+0000 mgr.y (mgr.25421) 989 : cluster 0 pgmap v1047: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 4351/106368 objects degraded (4.091%); 38280/106368 objects misplaced (35.988%); 15 MiB/s, 235 objects/s recovering 2023-10-09T09:10:19.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:19 smithi016 bash[75179]: audit 2023-10-09T09:10:19.241902+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:19.496 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:19 smithi016 bash[77892]: cluster 2023-10-09T09:10:18.000495+0000 mgr.y (mgr.25421) 989 : cluster 0 pgmap v1047: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 4351/106368 objects degraded (4.091%); 38280/106368 objects misplaced (35.988%); 15 MiB/s, 235 objects/s recovering 2023-10-09T09:10:19.496 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:19 smithi016 bash[77892]: audit 2023-10-09T09:10:19.241902+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:19.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:19.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:19.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:19.626 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:19.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:19.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:19.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:19 smithi094 bash[59677]: cluster 2023-10-09T09:10:18.000495+0000 mgr.y (mgr.25421) 989 : cluster 0 pgmap v1047: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 4351/106368 objects degraded (4.091%); 38280/106368 objects misplaced (35.988%); 15 MiB/s, 235 objects/s recovering 2023-10-09T09:10:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:19 smithi094 bash[59677]: audit 2023-10-09T09:10:19.241902+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:19.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:19.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:19.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:19.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:19.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:19.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:20.032 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:10:20.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:20.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:20.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:20.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:20.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:20.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:20.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:20.332 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:20.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:20.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:20.469 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:20 smithi094 bash[59677]: cluster 2023-10-09T09:10:19.652661+0000 mon.a (mon.0) 1323 : cluster 3 Health check update: Degraded data redundancy: 4351/106368 objects degraded (4.091%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:20.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:20.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:20.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:20.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:20.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:20.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:20.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:20.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:20.748 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:20 smithi016 bash[77892]: cluster 2023-10-09T09:10:19.652661+0000 mon.a (mon.0) 1323 : cluster 3 Health check update: Degraded data redundancy: 4351/106368 objects degraded (4.091%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:20.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:20 smithi016 bash[75179]: cluster 2023-10-09T09:10:19.652661+0000 mon.a (mon.0) 1323 : cluster 3 Health check update: Degraded data redundancy: 4351/106368 objects degraded (4.091%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:20.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:20.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:20.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:20.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:20.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:20.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:20.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:21.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:21.045 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:21.052 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:10:21.150 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:21.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:21.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:21.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:21.414 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:21.452 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:21.470 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:21 smithi016 bash[75179]: cluster 2023-10-09T09:10:20.001181+0000 mgr.y (mgr.25421) 990 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 4208/101673 objects degraded (4.139%); 36301/101673 objects misplaced (35.704%); 15 MiB/s, 233 objects/s recovering 2023-10-09T09:10:21.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:21.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:21.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:21.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:21.724 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:21 smithi016 bash[77892]: cluster 2023-10-09T09:10:20.001181+0000 mgr.y (mgr.25421) 990 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 4208/101673 objects degraded (4.139%); 36301/101673 objects misplaced (35.704%); 15 MiB/s, 233 objects/s recovering 2023-10-09T09:10:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:21 smithi094 bash[59677]: cluster 2023-10-09T09:10:20.001181+0000 mgr.y (mgr.25421) 990 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 4208/101673 objects degraded (4.139%); 36301/101673 objects misplaced (35.704%); 15 MiB/s, 233 objects/s recovering 2023-10-09T09:10:21.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:21.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:21.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:21.945 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:10:21.961 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:21.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:21.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:22.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:22.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:22.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:22.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:22.226 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:22.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:22.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:22.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:22.357 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:22.389 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:22.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:22.492 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:22.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:22.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:22.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:22.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:22.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:22.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:22.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:22.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:22.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:22.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:22.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:22.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[77892]: cluster 2023-10-09T09:10:21.662614+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[77892]: cluster 2023-10-09T09:10:21.666087+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T09:10:22.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[77892]: cluster 2023-10-09T09:10:21.666114+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[77892]: cluster 2023-10-09T09:10:21.666310+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[77892]: cluster 2023-10-09T09:10:22.001636+0000 mgr.y (mgr.25421) 991 : cluster 0 pgmap v1050: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 4041/98841 objects degraded (4.088%); 35267/98841 objects misplaced (35.681%); 13 MiB/s, 211 objects/s recovering 2023-10-09T09:10:22.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[75179]: cluster 2023-10-09T09:10:21.662614+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[75179]: cluster 2023-10-09T09:10:21.666087+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T09:10:22.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[75179]: cluster 2023-10-09T09:10:21.666114+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[75179]: cluster 2023-10-09T09:10:21.666310+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:22 smithi016 bash[75179]: cluster 2023-10-09T09:10:22.001636+0000 mgr.y (mgr.25421) 991 : cluster 0 pgmap v1050: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 4041/98841 objects degraded (4.088%); 35267/98841 objects misplaced (35.681%); 13 MiB/s, 211 objects/s recovering 2023-10-09T09:10:22.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:22.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:22.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:22 smithi094 bash[59677]: cluster 2023-10-09T09:10:21.662614+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:22 smithi094 bash[59677]: cluster 2023-10-09T09:10:21.666087+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T09:10:22.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:22 smithi094 bash[59677]: cluster 2023-10-09T09:10:21.666114+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:22 smithi094 bash[59677]: cluster 2023-10-09T09:10:21.666310+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T09:10:22.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:22 smithi094 bash[59677]: cluster 2023-10-09T09:10:22.001636+0000 mgr.y (mgr.25421) 991 : cluster 0 pgmap v1050: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 4041/98841 objects degraded (4.088%); 35267/98841 objects misplaced (35.681%); 13 MiB/s, 211 objects/s recovering 2023-10-09T09:10:23.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:23.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:23.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:23.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:23.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:23.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:23.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:23.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:23.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:23.430 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:23.457 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:23.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:23.560 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:23.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:23.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:23.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:23.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:23.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:23.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:23 smithi094 bash[59677]: cluster 2023-10-09T09:10:22.672937+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:23 smithi094 bash[59677]: cluster 2023-10-09T09:10:22.673833+0000 mon.a (mon.0) 1325 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T09:10:23.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:23 smithi094 bash[59677]: cluster 2023-10-09T09:10:22.674500+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:23.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:23 smithi094 bash[59677]: cluster 2023-10-09T09:10:22.688266+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:23.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:23 smithi094 bash[59677]: cluster 2023-10-09T09:10:22.728826+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:23.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:23.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:23.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:23.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:24.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[77892]: cluster 2023-10-09T09:10:22.672937+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[77892]: cluster 2023-10-09T09:10:22.673833+0000 mon.a (mon.0) 1325 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T09:10:24.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[77892]: cluster 2023-10-09T09:10:22.674500+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[77892]: cluster 2023-10-09T09:10:22.688266+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[77892]: cluster 2023-10-09T09:10:22.728826+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[75179]: cluster 2023-10-09T09:10:22.672937+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[75179]: cluster 2023-10-09T09:10:22.673833+0000 mon.a (mon.0) 1325 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T09:10:24.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[75179]: cluster 2023-10-09T09:10:22.674500+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[75179]: cluster 2023-10-09T09:10:22.688266+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:23 smithi016 bash[75179]: cluster 2023-10-09T09:10:22.728826+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T09:10:24.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:24.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:24.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:24.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:24.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:24.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:24.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:24.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:24.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:24.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:24.681 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:24 smithi094 bash[59677]: cluster 2023-10-09T09:10:24.002299+0000 mgr.y (mgr.25421) 992 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 3910/94527 objects degraded (4.136%); 33426/94527 objects misplaced (35.361%); 10 MiB/s, 162 objects/s recovering 2023-10-09T09:10:24.682 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:24 smithi094 bash[59677]: cluster 2023-10-09T09:10:24.653837+0000 mon.a (mon.0) 1326 : cluster 3 Health check update: Degraded data redundancy: 4041/98841 objects degraded (4.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:24.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:24.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:24.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:24.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:24.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:24.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:24.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:24.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:24.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:24 smithi016 bash[75179]: cluster 2023-10-09T09:10:24.002299+0000 mgr.y (mgr.25421) 992 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 3910/94527 objects degraded (4.136%); 33426/94527 objects misplaced (35.361%); 10 MiB/s, 162 objects/s recovering 2023-10-09T09:10:24.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:24 smithi016 bash[75179]: cluster 2023-10-09T09:10:24.653837+0000 mon.a (mon.0) 1326 : cluster 3 Health check update: Degraded data redundancy: 4041/98841 objects degraded (4.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:24.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:24 smithi016 bash[77892]: cluster 2023-10-09T09:10:24.002299+0000 mgr.y (mgr.25421) 992 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 3910/94527 objects degraded (4.136%); 33426/94527 objects misplaced (35.361%); 10 MiB/s, 162 objects/s recovering 2023-10-09T09:10:25.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:24 smithi016 bash[77892]: cluster 2023-10-09T09:10:24.653837+0000 mon.a (mon.0) 1326 : cluster 3 Health check update: Degraded data redundancy: 4041/98841 objects degraded (4.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:25.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:25.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:25.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:25.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:25.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:25.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:25.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:25.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:25.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:25.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:25.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:25.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:25.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:25.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:25.633 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:25.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:25.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:25.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:25.792 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:10:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:10:25] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.33.4" 2023-10-09T09:10:25.802 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:25.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:25.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:25.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:25.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:25.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:25.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:26.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:26.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:26.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:26.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:26.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:26.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:26.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:26.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:26.347 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:26.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:26.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:26.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:26.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:26.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:26.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:26.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:26.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:26.670 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:10:26.670 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:10:26.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:26.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:26.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:26.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:26.902 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:26.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:27.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:27.038 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:27.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:27 smithi016 bash[77892]: cluster 2023-10-09T09:10:26.002893+0000 mgr.y (mgr.25421) 993 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 3748/93189 objects degraded (4.022%); 33412/93189 objects misplaced (35.854%); 12 MiB/s, 184 objects/s recovering 2023-10-09T09:10:27.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:27 smithi016 bash[75179]: cluster 2023-10-09T09:10:26.002893+0000 mgr.y (mgr.25421) 993 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 3748/93189 objects degraded (4.022%); 33412/93189 objects misplaced (35.854%); 12 MiB/s, 184 objects/s recovering 2023-10-09T09:10:27.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:27.151 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:27 smithi094 bash[59677]: cluster 2023-10-09T09:10:26.002893+0000 mgr.y (mgr.25421) 993 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 3748/93189 objects degraded (4.022%); 33412/93189 objects misplaced (35.854%); 12 MiB/s, 184 objects/s recovering 2023-10-09T09:10:27.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:27.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:27.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:27.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:27.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:27.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:27.554 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:27.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:27.693 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:27.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:27.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:27.800 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:10:27.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:27.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:27.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:27.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:27.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:27.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:28.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:28.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:28.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:28.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:28.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:28.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:28.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:28.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:28.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:28.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:28.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:28.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:28.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:28.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:28.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:28.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:28.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:28.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:28.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:28.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:28.903 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:28.936 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:28.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:29.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:29.063 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:29 smithi094 bash[59677]: cluster 2023-10-09T09:10:28.003811+0000 mgr.y (mgr.25421) 994 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 409 op/s; 3748/91833 objects degraded (4.081%); 33090/91833 objects misplaced (36.033%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T09:10:29.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:29.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:29.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:29.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:29.201 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:29.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:29.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:29.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:29 smithi016 bash[75179]: cluster 2023-10-09T09:10:28.003811+0000 mgr.y (mgr.25421) 994 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 409 op/s; 3748/91833 objects degraded (4.081%); 33090/91833 objects misplaced (36.033%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T09:10:29.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:29 smithi016 bash[77892]: cluster 2023-10-09T09:10:28.003811+0000 mgr.y (mgr.25421) 994 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 409 op/s; 3748/91833 objects degraded (4.081%); 33090/91833 objects misplaced (36.033%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T09:10:29.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:29.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:29.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:29.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:29.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:29.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:29.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:29.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:29.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:29.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:29.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:29.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:29.858 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:29.891 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:29.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:29.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:30.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:30.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:30.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:30.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:30.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:30.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:30 smithi094 bash[59677]: cluster 2023-10-09T09:10:29.654857+0000 mon.a (mon.0) 1327 : cluster 3 Health check update: Degraded data redundancy: 3748/91833 objects degraded (4.081%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:30.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:30.295 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:30 smithi016 bash[75179]: cluster 2023-10-09T09:10:29.654857+0000 mon.a (mon.0) 1327 : cluster 3 Health check update: Degraded data redundancy: 3748/91833 objects degraded (4.081%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:30.295 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:30 smithi016 bash[77892]: cluster 2023-10-09T09:10:29.654857+0000 mon.a (mon.0) 1327 : cluster 3 Health check update: Degraded data redundancy: 3748/91833 objects degraded (4.081%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:30.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:30.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:30.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:30.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:30.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:30.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:30.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:30.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:30.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:30.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:30.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:30.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:30.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:30.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:30.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:30.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:30.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:30.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:31.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:31.135 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:31.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:31.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:31.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:31 smithi016 bash[75179]: cluster 2023-10-09T09:10:30.004672+0000 mgr.y (mgr.25421) 995 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 3599/87357 objects degraded (4.120%); 31197/87357 objects misplaced (35.712%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:10:31.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:31 smithi016 bash[77892]: cluster 2023-10-09T09:10:30.004672+0000 mgr.y (mgr.25421) 995 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 3599/87357 objects degraded (4.120%); 31197/87357 objects misplaced (35.712%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:10:31.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:31.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:31.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:31.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:31.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:31 smithi094 bash[59677]: cluster 2023-10-09T09:10:30.004672+0000 mgr.y (mgr.25421) 995 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 3599/87357 objects degraded (4.120%); 31197/87357 objects misplaced (35.712%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:10:31.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:31.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:31.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:31.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:31.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:31.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:31.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:31.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:31.930 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:31.967 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:31.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:31.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:32.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:32.101 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:32.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:32.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:32.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:32.238 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:32.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:32.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:32.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:32.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:32.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:32.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:32.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:32.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:32.570 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:10:32.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:32.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:32.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:32.683 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:10:32.745 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:10:32.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:32.788 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:10:32.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:32.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:32.828 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:10:32.866 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:10:32.904 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:10:32.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:32.951 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:10:32.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:32.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:32.987 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:10:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:10:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:10:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:10:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:10:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:10:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:10:32.995 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:10:32.996 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:10:33.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:33.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:33.093 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 9 pg_num to 26 2023-10-09T09:10:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:33.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:33.096 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:10:33.196 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:33.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:33.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:33 smithi094 bash[59677]: cluster 2023-10-09T09:10:32.005571+0000 mgr.y (mgr.25421) 996 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 3457/84483 objects degraded (4.092%); 30891/84483 objects misplaced (36.565%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:10:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:33 smithi094 bash[59677]: audit 2023-10-09T09:10:32.627054+0000 mon.a (mon.0) 1328 : audit 0 from='client.? 172.21.15.16:0/1084472290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:33 smithi094 bash[59677]: audit 2023-10-09T09:10:32.833819+0000 mon.a (mon.0) 1329 : audit 1 from='client.? 172.21.15.16:0/251281331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:10:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:33 smithi094 bash[59677]: audit 2023-10-09T09:10:33.036652+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.16:0/1629552459' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:10:33.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:33 smithi094 bash[59677]: audit 2023-10-09T09:10:33.038939+0000 mon.a (mon.0) 1330 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:10:33.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[75179]: cluster 2023-10-09T09:10:32.005571+0000 mgr.y (mgr.25421) 996 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 3457/84483 objects degraded (4.092%); 30891/84483 objects misplaced (36.565%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:10:33.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[75179]: audit 2023-10-09T09:10:32.627054+0000 mon.a (mon.0) 1328 : audit 0 from='client.? 172.21.15.16:0/1084472290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:33.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[75179]: audit 2023-10-09T09:10:32.833819+0000 mon.a (mon.0) 1329 : audit 1 from='client.? 172.21.15.16:0/251281331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:10:33.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[75179]: audit 2023-10-09T09:10:33.036652+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.16:0/1629552459' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:10:33.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[75179]: audit 2023-10-09T09:10:33.038939+0000 mon.a (mon.0) 1330 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:10:33.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[77892]: cluster 2023-10-09T09:10:32.005571+0000 mgr.y (mgr.25421) 996 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 3457/84483 objects degraded (4.092%); 30891/84483 objects misplaced (36.565%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:10:33.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[77892]: audit 2023-10-09T09:10:32.627054+0000 mon.a (mon.0) 1328 : audit 0 from='client.? 172.21.15.16:0/1084472290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:33.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[77892]: audit 2023-10-09T09:10:32.833819+0000 mon.a (mon.0) 1329 : audit 1 from='client.? 172.21.15.16:0/251281331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:10:33.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[77892]: audit 2023-10-09T09:10:33.036652+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.16:0/1629552459' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:10:33.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:33 smithi016 bash[77892]: audit 2023-10-09T09:10:33.038939+0000 mon.a (mon.0) 1330 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:10:33.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:33.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:33.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:33.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:33.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:33.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:33.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:33.608 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:33.633 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:33.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:33.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:33.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:33.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:33.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:33.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:33.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:33.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:33.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:33.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:34.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:34.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:34.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:34.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:34.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:34.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:34.316 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[75179]: cluster 2023-10-09T09:10:33.070419+0000 mon.a (mon.0) 1331 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:10:34.316 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[75179]: cluster 2023-10-09T09:10:33.081744+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T09:10:34.316 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[75179]: audit 2023-10-09T09:10:33.086797+0000 mon.a (mon.0) 1332 : audit 1 from='client.? 172.21.15.16:0/251281331' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:10:34.316 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[75179]: audit 2023-10-09T09:10:33.087043+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:10:34.316 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[75179]: cluster 2023-10-09T09:10:33.087324+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T09:10:34.316 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[75179]: audit 2023-10-09T09:10:34.007578+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:10:34.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[77892]: cluster 2023-10-09T09:10:33.070419+0000 mon.a (mon.0) 1331 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:10:34.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[77892]: cluster 2023-10-09T09:10:33.081744+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T09:10:34.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[77892]: audit 2023-10-09T09:10:33.086797+0000 mon.a (mon.0) 1332 : audit 1 from='client.? 172.21.15.16:0/251281331' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:10:34.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[77892]: audit 2023-10-09T09:10:33.087043+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:10:34.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[77892]: cluster 2023-10-09T09:10:33.087324+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T09:10:34.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:34 smithi016 bash[77892]: audit 2023-10-09T09:10:34.007578+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:10:34.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:34 smithi094 bash[59677]: cluster 2023-10-09T09:10:33.070419+0000 mon.a (mon.0) 1331 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:10:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:34 smithi094 bash[59677]: cluster 2023-10-09T09:10:33.081744+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T09:10:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:34 smithi094 bash[59677]: audit 2023-10-09T09:10:33.086797+0000 mon.a (mon.0) 1332 : audit 1 from='client.? 172.21.15.16:0/251281331' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:10:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:34 smithi094 bash[59677]: audit 2023-10-09T09:10:33.087043+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:10:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:34 smithi094 bash[59677]: cluster 2023-10-09T09:10:33.087324+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T09:10:34.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:34 smithi094 bash[59677]: audit 2023-10-09T09:10:34.007578+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:10:34.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:34.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:34.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:34.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:34.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:34.782 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:34.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:34.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:34.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:34.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:34.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:34.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:35.060 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:35.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:35.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:35.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:35.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:35.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:35.318 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:35.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.006298+0000 mgr.y (mgr.25421) 997 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 3287/79572 objects degraded (4.131%); 28859/79572 objects misplaced (36.268%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:10:35.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.097169+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T09:10:35.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: audit 2023-10-09T09:10:34.100787+0000 mon.a (mon.0) 1336 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:10:35.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.100880+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T09:10:35.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: audit 2023-10-09T09:10:34.242047+0000 mon.a (mon.0) 1338 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:35.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.655961+0000 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 3287/79572 objects degraded (4.131%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:35.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.658366+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.660926+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.661951+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.662493+0000 osd.2 (osd.2) 10 : cluster 1 9.2 continuing backfill to osd.7 from (218'1900,237'10366] MIN to 237'10366 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.663283+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.663906+0000 mon.a (mon.0) 1340 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.664083+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.664146+0000 osd.0 (osd.0) 17 : cluster 1 9.8 continuing backfill to osd.5 from (219'4100,237'10656] MIN to 237'10656 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.664650+0000 osd.2 (osd.2) 11 : cluster 1 9.7 continuing backfill to osd.5 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.664713+0000 osd.2 (osd.2) 12 : cluster 1 9.7 continuing backfill to osd.7 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.664863+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.665593+0000 osd.0 (osd.0) 18 : cluster 1 9.9 continuing backfill to osd.5 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:35.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.665660+0000 osd.0 (osd.0) 19 : cluster 1 9.9 continuing backfill to osd.7 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:35.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.666227+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.666670+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.667475+0000 osd.0 (osd.0) 20 : cluster 1 9.5 continuing backfill to osd.5 from (219'3700,237'10318] MIN to 237'10318 2023-10-09T09:10:35.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.669531+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:35 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.669948+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:35.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:35.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:35.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:35.463 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.006298+0000 mgr.y (mgr.25421) 997 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 3287/79572 objects degraded (4.131%); 28859/79572 objects misplaced (36.268%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.097169+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: audit 2023-10-09T09:10:34.100787+0000 mon.a (mon.0) 1336 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.100880+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: audit 2023-10-09T09:10:34.242047+0000 mon.a (mon.0) 1338 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.655961+0000 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 3287/79572 objects degraded (4.131%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.658366+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.660926+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.661951+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.662493+0000 osd.2 (osd.2) 10 : cluster 1 9.2 continuing backfill to osd.7 from (218'1900,237'10366] MIN to 237'10366 2023-10-09T09:10:35.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.663283+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.663906+0000 mon.a (mon.0) 1340 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.664083+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.664146+0000 osd.0 (osd.0) 17 : cluster 1 9.8 continuing backfill to osd.5 from (219'4100,237'10656] MIN to 237'10656 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.664650+0000 osd.2 (osd.2) 11 : cluster 1 9.7 continuing backfill to osd.5 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.664713+0000 osd.2 (osd.2) 12 : cluster 1 9.7 continuing backfill to osd.7 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.664863+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.665593+0000 osd.0 (osd.0) 18 : cluster 1 9.9 continuing backfill to osd.5 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.665660+0000 osd.0 (osd.0) 19 : cluster 1 9.9 continuing backfill to osd.7 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.666227+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.666670+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.667475+0000 osd.0 (osd.0) 20 : cluster 1 9.5 continuing backfill to osd.5 from (219'3700,237'10318] MIN to 237'10318 2023-10-09T09:10:35.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.669531+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.669948+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.006298+0000 mgr.y (mgr.25421) 997 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 3287/79572 objects degraded (4.131%); 28859/79572 objects misplaced (36.268%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.097169+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: audit 2023-10-09T09:10:34.100787+0000 mon.a (mon.0) 1336 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.100880+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: audit 2023-10-09T09:10:34.242047+0000 mon.a (mon.0) 1338 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.655961+0000 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 3287/79572 objects degraded (4.131%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.658366+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.660926+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.661951+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.662493+0000 osd.2 (osd.2) 10 : cluster 1 9.2 continuing backfill to osd.7 from (218'1900,237'10366] MIN to 237'10366 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.663283+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.663906+0000 mon.a (mon.0) 1340 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.664083+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.664146+0000 osd.0 (osd.0) 17 : cluster 1 9.8 continuing backfill to osd.5 from (219'4100,237'10656] MIN to 237'10656 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.664650+0000 osd.2 (osd.2) 11 : cluster 1 9.7 continuing backfill to osd.5 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.664713+0000 osd.2 (osd.2) 12 : cluster 1 9.7 continuing backfill to osd.7 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.664863+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.665593+0000 osd.0 (osd.0) 18 : cluster 1 9.9 continuing backfill to osd.5 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:35.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.665660+0000 osd.0 (osd.0) 19 : cluster 1 9.9 continuing backfill to osd.7 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:35.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.666227+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.666670+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.667475+0000 osd.0 (osd.0) 20 : cluster 1 9.5 continuing backfill to osd.5 from (219'3700,237'10318] MIN to 237'10318 2023-10-09T09:10:35.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.669531+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.669948+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T09:10:35.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:35.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:35.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:35.613 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:35.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:35.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:35.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:35.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:35.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:35.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:35.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:35.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:35.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:35.983 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:10:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:10:35] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.33.4" 2023-10-09T09:10:36.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:36.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:36.107 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:36.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:36.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:36.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:36.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.668518+0000 osd.3 (osd.3) 9 : cluster 1 9.3 continuing backfill to osd.5 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:36.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:34.668568+0000 osd.3 (osd.3) 10 : cluster 1 9.3 continuing backfill to osd.7 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:36.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.671140+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.674125+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.674584+0000 mon.a (mon.0) 1341 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T09:10:36.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.678202+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.680315+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.680381+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.681664+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[77892]: cluster 2023-10-09T09:10:35.682672+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.668518+0000 osd.3 (osd.3) 9 : cluster 1 9.3 continuing backfill to osd.5 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:36.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:34.668568+0000 osd.3 (osd.3) 10 : cluster 1 9.3 continuing backfill to osd.7 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:36.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.671140+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.674125+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.674584+0000 mon.a (mon.0) 1341 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T09:10:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.678202+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.680315+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.680381+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.681664+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:36 smithi016 bash[75179]: cluster 2023-10-09T09:10:35.682672+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:36.373 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:36.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.668518+0000 osd.3 (osd.3) 9 : cluster 1 9.3 continuing backfill to osd.5 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:36.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:34.668568+0000 osd.3 (osd.3) 10 : cluster 1 9.3 continuing backfill to osd.7 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:36.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.671140+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.674125+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.674584+0000 mon.a (mon.0) 1341 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T09:10:36.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.678202+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.680315+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.388 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.680381+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.388 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.681664+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.388 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:36 smithi094 bash[59677]: cluster 2023-10-09T09:10:35.682672+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T09:10:36.401 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:36.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:36.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:36.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:36.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:36.665 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:36.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:36.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:36.791 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:36.898 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:36.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:36.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:37.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:37.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:37.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:37.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:37.156 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:37.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.007369+0000 mgr.y (mgr.25421) 998 : cluster 0 pgmap v1062: 99 pgs: 1 active+recovering+remapped, 3 peering, 1 creating+peering, 6 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 1563/78417 objects degraded (1.993%); 25430/78417 objects misplaced (32.429%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T09:10:37.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.671023+0000 mon.a (mon.0) 1342 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:10:37.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.682581+0000 osd.2 (osd.2) 13 : cluster 1 9.17 continuing backfill to osd.5 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:37.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.682650+0000 osd.2 (osd.2) 14 : cluster 1 9.17 continuing backfill to osd.7 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:37.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.682769+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.683142+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.683505+0000 osd.2 (osd.2) 15 : cluster 1 9.12 continuing backfill to osd.7 from (218'1900,237'10366] MIN to 237'10366 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.683616+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.684915+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.685239+0000 osd.0 (osd.0) 21 : cluster 1 9.19 continuing backfill to osd.5 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.685304+0000 osd.0 (osd.0) 22 : cluster 1 9.19 continuing backfill to osd.7 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.685881+0000 osd.0 (osd.0) 23 : cluster 1 9.15 continuing backfill to osd.5 from (219'3700,237'10318] MIN to 237'10318 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.687475+0000 osd.0 (osd.0) 24 : cluster 1 9.18 continuing backfill to osd.5 from (219'4100,237'10656] MIN to 237'10656 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.689719+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.690122+0000 osd.0 (osd.0) 25 : cluster 1 9.13 continuing backfill to osd.5 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.690169+0000 osd.0 (osd.0) 26 : cluster 1 9.13 continuing backfill to osd.7 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:37.166 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:37 smithi094 bash[59677]: cluster 2023-10-09T09:10:36.733633+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.175 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:37.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:37.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:37.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:37.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:37.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:37.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:37.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:37.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.007369+0000 mgr.y (mgr.25421) 998 : cluster 0 pgmap v1062: 99 pgs: 1 active+recovering+remapped, 3 peering, 1 creating+peering, 6 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 1563/78417 objects degraded (1.993%); 25430/78417 objects misplaced (32.429%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T09:10:37.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.671023+0000 mon.a (mon.0) 1342 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.682581+0000 osd.2 (osd.2) 13 : cluster 1 9.17 continuing backfill to osd.5 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.682650+0000 osd.2 (osd.2) 14 : cluster 1 9.17 continuing backfill to osd.7 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.682769+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.683142+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.683505+0000 osd.2 (osd.2) 15 : cluster 1 9.12 continuing backfill to osd.7 from (218'1900,237'10366] MIN to 237'10366 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.683616+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.684915+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.685239+0000 osd.0 (osd.0) 21 : cluster 1 9.19 continuing backfill to osd.5 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.685304+0000 osd.0 (osd.0) 22 : cluster 1 9.19 continuing backfill to osd.7 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.685881+0000 osd.0 (osd.0) 23 : cluster 1 9.15 continuing backfill to osd.5 from (219'3700,237'10318] MIN to 237'10318 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.687475+0000 osd.0 (osd.0) 24 : cluster 1 9.18 continuing backfill to osd.5 from (219'4100,237'10656] MIN to 237'10656 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.689719+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.690122+0000 osd.0 (osd.0) 25 : cluster 1 9.13 continuing backfill to osd.5 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.690169+0000 osd.0 (osd.0) 26 : cluster 1 9.13 continuing backfill to osd.7 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[75179]: cluster 2023-10-09T09:10:36.733633+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.007369+0000 mgr.y (mgr.25421) 998 : cluster 0 pgmap v1062: 99 pgs: 1 active+recovering+remapped, 3 peering, 1 creating+peering, 6 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 1563/78417 objects degraded (1.993%); 25430/78417 objects misplaced (32.429%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.671023+0000 mon.a (mon.0) 1342 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:10:37.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.682581+0000 osd.2 (osd.2) 13 : cluster 1 9.17 continuing backfill to osd.5 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.682650+0000 osd.2 (osd.2) 14 : cluster 1 9.17 continuing backfill to osd.7 from (218'1300,237'10218] MIN to 237'10218 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.682769+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.683142+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.683505+0000 osd.2 (osd.2) 15 : cluster 1 9.12 continuing backfill to osd.7 from (218'1900,237'10366] MIN to 237'10366 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.683616+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.684915+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.685239+0000 osd.0 (osd.0) 21 : cluster 1 9.19 continuing backfill to osd.5 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.685304+0000 osd.0 (osd.0) 22 : cluster 1 9.19 continuing backfill to osd.7 from (219'3800,237'10413] MIN to 237'10413 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.685881+0000 osd.0 (osd.0) 23 : cluster 1 9.15 continuing backfill to osd.5 from (219'3700,237'10318] MIN to 237'10318 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.687475+0000 osd.0 (osd.0) 24 : cluster 1 9.18 continuing backfill to osd.5 from (219'4100,237'10656] MIN to 237'10656 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.689719+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.690122+0000 osd.0 (osd.0) 25 : cluster 1 9.13 continuing backfill to osd.5 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:37.418 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.690169+0000 osd.0 (osd.0) 26 : cluster 1 9.13 continuing backfill to osd.7 from (219'4500,237'10628] MIN to 237'10628 2023-10-09T09:10:37.418 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:37 smithi016 bash[77892]: cluster 2023-10-09T09:10:36.733633+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T09:10:37.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:37.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:37.537 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:37.559 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:37.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:37.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:37.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:37.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:37.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:37.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:37.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:37.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:37.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:37.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:10:37.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:10:37.934 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:37.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:38.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:38.076 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:10:38.076 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:10:38.076 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:10:38.076 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:10:38.076 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:10:38.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:38.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:38.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:38.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:38.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:38.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:38.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:38.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:38.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:38.663 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:38.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:38.794 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:38.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:38.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:38.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:38.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:38.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:39.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:39.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:39.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:39.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:39.187 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:39 smithi094 bash[59677]: cluster 2023-10-09T09:10:38.008009+0000 mgr.y (mgr.25421) 999 : cluster 0 pgmap v1064: 99 pgs: 1 active+recovering+remapped, 3 peering, 1 creating+peering, 4 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 1563/75210 objects degraded (2.078%); 23642/75210 objects misplaced (31.435%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:10:39.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:10:39.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:39.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:39.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:39.385 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:39 smithi016 bash[75179]: cluster 2023-10-09T09:10:38.008009+0000 mgr.y (mgr.25421) 999 : cluster 0 pgmap v1064: 99 pgs: 1 active+recovering+remapped, 3 peering, 1 creating+peering, 4 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 1563/75210 objects degraded (2.078%); 23642/75210 objects misplaced (31.435%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:10:39.385 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:39 smithi016 bash[77892]: cluster 2023-10-09T09:10:38.008009+0000 mgr.y (mgr.25421) 999 : cluster 0 pgmap v1064: 99 pgs: 1 active+recovering+remapped, 3 peering, 1 creating+peering, 4 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 1563/75210 objects degraded (2.078%); 23642/75210 objects misplaced (31.435%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:10:39.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:39.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:39.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:39.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:39.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:39.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:39.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:39.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:39.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:39.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:39.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:39.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:39.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:39.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:39.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:39.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:39.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:40.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:40.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:40.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:40.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:40.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:40.183 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:40 smithi016 bash[75179]: cluster 2023-10-09T09:10:39.657003+0000 mon.a (mon.0) 1344 : cluster 3 Health check update: Degraded data redundancy: 1563/75210 objects degraded (2.078%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:40.183 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:40 smithi016 bash[77892]: cluster 2023-10-09T09:10:39.657003+0000 mon.a (mon.0) 1344 : cluster 3 Health check update: Degraded data redundancy: 1563/75210 objects degraded (2.078%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:40.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:40.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:40.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:40.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:40.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:40.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:40.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:40.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:40.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:40.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:40 smithi094 bash[59677]: cluster 2023-10-09T09:10:39.657003+0000 mon.a (mon.0) 1344 : cluster 3 Health check update: Degraded data redundancy: 1563/75210 objects degraded (2.078%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:40.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:40.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:40.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:40.709 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:40.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:40.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:40.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:40.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:40.867 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:40.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:40.904 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:10:40.904 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:10:40.905 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T09:10:40.905 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:10:40.905 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:10:40.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T09:10:40.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:41.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:41.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:41.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:41.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:41.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:41.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:41 smithi016 bash[75179]: cluster 2023-10-09T09:10:40.008729+0000 mgr.y (mgr.25421) 1000 : cluster 0 pgmap v1065: 99 pgs: 1 active+recovering+remapped, 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 1509/75939 objects degraded (1.987%); 24486/75939 objects misplaced (32.244%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T09:10:41.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:41 smithi016 bash[75179]: cluster 2023-10-09T09:10:40.659079+0000 mon.a (mon.0) 1345 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:10:41.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:41 smithi016 bash[77892]: cluster 2023-10-09T09:10:40.008729+0000 mgr.y (mgr.25421) 1000 : cluster 0 pgmap v1065: 99 pgs: 1 active+recovering+remapped, 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 1509/75939 objects degraded (1.987%); 24486/75939 objects misplaced (32.244%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T09:10:41.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:41 smithi016 bash[77892]: cluster 2023-10-09T09:10:40.659079+0000 mon.a (mon.0) 1345 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:10:41.274 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:41.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:41.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:41.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:41.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:41 smithi094 bash[59677]: cluster 2023-10-09T09:10:40.008729+0000 mgr.y (mgr.25421) 1000 : cluster 0 pgmap v1065: 99 pgs: 1 active+recovering+remapped, 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 1509/75939 objects degraded (1.987%); 24486/75939 objects misplaced (32.244%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T09:10:41.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:41 smithi094 bash[59677]: cluster 2023-10-09T09:10:40.659079+0000 mon.a (mon.0) 1345 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:10:41.506 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:41.532 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:41.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:41.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:41.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:41.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:41.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:41.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:41.904 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:41.930 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:41.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:42.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:42.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:42.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:42.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:42.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:42.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:42.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:42.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:42.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:42.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:42.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:42.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:42.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:42.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:42.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:42.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:42.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:42.752 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:42.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:42.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:42.773 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:10:42.774 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:10:42.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:42.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:42.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:42.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:43.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:43.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:43.114 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:43 smithi016 bash[77892]: cluster 2023-10-09T09:10:42.009512+0000 mgr.y (mgr.25421) 1001 : cluster 0 pgmap v1066: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 6285/73812 objects degraded (8.515%); 24311/73812 objects misplaced (32.936%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:10:43.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:43 smithi016 bash[75179]: cluster 2023-10-09T09:10:42.009512+0000 mgr.y (mgr.25421) 1001 : cluster 0 pgmap v1066: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 6285/73812 objects degraded (8.515%); 24311/73812 objects misplaced (32.936%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:10:43.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:43.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:43.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:43.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:43 smithi094 bash[59677]: cluster 2023-10-09T09:10:42.009512+0000 mgr.y (mgr.25421) 1001 : cluster 0 pgmap v1066: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 6285/73812 objects degraded (8.515%); 24311/73812 objects misplaced (32.936%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:10:43.292 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:43.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:43.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:43.526 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:43.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:43.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:43.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:43.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:43.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:43.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:43.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:43.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:43.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:44.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:44.074 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:10:44.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:44.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:44.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:44.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:44.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:44.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:44.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:44 smithi016 bash[75179]: audit 2023-10-09T09:10:43.762859+0000 mon.a (mon.0) 1346 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:10:44.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:44 smithi016 bash[77892]: audit 2023-10-09T09:10:43.762859+0000 mon.a (mon.0) 1346 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:10:44.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:44.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:44.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:44.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:44.449 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:44 smithi094 bash[59677]: audit 2023-10-09T09:10:43.762859+0000 mon.a (mon.0) 1346 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:10:44.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:44.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:44.583 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:44.616 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:44.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:44.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:44.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:44.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:44.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:44.804 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:10:44.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:44.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:44.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:44.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:45.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:45.118 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:45.148 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:45.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:45 smithi016 bash[75179]: cluster 2023-10-09T09:10:44.010252+0000 mgr.y (mgr.25421) 1002 : cluster 0 pgmap v1067: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 6134/69399 objects degraded (8.839%); 22276/69399 objects misplaced (32.098%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T09:10:45.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:45 smithi016 bash[75179]: cluster 2023-10-09T09:10:44.658235+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 6134/69399 objects degraded (8.839%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:45.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:45 smithi016 bash[77892]: cluster 2023-10-09T09:10:44.010252+0000 mgr.y (mgr.25421) 1002 : cluster 0 pgmap v1067: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 6134/69399 objects degraded (8.839%); 22276/69399 objects misplaced (32.098%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T09:10:45.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:45 smithi016 bash[77892]: cluster 2023-10-09T09:10:44.658235+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 6134/69399 objects degraded (8.839%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:45.252 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:45.282 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:45.387 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:45.427 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:45.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:45.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:45 smithi094 bash[59677]: cluster 2023-10-09T09:10:44.010252+0000 mgr.y (mgr.25421) 1002 : cluster 0 pgmap v1067: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 6134/69399 objects degraded (8.839%); 22276/69399 objects misplaced (32.098%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T09:10:45.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:45 smithi094 bash[59677]: cluster 2023-10-09T09:10:44.658235+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 6134/69399 objects degraded (8.839%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:10:45.533 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:45.559 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:45.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:45.698 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:45.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:45.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:45.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:45.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:45.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:45.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:10:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:10:45] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.33.4" 2023-10-09T09:10:46.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:46.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:46.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:46.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:46.218 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:46.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:46.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:46.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:46.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:46.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:46.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:46.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:46.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:46.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:46.601 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:46.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:46.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:46.731 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:46.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:46.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:46.838 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:46.876 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:46.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:46.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:46.980 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:47.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:47.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:47.136 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:47 smithi094 bash[59677]: cluster 2023-10-09T09:10:46.010975+0000 mgr.y (mgr.25421) 1003 : cluster 0 pgmap v1068: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 5980/67491 objects degraded (8.860%); 22169/67491 objects misplaced (32.847%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T09:10:47.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:47.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:47.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:47.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:47.383 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:47 smithi016 bash[75179]: cluster 2023-10-09T09:10:46.010975+0000 mgr.y (mgr.25421) 1003 : cluster 0 pgmap v1068: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 5980/67491 objects degraded (8.860%); 22169/67491 objects misplaced (32.847%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T09:10:47.383 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:47 smithi016 bash[77892]: cluster 2023-10-09T09:10:46.010975+0000 mgr.y (mgr.25421) 1003 : cluster 0 pgmap v1068: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 5980/67491 objects degraded (8.860%); 22169/67491 objects misplaced (32.847%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T09:10:47.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:47.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:47.504 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:47.529 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:47.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:47.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:47.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:47.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:47.769 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:47.798 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:47.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:47.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:47.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:47.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:47.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:48.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:48.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:48.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:48.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:48.155 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:48.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:48.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:48.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:48.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:48.323 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:48.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:48.430 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:48.468 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:48.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:48.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:48.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:48.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:48.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:48.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:48.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:48.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:48.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:48.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:48.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:48.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:48.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:48.971 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:49.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:49.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:49.140 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:49 smithi016 bash[77892]: cluster 2023-10-09T09:10:48.011502+0000 mgr.y (mgr.25421) 1004 : cluster 0 pgmap v1069: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 17 MiB/s wr, 802 op/s; 5980/66315 objects degraded (9.018%); 22007/66315 objects misplaced (33.186%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:10:49.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:49.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:49.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:49 smithi094 bash[59677]: cluster 2023-10-09T09:10:48.011502+0000 mgr.y (mgr.25421) 1004 : cluster 0 pgmap v1069: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 17 MiB/s wr, 802 op/s; 5980/66315 objects degraded (9.018%); 22007/66315 objects misplaced (33.186%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:10:49.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:49.293 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:49.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:49.410 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:49 smithi016 bash[75179]: cluster 2023-10-09T09:10:48.011502+0000 mgr.y (mgr.25421) 1004 : cluster 0 pgmap v1069: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 17 MiB/s wr, 802 op/s; 5980/66315 objects degraded (9.018%); 22007/66315 objects misplaced (33.186%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:10:49.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:49.537 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:49.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:49.679 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:49.719 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:49.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:49.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:49.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:49.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:49.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:49.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:50.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:50.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:50.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:50.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:50.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:50.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:50.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:50.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:50.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:50.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:50.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:50 smithi094 bash[59677]: audit 2023-10-09T09:10:49.208107+0000 mon.a (mon.0) 1348 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:50.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:50 smithi094 bash[59677]: audit 2023-10-09T09:10:49.214867+0000 mon.a (mon.0) 1349 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:50.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:50 smithi094 bash[59677]: audit 2023-10-09T09:10:49.242872+0000 mon.a (mon.0) 1350 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:50.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:50 smithi094 bash[59677]: cluster 2023-10-09T09:10:49.659476+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 5980/66315 objects degraded (9.018%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:10:50.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[75179]: audit 2023-10-09T09:10:49.208107+0000 mon.a (mon.0) 1348 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:50.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[75179]: audit 2023-10-09T09:10:49.214867+0000 mon.a (mon.0) 1349 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:50.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[75179]: audit 2023-10-09T09:10:49.242872+0000 mon.a (mon.0) 1350 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:50.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[75179]: cluster 2023-10-09T09:10:49.659476+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 5980/66315 objects degraded (9.018%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:10:50.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[77892]: audit 2023-10-09T09:10:49.208107+0000 mon.a (mon.0) 1348 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:50.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[77892]: audit 2023-10-09T09:10:49.214867+0000 mon.a (mon.0) 1349 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:50.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[77892]: audit 2023-10-09T09:10:49.242872+0000 mon.a (mon.0) 1350 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:50.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:50 smithi016 bash[77892]: cluster 2023-10-09T09:10:49.659476+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 5980/66315 objects degraded (9.018%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:10:50.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:50.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167500 9.6 9:608998e5:::benchmark_data_smithi016_172771_object70914:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.670445+0000", 2023-10-09T09:10:50.606 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 15.925073077, 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7207328950000003, 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167500 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.607 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670445+0000", 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670445+0000", 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.608 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670443+0000", 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4409999999999993e-06 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670451+0000", 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4917e-05 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670466+0000", 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0020000000000001e-05 2023-10-09T09:10:50.609 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670476+0000", 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9809000000000003e-05 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670536+0000", 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0988e-05 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.610 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op must wait for map", 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670557+0000", 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.010199121 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.680756+0000", 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0258000000000001e-05 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:10:50.611 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.680767+0000", 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6815948920000001 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.362361+0000", 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0017832639999999999 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.364145+0000", 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.025215772000000001 2023-10-09T09:10:50.612 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.389361+0000", 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0018175439999999999 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.391178+0000", 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7207328950000003 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.613 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167525 9.6 9:6180dd57:::benchmark_data_smithi016_172771_object70939:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.762870+0000", 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 15.832648404, 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6350496899999998, 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.614 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167525 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762870+0000", 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762870+0000", 2023-10-09T09:10:50.615 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762868+0000", 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0620000000000003e-06 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762875+0000", 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9110000000000002e-05 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.616 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762895+0000", 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0680000000000002e-06 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762901+0000", 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2203999999999997e-05 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.617 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762953+0000", 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2585000000000001e-05 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762965+0000", 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6013517049999999 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.364317+0000", 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019394639999999999 2023-10-09T09:10:50.618 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.366257+0000", 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0031352530000000002 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.369392+0000", 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.019547374999999999 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.619 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.388939+0000", 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028240269999999998 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.391763+0000", 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00052667899999999995 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.620 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.392290+0000", 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000939492 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.393229+0000", 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0046900860000000004 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.397920+0000", 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6350496899999998 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.621 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167530 9.6 9:667f2bd8:::benchmark_data_smithi016_172771_object70944:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.796984+0000", 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 15.798533769000001, 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6069060149999999, 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.622 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167530 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796984+0000", 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.623 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796984+0000", 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796983+0000", 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.1619999999999997e-06 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796992+0000", 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7893999999999998e-05 2023-10-09T09:10:50.624 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797010+0000", 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8969999999999997e-06 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797019+0000", 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8380999999999995e-05 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.625 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797087+0000", 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3692e-05 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797101+0000", 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5762976860000002 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.626 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.373398+0000", 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0023449650000000001 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.375743+0000", 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029730749999999999 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.378716+0000", 2023-10-09T09:10:50.627 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.010364816000000001 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.389081+0000", 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0094196139999999998 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.398501+0000", 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00068523300000000004 2023-10-09T09:10:50.628 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.399186+0000", 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0011877840000000001 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.400374+0000", 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0035166569999999999 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.629 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.403890+0000", 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6069060149999999 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167537 9.6 9:62deb5e7:::benchmark_data_smithi016_172771_object70951:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.810885+0000", 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 15.784632989, 2023-10-09T09:10:50.630 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7884914350000001, 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167537 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810885+0000", 2023-10-09T09:10:50.631 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810885+0000", 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810884+0000", 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4e-06 2023-10-09T09:10:50.632 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810889+0000", 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7440000000000002e-06 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810899+0000", 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.369e-06 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.633 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810903+0000", 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.2126999999999997e-05 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810966+0000", 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0293e-05 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:10:50.634 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810986+0000", 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6004881640000002 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.411474+0000", 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0018830400000000001 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.413357+0000", 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.012155733 2023-10-09T09:10:50.635 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.425513+0000", 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019261440000000001 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.427439+0000", 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.156210243 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.636 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.583649+0000", 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001402569 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.585052+0000", 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028961970000000001 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.637 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.587948+0000", 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0031777770000000001 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.591126+0000", 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0041007320000000002 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595226+0000", 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014585499999999999 2023-10-09T09:10:50.638 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595372+0000", 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00072834699999999996 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.596101+0000", 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0032759899999999999 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.639 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.599377+0000", 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7884914350000001 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167551 9.17 9.309d0bf7 (undecoded) ondisk+write+known_if_redirected e239)", 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.856594+0000", 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 15.738923957000001, 2023-10-09T09:10:50.640 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.82954309000000004, 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167551 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856594+0000", 2023-10-09T09:10:50.641 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856594+0000", 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856592+0000", 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7029999999999995e-06 2023-10-09T09:10:50.642 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856601+0000", 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2045000000000001e-05 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856613+0000", 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3899999999999998e-06 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.643 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856619+0000", 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.0243e-05 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856669+0000", 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.6370000000000005e-06 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.644 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856678+0000", 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.82943719800000004 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.686115+0000", 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.213e-05 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.686137+0000", 2023-10-09T09:10:50.645 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.82954309000000004 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "MOSDPGPush(9.e 240/230 [PushOp(9:7d10c4b0:::benchmark_data_smithi016_172771_object72506:head, version: 219'4540, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:7d10c4b0:::benchmark_data_smithi016_172771_object72506:head@219'4540, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.105023+0000", 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 14.490495230000001, 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.377055428, 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.646 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105023+0000", 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105023+0000", 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.647 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105022+0000", 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.2552000000000003e-05 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105054+0000", 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7813999999999999e-05 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.648 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105072+0000", 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0685e-05 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105083+0000", 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.373906643 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.478990+0000", 2023-10-09T09:10:50.649 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1800000000000001e-05 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.479001+0000", 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0030770110000000002 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.482078+0000", 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.377055428 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.650 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167572 9.16 9:68db5b2c:::benchmark_data_smithi016_172771_object70986:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.733117+0000", 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.862400915, 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87027597099999998, 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.651 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167572 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733117+0000", 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.652 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733117+0000", 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733116+0000", 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.0509999999999997e-06 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733124+0000", 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1333999999999999e-05 2023-10-09T09:10:50.653 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733145+0000", 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.1690000000000003e-06 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733152+0000", 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.69478544200000003 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.654 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.427938+0000", 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020526250000000002 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.429990+0000", 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.022311549 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.452302+0000", 2023-10-09T09:10:50.655 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020267430000000001 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.454329+0000", 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.133338768 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.587667+0000", 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001300692 2023-10-09T09:10:50.656 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.588968+0000", 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002805227 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.591773+0000", 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0034174190000000001 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.657 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595191+0000", 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00044434500000000002 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595635+0000", 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0039520429999999997 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.599587+0000", 2023-10-09T09:10:50.658 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00095617100000000002 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.600543+0000", 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002850109 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.603393+0000", 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87027597099999998 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.659 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167573 9.6 9:6683712f:::benchmark_data_smithi016_172771_object70987:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.861954813000001, 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87949885500000002, 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.660 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167573 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:10:50.661 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4440000000000001e-06 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733567+0000", 2023-10-09T09:10:50.662 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.079e-05 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733578+0000", 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3959999999999997e-06 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733583+0000", 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.69850582100000003 2023-10-09T09:10:50.663 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.432089+0000", 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001875011 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.433964+0000", 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.018259147999999999 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.664 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.452223+0000", 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020477120000000001 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.454271+0000", 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14543932900000001 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.599710+0000", 2023-10-09T09:10:50.665 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00133927 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.601050+0000", 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0024509449999999999 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.603500+0000", 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.005160548 2023-10-09T09:10:50.666 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.608661+0000", 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00046506899999999998 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.609126+0000", 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014417600000000001 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.667 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.609270+0000", 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00076132999999999995 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.610032+0000", 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0030306650000000001 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.668 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.613062+0000", 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87949885500000002 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:167574 9.e e241/230 9:7ebc21e6:::benchmark_data_smithi016_172771_object70988:head v 241'10492, mlcod=219'4437)", 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.750278+0000", 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.845240236, 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.617605615, 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.669 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.750278+0000", 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.750278+0000", 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:10:50.670 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.750278+0000", 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.9469999999999996e-06 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.750282+0000", 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.985e-06 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.671 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.750289+0000", 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.2189999999999998e-06 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.750293+0000", 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.585939027 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.672 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.336232+0000", 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.834e-05 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.336260+0000", 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.031570266999999999 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.673 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.367831+0000", 2023-10-09T09:10:50.674 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3106999999999998e-05 2023-10-09T09:10:50.674 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.674 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.674 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.367884+0000", 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.617605615 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:167592 9.e e241/230 9:708e2129:::benchmark_data_smithi016_172771_object71006:head v 241'10493, mlcod=219'4437)", 2023-10-09T09:10:50.675 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.769160+0000", 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.82635853, 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.598760018, 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769160+0000", 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.676 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769160+0000", 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769159+0000", 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.0739999999999993e-06 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.677 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769167+0000", 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.076e-05 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769178+0000", 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0109999999999997e-06 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769184+0000", 2023-10-09T09:10:50.678 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.57578313800000003 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.344967+0000", 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5456999999999993e-05 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.345032+0000", 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.022855782000000002 2023-10-09T09:10:50.679 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.367888+0000", 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.1742000000000002e-05 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.367920+0000", 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.598760018 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.680 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "MOSDPGPush(9.e 241/230 [PushOp(9:74521862:::benchmark_data_smithi016_172771_object72509:head, version: 219'4541, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:74521862:::benchmark_data_smithi016_172771_object72509:head@219'4541, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.769190+0000", 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.826327916, 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.71845737200000004, 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.681 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769190+0000", 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769190+0000", 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769190+0000", 2023-10-09T09:10:50.682 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.5308000000000001e-05 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769225+0000", 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5703000000000001e-05 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769251+0000", 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.9680000000000001e-06 2023-10-09T09:10:50.683 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.769261+0000", 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.71413233700000001 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.483393+0000", 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6813e-05 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.684 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.483410+0000", 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0042376719999999996 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.487648+0000", 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.71845737200000004 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.685 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167610 9.6 9:65930b06:::benchmark_data_smithi016_172771_object71024:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.786339+0000", 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.809179608000001, 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.85008152999999997, 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167610 2023-10-09T09:10:50.686 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786339+0000", 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786339+0000", 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.687 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786337+0000", 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7220000000000005e-06 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786345+0000", 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5143000000000001e-05 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.688 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786360+0000", 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8770000000000004e-06 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786367+0000", 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.66811855799999997 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.454486+0000", 2023-10-09T09:10:50.689 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019376689999999999 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.456424+0000", 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0099610730000000008 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.466385+0000", 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001688169 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.690 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.468073+0000", 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.148861836 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.616935+0000", 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001150933 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.691 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.618086+0000", 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029704890000000002 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.621056+0000", 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.010123839000000001 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.631180+0000", 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00043493299999999998 2023-10-09T09:10:50.692 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.631615+0000", 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00053206800000000002 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.632147+0000", 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00090994600000000002 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.693 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.633057+0000", 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033633209999999998 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.636420+0000", 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.85008152999999997 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.694 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167613 9.6 9:63f11c86:::benchmark_data_smithi016_172771_object71027:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.788607+0000", 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.806911252000001, 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87494654599999999, 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167613 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.695 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788607+0000", 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788607+0000", 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.696 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788605+0000", 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5630000000000003e-06 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788613+0000", 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2583e-05 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788626+0000", 2023-10-09T09:10:50.697 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4509999999999997e-06 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788632+0000", 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.66800163499999998 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.456634+0000", 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00184935 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.698 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.458483+0000", 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024125830000000001 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.482609+0000", 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0021895199999999999 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.699 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.484798+0000", 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15935247699999999 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.644151+0000", 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001645175 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.645796+0000", 2023-10-09T09:10:50.700 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033096229999999998 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.649106+0000", 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0041119559999999999 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.653218+0000", 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00056759400000000004 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.701 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.653785+0000", 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.005000666 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.658786+0000", 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000948802 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.702 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.659735+0000", 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003818905 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.663554+0000", 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87494654599999999 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.703 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167615 9.16 9:6b4dc5a4:::benchmark_data_smithi016_172771_object71029:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.792429+0000", 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.803089497, 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87891387899999995, 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167615 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.704 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792429+0000", 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792429+0000", 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792427+0000", 2023-10-09T09:10:50.705 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.8710000000000002e-06 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792433+0000", 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0434e-05 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792444+0000", 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9355e-05 2023-10-09T09:10:50.706 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792463+0000", 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.69274825900000003 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.485211+0000", 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001540334 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.707 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.486752+0000", 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00884186 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.495593+0000", 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001833157 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.497427+0000", 2023-10-09T09:10:50.708 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.16112549900000001 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.658552+0000", 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0013227250000000001 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.659875+0000", 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0032222000000000002 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.709 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.663097+0000", 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028207079999999999 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.665918+0000", 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00069836799999999999 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.710 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.666616+0000", 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00021289 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.666829+0000", 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00096081200000000002 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.711 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.667790+0000", 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0035527670000000001 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.671343+0000", 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87891387899999995 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.712 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:167625 9.e e241/230 9:7bc03783:::benchmark_data_smithi016_172771_object71039:head v 241'10494, mlcod=219'4437)", 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.805793+0000", 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.789725289, 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.56215508999999997, 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.805793+0000", 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.713 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.805793+0000", 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.805792+0000", 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.2910000000000007e-06 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.714 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.805800+0000", 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2211999999999999e-05 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.805813+0000", 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.7309999999999996e-06 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.805819+0000", 2023-10-09T09:10:50.715 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.54675526900000004 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.352575+0000", 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3173000000000002e-05 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.352608+0000", 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.015314408 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.716 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.367922+0000", 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5831000000000001e-05 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.367948+0000", 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.56215508999999997 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.717 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167626 9.16 9:68673687:::benchmark_data_smithi016_172771_object71040:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.806080+0000", 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.789438512, 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90435950899999995, 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.718 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167626 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806080+0000", 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806080+0000", 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:10:50.719 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806078+0000", 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.888e-06 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806085+0000", 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5667999999999999e-05 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.720 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806111+0000", 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4659999999999996e-06 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806117+0000", 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.680842265 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.721 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.486960+0000", 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001570407 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.488530+0000", 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.016700744 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.505231+0000", 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00155108 2023-10-09T09:10:50.722 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.506782+0000", 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.18548661699999999 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.692268+0000", 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0017039589999999999 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.723 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.693972+0000", 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003072249 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.697045+0000", 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0080081349999999996 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705053+0000", 2023-10-09T09:10:50.724 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00072625800000000003 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705779+0000", 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000194952 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705974+0000", 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00094332200000000002 2023-10-09T09:10:50.725 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.706917+0000", 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0035220020000000002 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.710439+0000", 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90435950899999995 2023-10-09T09:10:50.726 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167631 9.6 9:61adcc4d:::benchmark_data_smithi016_172771_object71045:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.812915+0000", 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.782602749, 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.89219442299999996, 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.727 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167631 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812915+0000", 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.728 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812915+0000", 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812913+0000", 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8959999999999992e-06 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.729 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812922+0000", 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9108000000000001e-05 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812941+0000", 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8959999999999998e-06 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812951+0000", 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.67575335400000003 2023-10-09T09:10:50.730 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.488704+0000", 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001590089 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.490295+0000", 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.017790391999999999 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.731 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.508085+0000", 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019484459999999999 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.510033+0000", 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.17243972599999999 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.732 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.682473+0000", 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001713743 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.684187+0000", 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033628519999999999 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.687550+0000", 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0047191000000000004 2023-10-09T09:10:50.733 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.692269+0000", 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0082264759999999999 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.700495+0000", 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019154999999999999 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.734 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.700687+0000", 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00095907800000000004 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.701646+0000", 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0034639990000000002 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.735 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705110+0000", 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.89219442299999996 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:167638 9.e e241/230 9:7eb266db:::benchmark_data_smithi016_172771_object71052:head v 241'10495, mlcod=219'4437)", 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.831271+0000", 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.764246929, 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.55032552999999995, 2023-10-09T09:10:50.736 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.831271+0000", 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.831271+0000", 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:10:50.737 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.831270+0000", 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3468e-05 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.831284+0000", 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0233000000000001e-05 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.738 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.831304+0000", 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1514e-05 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.831315+0000", 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.52619003399999997 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.739 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.357505+0000", 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5739000000000003e-05 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.357561+0000", 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024009953000000001 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.381571+0000", 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5851e-05 2023-10-09T09:10:50.740 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.381597+0000", 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.55032552999999995 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:167642 9.e e241/230 9:7764c582:::benchmark_data_smithi016_172771_object71056:head v 241'10496, mlcod=219'4437)", 2023-10-09T09:10:50.741 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.849790+0000", 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 13.745728527000001, 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.53183270800000004, 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.849790+0000", 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.742 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.849790+0000", 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999938 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.849783+0000", 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7482999999999998e-05 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.743 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.849801+0000", 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.4990000000000002e-05 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.849836+0000", 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0763e-05 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.849847+0000", 2023-10-09T09:10:50.744 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.51214349199999998 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.361990+0000", 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.7435000000000003e-05 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.362028+0000", 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.019572407999999999 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.745 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.381600+0000", 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.2415000000000001e-05 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.381622+0000", 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.53183270800000004 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.746 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:168424 9.6 9:6303e1c8:::benchmark_data_smithi016_172771_object71838:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:39.053058+0000", 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 11.542460650000001, 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.48753119900000003, 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:10:50.747 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 168424 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.053058+0000", 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.053058+0000", 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:10:50.748 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.053056+0000", 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1899999999999995e-06 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.053064+0000", 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4883e-05 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.749 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.053079+0000", 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1659999999999994e-06 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.053087+0000", 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.033694472000000003 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.750 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.086782+0000", 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001953813 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.088735+0000", 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.020112749999999999 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.108848+0000", 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0016600040000000001 2023-10-09T09:10:50.751 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.110508+0000", 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.3958507 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.506359+0000", 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001592485 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.752 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.507951+0000", 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0054478720000000003 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.513399+0000", 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0088699209999999994 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.753 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.522269+0000", 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.012652089 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.534921+0000", 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00062443999999999998 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.535546+0000", 2023-10-09T09:10:50.754 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0010875489999999999 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.536633+0000", 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0039556599999999997 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:39.540589+0000", 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.48753119900000003 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.755 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.756 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.756 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:10:50.756 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:10:50.756 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:10:50.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:50.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:50.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:50.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:50.883 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:50.913 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:50.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:51.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:51.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:51.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:51.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:51.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:51.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:51.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:51.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:51.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:51.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:51.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:51.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:51.597 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:51 smithi094 bash[59677]: cluster 2023-10-09T09:10:50.012190+0000 mgr.y (mgr.25421) 1005 : cluster 0 pgmap v1070: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 16 MiB/s wr, 885 op/s; 5835/61695 objects degraded (9.458%); 19888/61695 objects misplaced (32.236%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:10:51.597 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:51 smithi094 bash[59677]: audit 2023-10-09T09:10:50.415905+0000 mon.a (mon.0) 1352 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.597 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:51 smithi094 bash[59677]: audit 2023-10-09T09:10:50.422471+0000 mon.a (mon.0) 1353 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.597 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:51 smithi094 bash[59677]: audit 2023-10-09T09:10:50.995775+0000 mon.a (mon.0) 1354 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:10:51.597 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:51 smithi094 bash[59677]: audit 2023-10-09T09:10:50.996841+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:10:51.597 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:51 smithi094 bash[59677]: audit 2023-10-09T09:10:51.007282+0000 mon.a (mon.0) 1356 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:51.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[75179]: cluster 2023-10-09T09:10:50.012190+0000 mgr.y (mgr.25421) 1005 : cluster 0 pgmap v1070: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 16 MiB/s wr, 885 op/s; 5835/61695 objects degraded (9.458%); 19888/61695 objects misplaced (32.236%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:10:51.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[75179]: audit 2023-10-09T09:10:50.415905+0000 mon.a (mon.0) 1352 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[75179]: audit 2023-10-09T09:10:50.422471+0000 mon.a (mon.0) 1353 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[75179]: audit 2023-10-09T09:10:50.995775+0000 mon.a (mon.0) 1354 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:10:51.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[75179]: audit 2023-10-09T09:10:50.996841+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:10:51.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[75179]: audit 2023-10-09T09:10:51.007282+0000 mon.a (mon.0) 1356 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[77892]: cluster 2023-10-09T09:10:50.012190+0000 mgr.y (mgr.25421) 1005 : cluster 0 pgmap v1070: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 16 MiB/s wr, 885 op/s; 5835/61695 objects degraded (9.458%); 19888/61695 objects misplaced (32.236%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:10:51.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[77892]: audit 2023-10-09T09:10:50.415905+0000 mon.a (mon.0) 1352 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[77892]: audit 2023-10-09T09:10:50.422471+0000 mon.a (mon.0) 1353 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[77892]: audit 2023-10-09T09:10:50.995775+0000 mon.a (mon.0) 1354 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:10:51.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[77892]: audit 2023-10-09T09:10:50.996841+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:10:51.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:51 smithi016 bash[77892]: audit 2023-10-09T09:10:51.007282+0000 mon.a (mon.0) 1356 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:10:51.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:51.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:51.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:51.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:51.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:51.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:52.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:52.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:52.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:52.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:52.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:52.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:52.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:52.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:10:52.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:52.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:52.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:52.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:52.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:52.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:52.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:52.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:52.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:52.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:52.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:52.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:52.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:52.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:52.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:52.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:53.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:53.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:53.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:53.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:53.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:53.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:53.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:53.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:53.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:53.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:53.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:53.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:53.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:53.674 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:53 smithi016 bash[75179]: cluster 2023-10-09T09:10:52.013288+0000 mgr.y (mgr.25421) 1006 : cluster 0 pgmap v1071: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 5692/59112 objects degraded (9.629%); 19623/59112 objects misplaced (33.196%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:10:53.674 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:53 smithi016 bash[77892]: cluster 2023-10-09T09:10:52.013288+0000 mgr.y (mgr.25421) 1006 : cluster 0 pgmap v1071: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 5692/59112 objects degraded (9.629%); 19623/59112 objects misplaced (33.196%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:10:53.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:53.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:53 smithi094 bash[59677]: cluster 2023-10-09T09:10:52.013288+0000 mgr.y (mgr.25421) 1006 : cluster 0 pgmap v1071: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 5692/59112 objects degraded (9.629%); 19623/59112 objects misplaced (33.196%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:10:53.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:53.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:53.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:53.947 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:53.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:53.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:54.052 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:54.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:54.180 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:54.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:54.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:54.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:54.340 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:54.445 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:54.467 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:54.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:54.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:54.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:54.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:54.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:54.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:54.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:54.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:54.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:54.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:54.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:54.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:54.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:55.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:55.123 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:55.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:55.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:55.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:55.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:55.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:55.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:55.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:55.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:55.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:55.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:55 smithi094 bash[59677]: cluster 2023-10-09T09:10:54.014111+0000 mgr.y (mgr.25421) 1007 : cluster 0 pgmap v1072: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 5549/54606 objects degraded (10.162%); 17572/54606 objects misplaced (32.180%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:10:55.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:55 smithi094 bash[59677]: cluster 2023-10-09T09:10:54.660761+0000 mon.a (mon.0) 1357 : cluster 3 Health check update: Degraded data redundancy: 5549/54606 objects degraded (10.162%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:10:55.502 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:55.537 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:55.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:55.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:55.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:55.655 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:55 smithi016 bash[75179]: cluster 2023-10-09T09:10:54.014111+0000 mgr.y (mgr.25421) 1007 : cluster 0 pgmap v1072: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 5549/54606 objects degraded (10.162%); 17572/54606 objects misplaced (32.180%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:10:55.655 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:55 smithi016 bash[75179]: cluster 2023-10-09T09:10:54.660761+0000 mon.a (mon.0) 1357 : cluster 3 Health check update: Degraded data redundancy: 5549/54606 objects degraded (10.162%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:10:55.655 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:55 smithi016 bash[77892]: cluster 2023-10-09T09:10:54.014111+0000 mgr.y (mgr.25421) 1007 : cluster 0 pgmap v1072: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 5549/54606 objects degraded (10.162%); 17572/54606 objects misplaced (32.180%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:10:55.655 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:55 smithi016 bash[77892]: cluster 2023-10-09T09:10:54.660761+0000 mon.a (mon.0) 1357 : cluster 3 Health check update: Degraded data redundancy: 5549/54606 objects degraded (10.162%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:10:55.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:55.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:55.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:55.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:55.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:55.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:55.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:55.914 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:10:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:10:55] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-10-09T09:10:55.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:55.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:56.037 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:56.060 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:56.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:56.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:56.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:56.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:56.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:56.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:56.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:56.457 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:56.493 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:10:56.493 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "" 2023-10-09T09:10:56.493 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:10:56.544 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:10:56.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:56.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:56.596 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:10:56.645 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:10:56.675 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:10:56.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:56.710 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:10:56.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:56.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:56.747 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:10:56.794 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:10:56.832 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:10:56.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:56.838 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.839 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.839 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.839 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.839 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.839 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.839 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.839 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:10:56.859 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:56.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:56.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:56.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:56.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:56.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:57.069 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:10:57.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:57.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:57.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:57.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:57.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:57.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:10:57.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:57.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:10:57.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:57.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:57.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:57.426 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:57 smithi094 bash[59677]: cluster 2023-10-09T09:10:56.014814+0000 mgr.y (mgr.25421) 1008 : cluster 0 pgmap v1073: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 5390/51771 objects degraded (10.411%); 17408/51771 objects misplaced (33.625%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T09:10:57.427 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:57 smithi094 bash[59677]: audit 2023-10-09T09:10:56.490767+0000 mon.a (mon.0) 1358 : audit 0 from='client.? 172.21.15.16:0/2841920475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:57.427 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:57 smithi094 bash[59677]: audit 2023-10-09T09:10:56.920492+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.16:0/966599088' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:57.427 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:57 smithi094 bash[59677]: audit 2023-10-09T09:10:56.920930+0000 mon.a (mon.0) 1359 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:57.443 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:10:57.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:57.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[75179]: cluster 2023-10-09T09:10:56.014814+0000 mgr.y (mgr.25421) 1008 : cluster 0 pgmap v1073: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 5390/51771 objects degraded (10.411%); 17408/51771 objects misplaced (33.625%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T09:10:57.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[75179]: audit 2023-10-09T09:10:56.490767+0000 mon.a (mon.0) 1358 : audit 0 from='client.? 172.21.15.16:0/2841920475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:57.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[75179]: audit 2023-10-09T09:10:56.920492+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.16:0/966599088' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:57.490 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[75179]: audit 2023-10-09T09:10:56.920930+0000 mon.a (mon.0) 1359 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:57.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[77892]: cluster 2023-10-09T09:10:56.014814+0000 mgr.y (mgr.25421) 1008 : cluster 0 pgmap v1073: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 5390/51771 objects degraded (10.411%); 17408/51771 objects misplaced (33.625%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T09:10:57.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[77892]: audit 2023-10-09T09:10:56.490767+0000 mon.a (mon.0) 1358 : audit 0 from='client.? 172.21.15.16:0/2841920475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:10:57.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[77892]: audit 2023-10-09T09:10:56.920492+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.16:0/966599088' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:57.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:57 smithi016 bash[77892]: audit 2023-10-09T09:10:56.920930+0000 mon.a (mon.0) 1359 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:10:57.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:57.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:57.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:57.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:57.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:57.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:57.735 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:57.764 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:57.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:57.867 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:57.892 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:57.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:57.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:58.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:58.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:58.162 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:58.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:58.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:58.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:58.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:58.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:58.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:58.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:58.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:58.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:10:58.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:10:58.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:58 smithi094 bash[59677]: cluster 2023-10-09T09:10:57.423410+0000 mon.a (mon.0) 1360 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:10:58.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:58 smithi094 bash[59677]: cluster 2023-10-09T09:10:57.430916+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T09:10:58.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:58 smithi094 bash[59677]: cluster 2023-10-09T09:10:57.432680+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T09:10:58.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:58 smithi094 bash[59677]: audit 2023-10-09T09:10:57.434059+0000 mon.a (mon.0) 1361 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:10:58.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:58 smithi094 bash[59677]: cluster 2023-10-09T09:10:57.434151+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T09:10:58.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:58.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:58.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[75179]: cluster 2023-10-09T09:10:57.423410+0000 mon.a (mon.0) 1360 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:10:58.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[75179]: cluster 2023-10-09T09:10:57.430916+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T09:10:58.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[75179]: cluster 2023-10-09T09:10:57.432680+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T09:10:58.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[75179]: audit 2023-10-09T09:10:57.434059+0000 mon.a (mon.0) 1361 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:10:58.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[75179]: cluster 2023-10-09T09:10:57.434151+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T09:10:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[77892]: cluster 2023-10-09T09:10:57.423410+0000 mon.a (mon.0) 1360 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:10:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[77892]: cluster 2023-10-09T09:10:57.430916+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T09:10:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[77892]: cluster 2023-10-09T09:10:57.432680+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T09:10:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[77892]: audit 2023-10-09T09:10:57.434059+0000 mon.a (mon.0) 1361 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:10:58.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:58 smithi016 bash[77892]: cluster 2023-10-09T09:10:57.434151+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T09:10:58.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:58.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:58.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:58.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:58.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:59.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:59.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:59.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:59.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:59.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:10:59.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:59.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:10:59.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:59.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:10:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:59.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:10:59.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:59.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:10:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:59.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:10:59.619 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:59.643 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:10:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:59.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:10:59.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:59.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:10:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:59.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:10:59.840 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:10:59 smithi094 bash[59677]: cluster 2023-10-09T09:10:58.015599+0000 mgr.y (mgr.25421) 1009 : cluster 0 pgmap v1075: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 5390/51525 objects degraded (10.461%); 17244/51525 objects misplaced (33.467%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T09:10:59.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:59.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:10:59.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:10:59.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:10:59.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:10:59 smithi016 bash[75179]: cluster 2023-10-09T09:10:58.015599+0000 mgr.y (mgr.25421) 1009 : cluster 0 pgmap v1075: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 5390/51525 objects degraded (10.461%); 17244/51525 objects misplaced (33.467%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T09:10:59.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:10:59 smithi016 bash[77892]: cluster 2023-10-09T09:10:58.015599+0000 mgr.y (mgr.25421) 1009 : cluster 0 pgmap v1075: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 5390/51525 objects degraded (10.461%); 17244/51525 objects misplaced (33.467%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T09:11:00.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:00.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:00.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:00.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:00.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:00.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:00.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:00.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:00.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:00.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:00.437 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:00 smithi016 bash[77892]: cluster 2023-10-09T09:10:59.661956+0000 mon.a (mon.0) 1363 : cluster 3 Health check update: Degraded data redundancy: 5390/51525 objects degraded (10.461%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:00.443 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:00.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:00.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:00.676 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:00 smithi094 bash[59677]: cluster 2023-10-09T09:10:59.661956+0000 mon.a (mon.0) 1363 : cluster 3 Health check update: Degraded data redundancy: 5390/51525 objects degraded (10.461%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:00.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:00.690 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:00 smithi016 bash[75179]: cluster 2023-10-09T09:10:59.661956+0000 mon.a (mon.0) 1363 : cluster 3 Health check update: Degraded data redundancy: 5390/51525 objects degraded (10.461%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:00.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:00.814 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:00.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:00.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:00.980 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:00.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:01.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:01.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:01.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:01.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:01.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:01.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:01.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:01.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:01.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:01.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:01.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:01.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:01.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:01.616 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:01.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:01.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:01.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:01.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:01 smithi094 bash[59677]: cluster 2023-10-09T09:11:00.016382+0000 mgr.y (mgr.25421) 1010 : cluster 0 pgmap v1076: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 979 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 5221/47034 objects degraded (11.100%); 15187/47034 objects misplaced (32.289%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:11:01.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:01 smithi016 bash[75179]: cluster 2023-10-09T09:11:00.016382+0000 mgr.y (mgr.25421) 1010 : cluster 0 pgmap v1076: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 979 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 5221/47034 objects degraded (11.100%); 15187/47034 objects misplaced (32.289%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:11:01.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:01 smithi016 bash[77892]: cluster 2023-10-09T09:11:00.016382+0000 mgr.y (mgr.25421) 1010 : cluster 0 pgmap v1076: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 979 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 5221/47034 objects degraded (11.100%); 15187/47034 objects misplaced (32.289%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:11:01.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:01.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:01.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:01.880 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:01.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:01.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:01.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:02.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:02.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:02.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:02.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:02.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:02.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:02.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:02.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:02.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:02.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:02.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:02.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:02.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:02.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:11:02.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:02.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:11:02.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:02.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:02.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:02.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:02.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:02.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:02.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:02.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:02.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:03.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:03.041 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:03.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:03.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:03.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:03.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:03.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:03.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:03.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:03.443 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:03 smithi094 bash[59677]: cluster 2023-10-09T09:11:02.017350+0000 mgr.y (mgr.25421) 1011 : cluster 0 pgmap v1077: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 909 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 5065/43656 objects degraded (11.602%); 14813/43656 objects misplaced (33.931%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T09:11:03.538 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:03.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:03.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:03.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:03 smithi016 bash[75179]: cluster 2023-10-09T09:11:02.017350+0000 mgr.y (mgr.25421) 1011 : cluster 0 pgmap v1077: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 909 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 5065/43656 objects degraded (11.602%); 14813/43656 objects misplaced (33.931%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T09:11:03.695 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:03 smithi016 bash[77892]: cluster 2023-10-09T09:11:02.017350+0000 mgr.y (mgr.25421) 1011 : cluster 0 pgmap v1077: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 909 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 5065/43656 objects degraded (11.602%); 14813/43656 objects misplaced (33.931%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T09:11:03.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:03.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:03.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:03.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:03.890 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:11:03.890 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:11:03.890 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T09:11:03.890 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:11:03.890 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:11:03.891 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T09:11:03.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:03.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:03.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:03.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:04.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:04.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:04.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:04.210 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:11:04.210 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:11:04.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:04.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:04.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:04.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:04.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:04.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:04.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:04.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:04.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:04.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:04.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:04.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:04.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:04.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:04.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:04.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:04.780 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:04.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:04.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:04.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:04.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:04.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:05.018 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:05.042 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:05.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:05.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:05.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:05.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:05.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:05.277 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:05.321 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:05.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:05.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:05.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:05.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[77892]: cluster 2023-10-09T09:11:04.018225+0000 mgr.y (mgr.25421) 1012 : cluster 0 pgmap v1078: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 802 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 4901/38547 objects degraded (12.714%); 12489/38547 objects misplaced (32.399%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:11:05.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[77892]: audit 2023-10-09T09:11:04.250710+0000 mon.a (mon.0) 1364 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:05.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[77892]: audit 2023-10-09T09:11:04.252042+0000 mon.a (mon.0) 1365 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:05.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[77892]: cluster 2023-10-09T09:11:04.663234+0000 mon.a (mon.0) 1366 : cluster 3 Health check update: Degraded data redundancy: 4901/38547 objects degraded (12.714%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:05.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[75179]: cluster 2023-10-09T09:11:04.018225+0000 mgr.y (mgr.25421) 1012 : cluster 0 pgmap v1078: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 802 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 4901/38547 objects degraded (12.714%); 12489/38547 objects misplaced (32.399%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:11:05.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[75179]: audit 2023-10-09T09:11:04.250710+0000 mon.a (mon.0) 1364 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:05.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[75179]: audit 2023-10-09T09:11:04.252042+0000 mon.a (mon.0) 1365 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:05.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[75179]: cluster 2023-10-09T09:11:04.663234+0000 mon.a (mon.0) 1366 : cluster 3 Health check update: Degraded data redundancy: 4901/38547 objects degraded (12.714%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:05.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:05.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:05.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:05.549 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:05 smithi094 bash[59677]: cluster 2023-10-09T09:11:04.018225+0000 mgr.y (mgr.25421) 1012 : cluster 0 pgmap v1078: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 802 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 4901/38547 objects degraded (12.714%); 12489/38547 objects misplaced (32.399%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:11:05.549 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:05 smithi094 bash[59677]: audit 2023-10-09T09:11:04.250710+0000 mon.a (mon.0) 1364 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:05.549 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:05 smithi094 bash[59677]: audit 2023-10-09T09:11:04.252042+0000 mon.a (mon.0) 1365 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:05.549 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:05 smithi094 bash[59677]: cluster 2023-10-09T09:11:04.663234+0000 mon.a (mon.0) 1366 : cluster 3 Health check update: Degraded data redundancy: 4901/38547 objects degraded (12.714%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:05.550 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:11:05.565 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:05.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:05.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:05.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:05.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:05.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:05.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:05.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:05.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:11:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:11:05] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-10-09T09:11:06.008 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:06.040 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:06.146 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:06.174 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:06.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:06.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:06.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:06.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:06.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:06.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:06.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:06.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:06.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:06.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:06.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:06.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:06.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:06.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:06.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:06.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:07.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:07.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:07.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:07.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:07.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:07.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:07.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:07.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:07.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:07.343 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:07 smithi016 bash[75179]: cluster 2023-10-09T09:11:06.019171+0000 mgr.y (mgr.25421) 1013 : cluster 0 pgmap v1079: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 733 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 4729/35220 objects degraded (13.427%); 12307/35220 objects misplaced (34.943%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:11:07.347 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:07 smithi016 bash[77892]: cluster 2023-10-09T09:11:06.019171+0000 mgr.y (mgr.25421) 1013 : cluster 0 pgmap v1079: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 733 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 4729/35220 objects degraded (13.427%); 12307/35220 objects misplaced (34.943%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:11:07.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:07.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:07.389 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:07.390 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:07.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:07.507 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:07.508 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:07 smithi094 bash[59677]: cluster 2023-10-09T09:11:06.019171+0000 mgr.y (mgr.25421) 1013 : cluster 0 pgmap v1079: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 733 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 4729/35220 objects degraded (13.427%); 12307/35220 objects misplaced (34.943%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:11:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:07.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:07.610 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:07.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:07.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:07.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:07.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:07.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:07.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:07.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:07.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:07.914 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:07.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:07.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:08.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:08.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:08.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:08.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:08.183 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:08.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:08.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:08.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:08.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:08.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:08.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:08.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:08.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:08.731 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:08.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:08.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:08.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:08.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:08.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:08.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:08.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:09.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:09.101 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:11:09.128 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:09.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:09.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:09.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:09.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:09.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:09.376 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:09 smithi094 bash[59677]: cluster 2023-10-09T09:11:08.019812+0000 mgr.y (mgr.25421) 1014 : cluster 0 pgmap v1080: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 727 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 4729/34944 objects degraded (13.533%); 12123/34944 objects misplaced (34.693%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T09:11:09.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:09.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:09.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:09.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:09.514 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:09 smithi016 bash[75179]: cluster 2023-10-09T09:11:08.019812+0000 mgr.y (mgr.25421) 1014 : cluster 0 pgmap v1080: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 727 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 4729/34944 objects degraded (13.533%); 12123/34944 objects misplaced (34.693%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T09:11:09.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:09 smithi016 bash[77892]: cluster 2023-10-09T09:11:08.019812+0000 mgr.y (mgr.25421) 1014 : cluster 0 pgmap v1080: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 727 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 4729/34944 objects degraded (13.533%); 12123/34944 objects misplaced (34.693%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T09:11:09.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:09.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:09.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:09.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:09.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:09.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:10.116 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:11:10.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:10.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:10.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:10.300 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:10.332 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:10.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:10.437 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:10.446 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:10 smithi094 bash[59677]: cluster 2023-10-09T09:11:09.664733+0000 mon.a (mon.0) 1367 : cluster 3 Health check update: Degraded data redundancy: 4729/34944 objects degraded (13.533%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:10.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:10.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:10.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:10 smithi016 bash[75179]: cluster 2023-10-09T09:11:09.664733+0000 mon.a (mon.0) 1367 : cluster 3 Health check update: Degraded data redundancy: 4729/34944 objects degraded (13.533%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:10.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:10 smithi016 bash[77892]: cluster 2023-10-09T09:11:09.664733+0000 mon.a (mon.0) 1367 : cluster 3 Health check update: Degraded data redundancy: 4729/34944 objects degraded (13.533%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:10.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:10.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:10.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:10.711 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:10.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:10.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:10.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:10.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:10.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:10.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:11.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:11.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:11.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:11.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:11.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:11 smithi016 bash[75179]: cluster 2023-10-09T09:11:10.020622+0000 mgr.y (mgr.25421) 1015 : cluster 0 pgmap v1081: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 617 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 4560/29679 objects degraded (15.364%); 9742/29679 objects misplaced (32.825%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:11:11.520 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:11 smithi016 bash[77892]: cluster 2023-10-09T09:11:10.020622+0000 mgr.y (mgr.25421) 1015 : cluster 0 pgmap v1081: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 617 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 4560/29679 objects degraded (15.364%); 9742/29679 objects misplaced (32.825%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:11:11.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:11.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:11.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:11.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:11.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:11.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:11.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:11 smithi094 bash[59677]: cluster 2023-10-09T09:11:10.020622+0000 mgr.y (mgr.25421) 1015 : cluster 0 pgmap v1081: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 617 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 4560/29679 objects degraded (15.364%); 9742/29679 objects misplaced (32.825%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:11:11.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:11.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:11.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:12.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:12.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:12.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:12.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:12.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:12.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:12.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:12.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:12.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:12.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:12.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:12.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:12.765 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:12.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:12.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:12.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:12.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:12.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:13.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:13.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:13.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:13.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:13.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:13.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:13.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:13.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:13.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:13.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:13.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:13 smithi016 bash[77892]: cluster 2023-10-09T09:11:12.021466+0000 mgr.y (mgr.25421) 1016 : cluster 0 pgmap v1082: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 539 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 4370/25917 objects degraded (16.862%); 9347/25917 objects misplaced (36.065%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T09:11:13.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:13 smithi016 bash[75179]: cluster 2023-10-09T09:11:12.021466+0000 mgr.y (mgr.25421) 1016 : cluster 0 pgmap v1082: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 539 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 4370/25917 objects degraded (16.862%); 9347/25917 objects misplaced (36.065%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T09:11:13.513 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:13.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:13 smithi094 bash[59677]: cluster 2023-10-09T09:11:12.021466+0000 mgr.y (mgr.25421) 1016 : cluster 0 pgmap v1082: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 539 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 4370/25917 objects degraded (16.862%); 9347/25917 objects misplaced (36.065%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T09:11:13.544 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:13.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:13.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:13.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:13.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:13.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:13.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:13.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:13.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:13.906 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:14.036 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:14.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:14.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:14.178 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:14.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:14.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:14.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:14.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:14.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:14.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:14.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:14.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:14.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:14.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:14.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:14.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:14.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:14.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:14.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:14.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:14.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:15.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:15.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:15.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:15.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:15.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:15.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:15.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:15.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:15.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:15.408 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:15.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:15.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:15 smithi094 bash[59677]: cluster 2023-10-09T09:11:14.022083+0000 mgr.y (mgr.25421) 1017 : cluster 0 pgmap v1083: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 431 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 4182/20733 objects degraded (20.171%); 6956/20733 objects misplaced (33.550%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:11:15.495 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:15 smithi094 bash[59677]: cluster 2023-10-09T09:11:14.665813+0000 mon.a (mon.0) 1368 : cluster 3 Health check update: Degraded data redundancy: 4182/20733 objects degraded (20.171%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:15.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:15.544 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:15.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:15.612 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:15 smithi016 bash[75179]: cluster 2023-10-09T09:11:14.022083+0000 mgr.y (mgr.25421) 1017 : cluster 0 pgmap v1083: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 431 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 4182/20733 objects degraded (20.171%); 6956/20733 objects misplaced (33.550%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:11:15.612 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:15 smithi016 bash[75179]: cluster 2023-10-09T09:11:14.665813+0000 mon.a (mon.0) 1368 : cluster 3 Health check update: Degraded data redundancy: 4182/20733 objects degraded (20.171%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:15.613 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:15 smithi016 bash[77892]: cluster 2023-10-09T09:11:14.022083+0000 mgr.y (mgr.25421) 1017 : cluster 0 pgmap v1083: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 431 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 4182/20733 objects degraded (20.171%); 6956/20733 objects misplaced (33.550%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T09:11:15.613 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:15 smithi016 bash[77892]: cluster 2023-10-09T09:11:14.665813+0000 mon.a (mon.0) 1368 : cluster 3 Health check update: Degraded data redundancy: 4182/20733 objects degraded (20.171%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:15.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:15.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:15.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:15.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:15.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:15.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:15.922 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:15.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:15.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:16.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:16.069 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:11:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:11:15] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-10-09T09:11:16.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:16.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:16.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:16.234 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:16.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:16.340 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:16.367 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:16.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:16.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:16.472 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:16.504 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:16.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:16.608 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:16.638 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:16.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:16.748 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:16.770 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:16.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:16.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:16.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:16.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:16.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:17.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:17.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:17.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:17.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:17.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:17.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:17.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.293 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:17.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:17.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:17.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:17.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:17.552 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:17 smithi016 bash[77892]: cluster 2023-10-09T09:11:16.023010+0000 mgr.y (mgr.25421) 1018 : cluster 0 pgmap v1084: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 360 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 4013/17319 objects degraded (23.171%); 6753/17319 objects misplaced (38.992%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:11:17.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:17 smithi016 bash[75179]: cluster 2023-10-09T09:11:16.023010+0000 mgr.y (mgr.25421) 1018 : cluster 0 pgmap v1084: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 360 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 4013/17319 objects degraded (23.171%); 6753/17319 objects misplaced (38.992%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:11:17.560 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:17.564 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:17 smithi094 bash[59677]: cluster 2023-10-09T09:11:16.023010+0000 mgr.y (mgr.25421) 1018 : cluster 0 pgmap v1084: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 360 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 4013/17319 objects degraded (23.171%); 6753/17319 objects misplaced (38.992%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:11:17.664 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:17.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:17.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:17.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:17.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:17.891 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:11:17.934 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:11:17.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:17.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:17.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:17.977 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:11:18.019 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:11:18.051 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:11:18.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:18.100 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:11:18.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:18.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:18.144 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:11:18.192 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.198 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:11:18.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:18.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:18.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:18.299 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:11:18.345 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:18 smithi016 bash[75179]: audit 2023-10-09T09:11:17.840516+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.16:0/1998977764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:18.345 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:18 smithi016 bash[75179]: audit 2023-10-09T09:11:18.257780+0000 mon.a (mon.0) 1369 : audit 1 from='client.? 172.21.15.16:0/2760408859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:11:18.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:18 smithi016 bash[77892]: audit 2023-10-09T09:11:17.840516+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.16:0/1998977764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:18.346 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:18 smithi016 bash[77892]: audit 2023-10-09T09:11:18.257780+0000 mon.a (mon.0) 1369 : audit 1 from='client.? 172.21.15.16:0/2760408859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:11:18.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:18.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:18.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:18.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:18.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:18.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:18.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:18.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:18.632 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:18 smithi094 bash[59677]: audit 2023-10-09T09:11:17.840516+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.16:0/1998977764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:18.633 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:18 smithi094 bash[59677]: audit 2023-10-09T09:11:18.257780+0000 mon.a (mon.0) 1369 : audit 1 from='client.? 172.21.15.16:0/2760408859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:11:18.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:18.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:18.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:18.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:18.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:18.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:18.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:18.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:18.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:18.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:18.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:19.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:19.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:19.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:19.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:19.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: cluster 2023-10-09T09:11:18.023883+0000 mgr.y (mgr.25421) 1019 : cluster 0 pgmap v1085: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 354 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 4013/17019 objects degraded (23.580%); 6553/17019 objects misplaced (38.504%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: cluster 2023-10-09T09:11:18.274502+0000 mon.a (mon.0) 1370 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: cluster 2023-10-09T09:11:18.285787+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: audit 2023-10-09T09:11:18.291105+0000 mon.a (mon.0) 1371 : audit 1 from='client.? 172.21.15.16:0/2760408859' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: cluster 2023-10-09T09:11:18.291376+0000 mon.a (mon.0) 1372 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: cluster 2023-10-09T09:11:18.297594+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: cluster 2023-10-09T09:11:18.298536+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: audit 2023-10-09T09:11:19.249284+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:19.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:19.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:19.455 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:19.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: cluster 2023-10-09T09:11:18.023883+0000 mgr.y (mgr.25421) 1019 : cluster 0 pgmap v1085: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 354 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 4013/17019 objects degraded (23.580%); 6553/17019 objects misplaced (38.504%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T09:11:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: cluster 2023-10-09T09:11:18.274502+0000 mon.a (mon.0) 1370 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:11:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: cluster 2023-10-09T09:11:18.285787+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: audit 2023-10-09T09:11:18.291105+0000 mon.a (mon.0) 1371 : audit 1 from='client.? 172.21.15.16:0/2760408859' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:11:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: cluster 2023-10-09T09:11:18.291376+0000 mon.a (mon.0) 1372 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: cluster 2023-10-09T09:11:18.297594+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: cluster 2023-10-09T09:11:18.298536+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: audit 2023-10-09T09:11:19.249284+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[75179]: audit 2023-10-09T09:11:19.250546+0000 mon.a (mon.0) 1374 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: cluster 2023-10-09T09:11:18.023883+0000 mgr.y (mgr.25421) 1019 : cluster 0 pgmap v1085: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 354 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 4013/17019 objects degraded (23.580%); 6553/17019 objects misplaced (38.504%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: cluster 2023-10-09T09:11:18.274502+0000 mon.a (mon.0) 1370 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: cluster 2023-10-09T09:11:18.285787+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: audit 2023-10-09T09:11:18.291105+0000 mon.a (mon.0) 1371 : audit 1 from='client.? 172.21.15.16:0/2760408859' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:11:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: cluster 2023-10-09T09:11:18.291376+0000 mon.a (mon.0) 1372 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T09:11:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: cluster 2023-10-09T09:11:18.297594+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: cluster 2023-10-09T09:11:18.298536+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T09:11:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: audit 2023-10-09T09:11:19.249284+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:19 smithi016 bash[77892]: audit 2023-10-09T09:11:19.250546+0000 mon.a (mon.0) 1374 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:19.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:19.580 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:11:19.580 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T09:11:19.580 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:11:19.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:19.611 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:19 smithi094 bash[59677]: audit 2023-10-09T09:11:19.250546+0000 mon.a (mon.0) 1374 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:19.667 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:11:19.667 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:11:19.667 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:11:19.667 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:11:19.667 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:11:19.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:19.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:19.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:19.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:19.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:19.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:19.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:20.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:20.115 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:20.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:20.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:20.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:20.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:20.356 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:20.365 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:20 smithi094 bash[59677]: cluster 2023-10-09T09:11:19.666930+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: Degraded data redundancy: 4013/17019 objects degraded (23.580%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:20.381 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:20.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:20.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:20.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:20 smithi016 bash[75179]: cluster 2023-10-09T09:11:19.666930+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: Degraded data redundancy: 4013/17019 objects degraded (23.580%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:20.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:20 smithi016 bash[77892]: cluster 2023-10-09T09:11:19.666930+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: Degraded data redundancy: 4013/17019 objects degraded (23.580%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:20.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:20.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:20.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:20.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:20.891 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:20.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:20.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:21.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:21.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:21.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:21.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:21.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:21.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:21.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:21.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:21.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:21.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:21.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:21.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:21.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:21 smithi094 bash[59677]: cluster 2023-10-09T09:11:20.024557+0000 mgr.y (mgr.25421) 1020 : cluster 0 pgmap v1087: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 231 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 3803/11148 objects degraded (34.114%); 3860/11148 objects misplaced (34.625%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:21.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:21.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:21 smithi016 bash[75179]: cluster 2023-10-09T09:11:20.024557+0000 mgr.y (mgr.25421) 1020 : cluster 0 pgmap v1087: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 231 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 3803/11148 objects degraded (34.114%); 3860/11148 objects misplaced (34.625%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:21.539 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:21 smithi016 bash[77892]: cluster 2023-10-09T09:11:20.024557+0000 mgr.y (mgr.25421) 1020 : cluster 0 pgmap v1087: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 231 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 3803/11148 objects degraded (34.114%); 3860/11148 objects misplaced (34.625%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:21.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:21.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:21.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:21.664 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:22.263 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:22.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:22.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:22.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:22.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:22.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:22.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:22.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:22.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:22.641 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:22.661 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 96583 objects 2023-10-09T09:11:22.661 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :174.223 2023-10-09T09:11:22.670 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:22.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:22.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:22.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:22.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:22.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:22.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:22.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:23.047 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:23.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:11:23.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:11:23.281 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:23.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:23.293 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:11:23.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:23.411 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:23.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:23.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:23 smithi016 bash[75179]: cluster 2023-10-09T09:11:22.025384+0000 mgr.y (mgr.25421) 1021 : cluster 0 pgmap v1088: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 150 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 3598/7248 objects degraded (49.641%); 3468/7248 objects misplaced (47.848%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:11:23.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:23 smithi016 bash[77892]: cluster 2023-10-09T09:11:22.025384+0000 mgr.y (mgr.25421) 1021 : cluster 0 pgmap v1088: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 150 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 3598/7248 objects degraded (49.641%); 3468/7248 objects misplaced (47.848%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:11:23.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:23.525 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:23 smithi094 bash[59677]: cluster 2023-10-09T09:11:22.025384+0000 mgr.y (mgr.25421) 1021 : cluster 0 pgmap v1088: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 150 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 3598/7248 objects degraded (49.641%); 3468/7248 objects misplaced (47.848%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:11:23.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:23.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:23.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:23.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:23.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:23.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:23.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:23.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:23.838 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:23.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:23.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:23.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:23.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:23.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:24.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:24.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:24.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:24.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:24.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:24.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:24.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:24.348 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:24.377 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:24.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:24.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:24.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:24.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:24.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:24.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:24.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:24.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:24.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:24.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:24.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:24.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:24.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:24.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:24.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:24.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:25.022 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:25.051 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:25.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:25.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:25.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:25.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:25.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:25.291 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:25.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:25 smithi016 bash[77892]: cluster 2023-10-09T09:11:24.026540+0000 mgr.y (mgr.25421) 1022 : cluster 0 pgmap v1089: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 49 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 3430/2391 objects degraded (143.455%); 1330/2391 objects misplaced (55.625%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:11:25.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:25 smithi016 bash[77892]: cluster 2023-10-09T09:11:24.668019+0000 mon.a (mon.0) 1376 : cluster 3 Health check update: Degraded data redundancy: 3430/2391 objects degraded (143.455%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:25 smithi016 bash[75179]: cluster 2023-10-09T09:11:24.026540+0000 mgr.y (mgr.25421) 1022 : cluster 0 pgmap v1089: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 49 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 3430/2391 objects degraded (143.455%); 1330/2391 objects misplaced (55.625%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:11:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:25 smithi016 bash[75179]: cluster 2023-10-09T09:11:24.668019+0000 mon.a (mon.0) 1376 : cluster 3 Health check update: Degraded data redundancy: 3430/2391 objects degraded (143.455%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:25 smithi094 bash[59677]: cluster 2023-10-09T09:11:24.026540+0000 mgr.y (mgr.25421) 1022 : cluster 0 pgmap v1089: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 49 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 3430/2391 objects degraded (143.455%); 1330/2391 objects misplaced (55.625%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:11:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:25 smithi094 bash[59677]: cluster 2023-10-09T09:11:24.668019+0000 mon.a (mon.0) 1376 : cluster 3 Health check update: Degraded data redundancy: 3430/2391 objects degraded (143.455%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:25.755 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T09:11:25.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T09:11:25.768 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:25.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:25.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:25.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:25.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:25.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:25.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:11:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:11:25] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.33.4" 2023-10-09T09:11:26.008 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:26.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:26.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:26.164 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:26.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:26.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:26.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:26.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:26.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:26.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:26.541 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:26.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:26.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:26.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:26.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:26.809 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:26.841 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:26.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:26.974 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:26.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:26.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:27.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:27.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:27.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:27.213 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:27.258 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:27.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:27.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:27.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:27 smithi016 bash[75179]: cluster 2023-10-09T09:11:26.027911+0000 mgr.y (mgr.25421) 1023 : cluster 0 pgmap v1090: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 12 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3373/606 objects degraded (556.601%); 1274/606 objects misplaced (210.231%); 5.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:27.374 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:27 smithi016 bash[77892]: cluster 2023-10-09T09:11:26.027911+0000 mgr.y (mgr.25421) 1023 : cluster 0 pgmap v1090: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 12 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3373/606 objects degraded (556.601%); 1274/606 objects misplaced (210.231%); 5.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:27.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:27.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:27.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:27.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:27 smithi094 bash[59677]: cluster 2023-10-09T09:11:26.027911+0000 mgr.y (mgr.25421) 1023 : cluster 0 pgmap v1090: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 12 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3373/606 objects degraded (556.601%); 1274/606 objects misplaced (210.231%); 5.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:27.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:27.527 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:27.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:27.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:27.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:27.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:27.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:27.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:27.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:27.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:27.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:27.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:27.854 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:27.855 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:27.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:27.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:27.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:27.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:28.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:28.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:28.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:28.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:28.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:28.246 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:11:28.246 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:11:28.246 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T09:11:28.246 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:11:28.246 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:11:28.246 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T09:11:28.298 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:28.322 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:28.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:28.365 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:11:28.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:28.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:28.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:28.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:28.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:28.686 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:28.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:28.809 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:28.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:28.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:28.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:28.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:28.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:28.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:29.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:29.107 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:29.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:29.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:29.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:29.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:29.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:29.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:29.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:29.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:29.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:29.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:29 smithi094 bash[59677]: cluster 2023-10-09T09:11:28.028548+0000 mgr.y (mgr.25421) 1024 : cluster 0 pgmap v1091: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 5.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:29.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:29.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:29.607 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:29.635 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:29.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:29.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:29.738 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:29.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:29 smithi016 bash[75179]: cluster 2023-10-09T09:11:28.028548+0000 mgr.y (mgr.25421) 1024 : cluster 0 pgmap v1091: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 5.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:29.747 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:29 smithi016 bash[77892]: cluster 2023-10-09T09:11:28.028548+0000 mgr.y (mgr.25421) 1024 : cluster 0 pgmap v1091: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 5.2 MiB/s, 115 objects/s recovering 2023-10-09T09:11:29.764 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:29.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:29.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:29.898 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:29.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:29.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:30.027 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:30.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:30.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:30.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:30.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:30.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:30.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:30.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:30.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:30.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:30.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:30.424 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:30 smithi016 bash[75179]: cluster 2023-10-09T09:11:29.670760+0000 mon.a (mon.0) 1377 : cluster 3 Health check update: Degraded data redundancy: 3373/543 objects degraded (621.179%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:30.424 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:30 smithi016 bash[77892]: cluster 2023-10-09T09:11:29.670760+0000 mon.a (mon.0) 1377 : cluster 3 Health check update: Degraded data redundancy: 3373/543 objects degraded (621.179%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:30.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:30.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:30.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:30.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:30.605 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:30 smithi094 bash[59677]: cluster 2023-10-09T09:11:29.670760+0000 mon.a (mon.0) 1377 : cluster 3 Health check update: Degraded data redundancy: 3373/543 objects degraded (621.179%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:11:30.620 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:30.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:30.724 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:30.746 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:30.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:30.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:30.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:30.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:30.976 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:31.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:31.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:31.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:31.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:31.233 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:31.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:31.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:31.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:31.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:31 smithi016 bash[75179]: cluster 2023-10-09T09:11:30.029671+0000 mgr.y (mgr.25421) 1025 : cluster 0 pgmap v1092: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 4.5 MiB/s, 98 objects/s recovering 2023-10-09T09:11:31.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:31 smithi016 bash[77892]: cluster 2023-10-09T09:11:30.029671+0000 mgr.y (mgr.25421) 1025 : cluster 0 pgmap v1092: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 4.5 MiB/s, 98 objects/s recovering 2023-10-09T09:11:31.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:31.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:31.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:31.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.720 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:31 smithi094 bash[59677]: cluster 2023-10-09T09:11:30.029671+0000 mgr.y (mgr.25421) 1025 : cluster 0 pgmap v1092: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 4.5 MiB/s, 98 objects/s recovering 2023-10-09T09:11:31.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:31.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:31.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:31.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:31.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:31.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:32.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:32.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:32.145 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:32.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:32.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:32.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:32.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:32.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:32.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:32.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:32.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:32.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:32.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:32.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:32.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:32.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:32.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:32.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:32.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:32.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:33.022 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:33.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:33.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:33.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:33.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:33.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:33.310 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:33.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:33.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:33.421 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:33 smithi016 bash[75179]: cluster 2023-10-09T09:11:32.030366+0000 mgr.y (mgr.25421) 1026 : cluster 0 pgmap v1093: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 3.3 MiB/s, 132 objects/s recovering 2023-10-09T09:11:33.422 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:33 smithi016 bash[77892]: cluster 2023-10-09T09:11:32.030366+0000 mgr.y (mgr.25421) 1026 : cluster 0 pgmap v1093: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 3.3 MiB/s, 132 objects/s recovering 2023-10-09T09:11:33.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:33.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:33.538 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:33.552 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:33 smithi094 bash[59677]: cluster 2023-10-09T09:11:32.030366+0000 mgr.y (mgr.25421) 1026 : cluster 0 pgmap v1093: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 3.3 MiB/s, 132 objects/s recovering 2023-10-09T09:11:33.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:33.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:33.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:33.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:33.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:33.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:33.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:33.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:33.969 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:33.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:33.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:33.994 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:11:34.072 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:34.096 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:34.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:34.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:34.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:34.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:34.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:34.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:34.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:34.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:34.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:34 smithi016 bash[77892]: audit 2023-10-09T09:11:34.244137+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:34.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:34 smithi016 bash[75179]: audit 2023-10-09T09:11:34.244137+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:34.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:34.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:34.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:34.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:34.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:34.766 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:34 smithi094 bash[59677]: audit 2023-10-09T09:11:34.244137+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:34.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:34.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:34.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:34.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:34.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:34.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:35.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:35.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:35.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:35.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:35.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:35.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:35.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:35.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:35.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:35.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:35.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:35.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:35.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:35 smithi016 bash[77892]: cluster 2023-10-09T09:11:34.031496+0000 mgr.y (mgr.25421) 1027 : cluster 0 pgmap v1094: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 1.2 MiB/s, 99 objects/s recovering 2023-10-09T09:11:35.494 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:35 smithi016 bash[75179]: cluster 2023-10-09T09:11:34.031496+0000 mgr.y (mgr.25421) 1027 : cluster 0 pgmap v1094: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 1.2 MiB/s, 99 objects/s recovering 2023-10-09T09:11:35.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:35.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:35.617 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:35.623 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:35 smithi094 bash[59677]: cluster 2023-10-09T09:11:34.031496+0000 mgr.y (mgr.25421) 1027 : cluster 0 pgmap v1094: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 1.2 MiB/s, 99 objects/s recovering 2023-10-09T09:11:35.638 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:35.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:35.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:35.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:11:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:11:35] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.33.4" 2023-10-09T09:11:35.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:35.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:35.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:35.892 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:35.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:35.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:36.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:36.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:36.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:36.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:36.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:36.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:36.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:36.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:36.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:36.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:36.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:36.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:36.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:36.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:36.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:36.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:36.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:36.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:36.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:36.749 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:36.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:36.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:36.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:36.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:36.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:36.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:36.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:37.007 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:37.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:37.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:37.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:37.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:37.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:37.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:37.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:37.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:37 smithi016 bash[75179]: cluster 2023-10-09T09:11:36.032832+0000 mgr.y (mgr.25421) 1028 : cluster 0 pgmap v1095: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 56 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 608 KiB/s, 164 objects/s recovering 2023-10-09T09:11:37.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:37 smithi016 bash[77892]: cluster 2023-10-09T09:11:36.032832+0000 mgr.y (mgr.25421) 1028 : cluster 0 pgmap v1095: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 56 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 608 KiB/s, 164 objects/s recovering 2023-10-09T09:11:37.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:37.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:37.537 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:37.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:37 smithi094 bash[59677]: cluster 2023-10-09T09:11:36.032832+0000 mgr.y (mgr.25421) 1028 : cluster 0 pgmap v1095: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 56 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 608 KiB/s, 164 objects/s recovering 2023-10-09T09:11:37.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:37.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:37.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:37.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:37.822 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:37.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:37.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:37.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:37.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:38.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:38.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:38.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:38.176 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:38.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:38.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:38.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:38.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:38.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:38.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:38.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:38.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:38.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:38.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:38.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:38.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:38.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:38.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:38.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:38.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:38.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:38.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:38.799 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:11:38.799 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T09:11:38.799 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T09:11:38.799 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:11:38.799 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.799 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68200 8.3 8:c42586fb:::benchmark_data_smithi016_151580_object68199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.799 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:37.896483+0000", 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.89809302200001, 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.017678882, 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68200 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.800 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.896483+0000", 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.896483+0000", 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.896482+0000", 2023-10-09T09:11:38.801 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.9401999999999999e-05 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.896531+0000", 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5994999999999998e-05 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.896547+0000", 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.0389999999999993e-06 2023-10-09T09:11:38.802 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.896556+0000", 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0042850980000000002 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.900841+0000", 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1749000000000001e-05 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.803 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.900853+0000", 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.006474834 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.907328+0000", 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012303600000000001 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.804 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.907451+0000", 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000189206 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.907640+0000", 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0063663310000000002 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914006+0000", 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6477000000000002e-05 2023-10-09T09:11:38.805 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914023+0000", 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.4891999999999997e-05 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914058+0000", 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6449000000000002e-05 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.806 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914104+0000", 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.8174999999999998e-05 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914162+0000", 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.017678882 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.807 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68221 8.3 8:c8b3531b:::benchmark_data_smithi016_151580_object68220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:37.922755+0000", 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.87182122299998, 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99954544400000001, 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68221 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.808 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922755+0000", 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922755+0000", 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.809 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922754+0000", 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1913000000000003e-05 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922805+0000", 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3978000000000001e-05 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.810 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922819+0000", 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5190000000000003e-06 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922827+0000", 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4059000000000005e-05 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922891+0000", 2023-10-09T09:11:38.811 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2977e-05 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.922904+0000", 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.98744285499999995 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.910347+0000", 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012157799999999999 2023-10-09T09:11:38.812 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.910468+0000", 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015218400000000001 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.910621+0000", 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003703057 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.813 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914324+0000", 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0036061499999999998 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.917930+0000", 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.004264889 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922195+0000", 2023-10-09T09:11:38.814 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.2043000000000002e-05 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922257+0000", 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3850000000000002e-05 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922301+0000", 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99954544400000001 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.815 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68224 8.3 8:c874c06f:::benchmark_data_smithi016_151580_object68223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:37.926613+0000", 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.86796368600005, 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99583267099999995, 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.816 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68224 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926613+0000", 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.817 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926613+0000", 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926611+0000", 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.9434000000000001e-05 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926660+0000", 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2696e-05 2023-10-09T09:11:38.818 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926673+0000", 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3459999999999999e-06 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926679+0000", 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00011440199999999999 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.819 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926794+0000", 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1242e-05 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.926805+0000", 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.98558710000000005 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.912392+0000", 2023-10-09T09:11:38.820 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000136317 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.912528+0000", 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013568 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.912664+0000", 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.005279499 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.821 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.917943+0000", 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0044105419999999999 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922354+0000", 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.786e-05 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.822 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922392+0000", 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.9750000000000001e-05 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922422+0000", 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.3825999999999999e-05 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922445+0000", 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99583267099999995 2023-10-09T09:11:38.823 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68225 8.3 8:cc10c687:::benchmark_data_smithi016_151580_object68224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:37.927104+0000", 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.86747239099998, 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0019742389999999, 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.824 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68225 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927104+0000", 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.825 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927104+0000", 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927103+0000", 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1254000000000001e-05 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927154+0000", 2023-10-09T09:11:38.826 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8088999999999999e-05 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927172+0000", 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5979999999999996e-06 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927178+0000", 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00059930499999999995 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.827 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927777+0000", 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0013000000000001e-05 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.927787+0000", 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.98657587099999999 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.828 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914363+0000", 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019503399999999999 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914558+0000", 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00024122299999999999 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.914799+0000", 2023-10-09T09:11:38.829 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0075641959999999996 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922364+0000", 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012345699999999999 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922487+0000", 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0064704020000000001 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.830 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.928957+0000", 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5829999999999998e-05 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.929023+0000", 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4982000000000003e-05 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.831 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.929078+0000", 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0019742389999999 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68234 8.3 8:c90e3f09:::benchmark_data_smithi016_151580_object68233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:37.934696+0000", 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.85988009499999, 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0040808109999999, 2023-10-09T09:11:38.832 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68234 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934696+0000", 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.833 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934696+0000", 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934694+0000", 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2490000000000001e-05 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.834 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934747+0000", 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0004e-05 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934767+0000", 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.4889999999999998e-06 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.835 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934774+0000", 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.5161999999999995e-05 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934869+0000", 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.038e-06 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.934877+0000", 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.98312306800000004 2023-10-09T09:11:38.836 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.918000+0000", 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000163991 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.918164+0000", 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019876900000000001 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.837 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.918363+0000", 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0040060479999999999 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922369+0000", 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0067574449999999999 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.838 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.929127+0000", 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0095376320000000007 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938664+0000", 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.2132999999999995e-05 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938736+0000", 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.0756000000000001e-05 2023-10-09T09:11:38.839 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938777+0000", 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0040808109999999 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68246 8.3 8:caf7c313:::benchmark_data_smithi016_151580_object68245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:37.946053+0000", 2023-10-09T09:11:38.840 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.84852375699995, 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99295290199999997, 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68246 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.841 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946053+0000", 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946053+0000", 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946050+0000", 2023-10-09T09:11:38.842 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0185000000000001e-05 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946110+0000", 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.218e-05 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946123+0000", 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5590000000000002e-06 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.843 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946129+0000", 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5561999999999993e-05 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946195+0000", 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0606e-05 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.844 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.946205+0000", 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.97363601499999997 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.919841+0000", 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000112187 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.919953+0000", 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017169 2023-10-09T09:11:38.845 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.920125+0000", 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00237025 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922495+0000", 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.010678261 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.846 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.933174+0000", 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.005744931 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938919+0000", 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.8044000000000003e-05 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.847 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938967+0000", 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.8850000000000002e-05 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.939005+0000", 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99295290199999997 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.848 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68260 8.3 8:c9903679:::benchmark_data_smithi016_151580_object68259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:37.963226+0000", 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.83135061799999, 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.97593001700000004, 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68260 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.849 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963226+0000", 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963226+0000", 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.850 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963225+0000", 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7416000000000004e-05 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963322+0000", 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0475999999999999e-05 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963332+0000", 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.0829999999999993e-06 2023-10-09T09:11:38.851 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963342+0000", 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6264e-05 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963418+0000", 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.2232e-05 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.852 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:37.963440+0000", 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.95908235200000003 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922522+0000", 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7835000000000001e-05 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922620+0000", 2023-10-09T09:11:38.853 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010914899999999999 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.922729+0000", 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0063650820000000002 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.929094+0000", 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0095184469999999993 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.854 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938613+0000", 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000461724 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.939075+0000", 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.0729000000000002e-05 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.855 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.939125+0000", 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.0420999999999999e-05 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.939156+0000", 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.97593001700000004 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68304 8.3 8:cf7c91a2:::benchmark_data_smithi016_151580_object68303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.856 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:38.016669+0000", 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.77790751600003, 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.92261869399999996, 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68304 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.857 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016669+0000", 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016669+0000", 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016667+0000", 2023-10-09T09:11:38.858 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6734999999999999e-05 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016714+0000", 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2326e-05 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016726+0000", 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1970000000000004e-06 2023-10-09T09:11:38.859 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016731+0000", 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0730000000000003e-05 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016792+0000", 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4063000000000001e-05 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.860 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.016806+0000", 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90796790100000002 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.924774+0000", 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000147956 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.924922+0000", 2023-10-09T09:11:38.861 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000267586 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.925190+0000", 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0060693869999999999 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.931259+0000", 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0075927700000000004 2023-10-09T09:11:38.862 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938852+0000", 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00035573700000000001 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.939208+0000", 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3625999999999999e-05 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.863 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.939251+0000", 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6313000000000003e-05 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.939288+0000", 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.92261869399999996 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.864 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68326 8.3 8:ca876072:::benchmark_data_smithi016_151580_object68325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:38.044579+0000", 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.74999707500001, 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90385495900000001, 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68326 2023-10-09T09:11:38.865 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044579+0000", 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044579+0000", 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.866 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044578+0000", 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.6243999999999998e-05 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044644+0000", 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3757000000000001e-05 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.867 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044658+0000", 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3300000000000001e-06 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044665+0000", 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5430000000000002e-05 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044731+0000", 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1062e-05 2023-10-09T09:11:38.868 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.044742+0000", 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.88442306299999995 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.929165+0000", 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014269700000000001 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.929308+0000", 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019726499999999999 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.929505+0000", 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0090486349999999993 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938553+0000", 2023-10-09T09:11:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0057643360000000001 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.944318+0000", 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0040030669999999999 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.948321+0000", 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.6625999999999995e-05 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.948387+0000", 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6737999999999999e-05 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.948434+0000", 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90385495900000001 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45122.0:68334 8.3 8:c6cbf21c:::benchmark_data_smithi016_151580_object68333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:02:38.054851+0000", 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 540.73972546200002, 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.91407541400000003, 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45122", 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2086665922", 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 68334 2023-10-09T09:11:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.054851+0000", 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.054851+0000", 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:11:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.054849+0000", 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6649e-05 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.054896+0000", 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3393999999999999e-05 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.054909+0000", 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8560000000000007e-06 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.054917+0000", 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4320999999999994e-05 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.055002+0000", 2023-10-09T09:11:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5383000000000002e-05 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.055017+0000", 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87819371499999999 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.933211+0000", 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013767099999999999 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.933348+0000", 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018886700000000001 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.933537+0000", 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0052682739999999999 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.938806+0000", 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.016764235999999998 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.955570+0000", 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.013202029000000001 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.968772+0000", 2023-10-09T09:11:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6987e-05 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.968849+0000", 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.7473999999999997e-05 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:02:38.968926+0000", 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.91407541400000003 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:143532 9.1 9:8499d7c8:::benchmark_data_smithi016_172771_object46946:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:43.387956+0000", 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 115.406620637, 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.164029567, 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 143532 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.387956+0000", 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.387956+0000", 2023-10-09T09:11:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.387954+0000", 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6190000000000002e-06 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.387962+0000", 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7445e-05 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.387979+0000", 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1731e-05 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.387991+0000", 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.5828999999999993e-05 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.388087+0000", 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3591000000000001e-05 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.388100+0000", 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.95994568800000002 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.348046+0000", 2023-10-09T09:11:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9278000000000001e-05 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.348065+0000", 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1808809259999999 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.528946+0000", 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000149414 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.529095+0000", 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010077 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.529196+0000", 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0041579590000000001 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.533354+0000", 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0094360580000000006 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.542790+0000", 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1919e-05 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.542872+0000", 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0088842369999999997 2023-10-09T09:11:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.551756+0000", 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.6055000000000001e-05 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.551852+0000", 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013281299999999999 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.551985+0000", 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.164029567 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:143534 9.1 9:8a59e857:::benchmark_data_smithi016_172771_object46948:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:43.395202+0000", 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 115.399374788, 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1570437240000002, 2023-10-09T09:11:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 143534 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395202+0000", 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395202+0000", 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395199+0000", 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1929e-05 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395211+0000", 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7226999999999999e-05 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395238+0000", 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3416e-05 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395252+0000", 2023-10-09T09:11:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014888999999999999 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395401+0000", 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6339000000000001e-05 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:43.395417+0000", 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.95270187100000003 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.348119+0000", 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3753e-05 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.348133+0000", 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1854594039999999 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.533592+0000", 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016434699999999999 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.533756+0000", 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018517100000000001 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.533942+0000", 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0089967899999999993 2023-10-09T09:11:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.542938+0000", 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3152999999999999e-05 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.542951+0000", 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6421000000000006e-05 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.543028+0000", 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0090239159999999999 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.552052+0000", 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5865999999999996e-05 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.552118+0000", 2023-10-09T09:11:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000127577 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.552245+0000", 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1570437240000002 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:143547 9.1 9:80fe2525:::benchmark_data_smithi016_172771_object46961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T09:11:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:44.631894+0000", 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 114.162682327, 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.92053106799999995, 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 143547 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631894+0000", 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631894+0000", 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631892+0000", 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3849999999999996e-06 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631900+0000", 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3597e-05 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631913+0000", 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3689999999999998e-06 2023-10-09T09:11:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631919+0000", 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.7666e-05 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631976+0000", 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1797000000000001e-05 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:44.631988+0000", 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90561114799999998 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.537599+0000", 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000211731 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.537811+0000", 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00025437700000000002 2023-10-09T09:11:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.538065+0000", 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0048973660000000002 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.542963+0000", 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012542200000000001 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.543088+0000", 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.6594999999999995e-05 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.543155+0000", 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0091443040000000007 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.552299+0000", 2023-10-09T09:11:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.0891e-05 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.552330+0000", 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.5167999999999994e-05 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:45.552425+0000", 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.92053106799999995 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:154148 9.1 9:88a67295:::benchmark_data_smithi016_172771_object57562:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:06.413767+0000", 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 92.380809604000007, 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0489594719999999, 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 154148 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413767+0000", 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413767+0000", 2023-10-09T09:11:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413764+0000", 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8949999999999993e-06 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413774+0000", 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5377000000000001e-05 2023-10-09T09:11:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413800+0000", 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.0220000000000004e-06 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413808+0000", 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6119999999999999e-05 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413854+0000", 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000143898 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.413998+0000", 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016378700000000001 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.414162+0000", 2023-10-09T09:11:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0036628519999999999 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.417824+0000", 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0089726059999999993 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.426797+0000", 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00042728299999999998 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.427224+0000", 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.006178317 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.433403+0000", 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8210000000000008e-06 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.433411+0000", 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.029191596 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.462603+0000", 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012318299999999999 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.462726+0000", 2023-10-09T09:11:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0489594719999999 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:154176 9.1 9:8b276850:::benchmark_data_smithi016_172771_object57590:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:06.475214+0000", 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 92.319362115000004, 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.009085893, 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 154176 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475214+0000", 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475214+0000", 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475212+0000", 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0339999999999999e-05 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475222+0000", 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.3572999999999999e-05 2023-10-09T09:11:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475246+0000", 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1450000000000008e-06 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475254+0000", 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000115284 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475369+0000", 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7860000000000008e-06 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.475379+0000", 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.98782219500000001 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.463201+0000", 2023-10-09T09:11:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016084899999999999 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.463362+0000", 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000191347 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.463554+0000", 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0090660159999999997 2023-10-09T09:11:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.472620+0000", 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014848699999999999 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.472768+0000", 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.011372915000000001 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484141+0000", 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8852999999999998e-05 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484230+0000", 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0214000000000002e-05 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484300+0000", 2023-10-09T09:11:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.009085893 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:154181 9.1 9:81483f3f:::benchmark_data_smithi016_172771_object57595:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:06.487007+0000", 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 92.307569393999998, 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99757711500000001, 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.924 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 154181 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487007+0000", 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.925 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487007+0000", 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487005+0000", 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5629999999999997e-06 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.926 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487012+0000", 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2074e-05 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487024+0000", 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4620000000000004e-06 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487029+0000", 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000105486 2023-10-09T09:11:38.927 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487135+0000", 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.093e-05 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.487146+0000", 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.98123242600000005 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.928 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.468378+0000", 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000146749 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.468525+0000", 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012593000000000001 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.468651+0000", 2023-10-09T09:11:38.929 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0040370370000000003 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.472688+0000", 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.011385947 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484074+0000", 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000356752 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.930 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484430+0000", 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8799000000000001e-05 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484519+0000", 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4845999999999995e-05 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.931 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484584+0000", 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99757711500000001 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:154187 9.1 9:8f4184ce:::benchmark_data_smithi016_172771_object57601:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:06.492831+0000", 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 92.301745041000004, 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99200780099999997, 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.932 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 154187 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492831+0000", 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.933 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492831+0000", 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492830+0000", 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.43e-06 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.934 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492837+0000", 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.311e-05 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492851+0000", 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8779999999999998e-06 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492858+0000", 2023-10-09T09:11:38.935 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.7453000000000001e-05 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492906+0000", 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4509999999999997e-06 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.492912+0000", 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.97990456999999997 2023-10-09T09:11:38.936 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.472817+0000", 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015950400000000001 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.472976+0000", 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000159448 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.937 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.473136+0000", 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.011179147 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484315+0000", 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.8675000000000003e-05 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484374+0000", 2023-10-09T09:11:38.938 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00032957400000000001 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484703+0000", 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.2636e-05 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484776+0000", 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3168000000000002e-05 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.939 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484839+0000", 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99200780099999997 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:154190 9.1 9:87207b03:::benchmark_data_smithi016_172771_object57604:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:06.498127+0000", 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 92.296448920000003, 2023-10-09T09:11:38.940 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.000220324, 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 154190 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498127+0000", 2023-10-09T09:11:38.941 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498127+0000", 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498125+0000", 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.9800000000000004e-06 2023-10-09T09:11:38.942 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498134+0000", 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7637000000000001e-05 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498152+0000", 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8879999999999992e-06 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.943 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498162+0000", 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0380999999999997e-05 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498222+0000", 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2459e-05 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.498235+0000", 2023-10-09T09:11:38.944 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.97986447399999999 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.478099+0000", 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017497300000000001 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.478274+0000", 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000158839 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.945 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.478433+0000", 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0062134729999999997 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484646+0000", 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00025544599999999999 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.946 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.484902+0000", 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.013355328 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.498257+0000", 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.1057999999999999e-05 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.947 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.949 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.498298+0000", 2023-10-09T09:11:38.949 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.9449e-05 2023-10-09T09:11:38.949 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.949 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.949 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.949 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.498348+0000", 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.000220324 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:154227 9.1 9:8302de31:::benchmark_data_smithi016_172771_object57641:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:06.598096+0000", 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 92.196480097999995, 2023-10-09T09:11:38.950 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.91208339000000005, 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 154227 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598096+0000", 2023-10-09T09:11:38.951 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598096+0000", 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598094+0000", 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0961999999999999e-05 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.952 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598105+0000", 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9426e-05 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598124+0000", 2023-10-09T09:11:38.953 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7390000000000005e-06 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598133+0000", 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3096000000000001e-05 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.954 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598186+0000", 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3157e-05 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.598199+0000", 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.892117202 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.490317+0000", 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013857999999999999 2023-10-09T09:11:38.955 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.490455+0000", 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013955900000000001 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.490595+0000", 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.007942635 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.956 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.498537+0000", 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029714369999999999 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.501509+0000", 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0085855500000000008 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.957 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.510094+0000", 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.8930999999999997e-05 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.510133+0000", 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6372000000000003e-05 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.510180+0000", 2023-10-09T09:11:38.958 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.91208339000000005 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:154233 9.1 9:8cc6735e:::benchmark_data_smithi016_172771_object57647:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:06.612972+0000", 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 92.181604730999993, 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90558593099999996, 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:11:38.959 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 154233 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.612972+0000", 2023-10-09T09:11:38.960 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.612972+0000", 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999886 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.612960+0000", 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9344999999999999e-05 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.961 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.612980+0000", 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8128e-05 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.612998+0000", 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.3860000000000007e-06 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.962 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.613006+0000", 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6254000000000001e-05 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.613052+0000", 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8879e-05 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:06.613071+0000", 2023-10-09T09:11:38.963 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.88121342000000003 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.494285+0000", 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013260799999999999 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.494417+0000", 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00018261899999999999 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.964 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.494600+0000", 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.015617865999999999 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.510218+0000", 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1787999999999999e-05 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:11:38.965 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.510240+0000", 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0082245500000000006 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.518464+0000", 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1693999999999997e-05 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.518516+0000", 2023-10-09T09:11:38.966 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.1767000000000002e-05 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.518558+0000", 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90558593099999996 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:11:38.967 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:11:38.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:38.991 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:38.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:39.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:39.116 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:39.163 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:11:39.185 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:11:39.206 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:11:39.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:39.226 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:11:39.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:39.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:39.252 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:11:39.281 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:11:39.305 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:11:39.340 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.344 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:11:39.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:39.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:39.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:39.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:39.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:39.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:39 smithi016 bash[75179]: cluster 2023-10-09T09:11:38.033430+0000 mgr.y (mgr.25421) 1029 : cluster 0 pgmap v1096: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 0 B/s, 129 objects/s recovering 2023-10-09T09:11:39.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:39 smithi016 bash[75179]: audit 2023-10-09T09:11:39.135206+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.16:0/3584132279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:39.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:39 smithi016 bash[77892]: cluster 2023-10-09T09:11:38.033430+0000 mgr.y (mgr.25421) 1029 : cluster 0 pgmap v1096: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 0 B/s, 129 objects/s recovering 2023-10-09T09:11:39.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:39 smithi016 bash[77892]: audit 2023-10-09T09:11:39.135206+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.16:0/3584132279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:39.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:39.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:39.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:39 smithi094 bash[59677]: cluster 2023-10-09T09:11:38.033430+0000 mgr.y (mgr.25421) 1029 : cluster 0 pgmap v1096: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 0 B/s, 129 objects/s recovering 2023-10-09T09:11:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:39 smithi094 bash[59677]: audit 2023-10-09T09:11:39.135206+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.16:0/3584132279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:39.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:39.856 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:39.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:39.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:39.977 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:39.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:39.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:40.099 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:40.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:40.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:40.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:40.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:40.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:40.350 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:11:40.351 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_3' removed 2023-10-09T09:11:40.355 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:40.428 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:40.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:40.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: cluster 2023-10-09T09:11:39.322124+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: cluster 2023-10-09T09:11:39.325705+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T09:11:40.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: cluster 2023-10-09T09:11:39.332651+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: cluster 2023-10-09T09:11:39.334496+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: cluster 2023-10-09T09:11:39.334680+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: audit 2023-10-09T09:11:39.818085+0000 mon.a (mon.0) 1380 : audit 1 from='client.? 172.21.15.16:0/2534371447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: audit 2023-10-09T09:11:39.893140+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.16:0/1174401825' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[75179]: audit 2023-10-09T09:11:39.893772+0000 mon.a (mon.0) 1381 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: cluster 2023-10-09T09:11:39.322124+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: cluster 2023-10-09T09:11:39.325705+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: cluster 2023-10-09T09:11:39.332651+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: cluster 2023-10-09T09:11:39.334496+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: cluster 2023-10-09T09:11:39.334680+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: audit 2023-10-09T09:11:39.818085+0000 mon.a (mon.0) 1380 : audit 1 from='client.? 172.21.15.16:0/2534371447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:11:40.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: audit 2023-10-09T09:11:39.893140+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.16:0/1174401825' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:11:40.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:40 smithi016 bash[77892]: audit 2023-10-09T09:11:39.893772+0000 mon.a (mon.0) 1381 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:11:40.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:40.554 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:40.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:40.656 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:40.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: cluster 2023-10-09T09:11:39.322124+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: cluster 2023-10-09T09:11:39.325705+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T09:11:40.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: cluster 2023-10-09T09:11:39.332651+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: cluster 2023-10-09T09:11:39.334496+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: cluster 2023-10-09T09:11:39.334680+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T09:11:40.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: audit 2023-10-09T09:11:39.818085+0000 mon.a (mon.0) 1380 : audit 1 from='client.? 172.21.15.16:0/2534371447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:11:40.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: audit 2023-10-09T09:11:39.893140+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.16:0/1174401825' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:11:40.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:40 smithi094 bash[59677]: audit 2023-10-09T09:11:39.893772+0000 mon.a (mon.0) 1381 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:11:40.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:40.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:40.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:40.812 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:40.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:40.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:40.916 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:40.944 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:40.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:41.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:41.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:41.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:41.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:41.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:41.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:41.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:41.273 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:11:41.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:41.344 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:41.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:41.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:41.471 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:41.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:41.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:41.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: cluster 2023-10-09T09:11:40.034389+0000 mgr.y (mgr.25421) 1030 : cluster 0 pgmap v1098: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 0 B/s, 155 objects/s recovering 2023-10-09T09:11:41.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: cluster 2023-10-09T09:11:40.329053+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:11:41.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: cluster 2023-10-09T09:11:40.338056+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: cluster 2023-10-09T09:11:40.338377+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: audit 2023-10-09T09:11:40.341014+0000 mon.a (mon.0) 1383 : audit 1 from='client.? 172.21.15.16:0/2534371447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: cluster 2023-10-09T09:11:40.341046+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: audit 2023-10-09T09:11:40.341132+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: cluster 2023-10-09T09:11:40.341241+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[77892]: cluster 2023-10-09T09:11:40.347463+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: cluster 2023-10-09T09:11:40.034389+0000 mgr.y (mgr.25421) 1030 : cluster 0 pgmap v1098: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 0 B/s, 155 objects/s recovering 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: cluster 2023-10-09T09:11:40.329053+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: cluster 2023-10-09T09:11:40.338056+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: cluster 2023-10-09T09:11:40.338377+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: audit 2023-10-09T09:11:40.341014+0000 mon.a (mon.0) 1383 : audit 1 from='client.? 172.21.15.16:0/2534371447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:11:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: cluster 2023-10-09T09:11:40.341046+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: audit 2023-10-09T09:11:40.341132+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:11:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: cluster 2023-10-09T09:11:40.341241+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T09:11:41.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:41 smithi016 bash[75179]: cluster 2023-10-09T09:11:40.347463+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:41.584 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: cluster 2023-10-09T09:11:40.034389+0000 mgr.y (mgr.25421) 1030 : cluster 0 pgmap v1098: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 10 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 3373/543 objects degraded (621.179%); 1232/543 objects misplaced (226.888%); 0 B/s, 155 objects/s recovering 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: cluster 2023-10-09T09:11:40.329053+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: cluster 2023-10-09T09:11:40.338056+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: cluster 2023-10-09T09:11:40.338377+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: audit 2023-10-09T09:11:40.341014+0000 mon.a (mon.0) 1383 : audit 1 from='client.? 172.21.15.16:0/2534371447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: cluster 2023-10-09T09:11:40.341046+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: audit 2023-10-09T09:11:40.341132+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: cluster 2023-10-09T09:11:40.341241+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T09:11:41.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:41 smithi094 bash[59677]: cluster 2023-10-09T09:11:40.347463+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T09:11:41.599 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:41.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:41.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:41.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:41.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:41.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:41.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:41.761 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:11:41.761 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T09:11:41.761 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:11:41.834 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:41.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:41.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:41.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:41.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:41.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:42.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:42.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:42.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:42.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:42.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:42.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:42.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:42.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:42.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:42.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:42.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:42.604 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:11:42.605 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:11:42.605 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T09:11:42.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T09:11:42.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:42.654 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:42 smithi094 bash[59677]: cluster 2023-10-09T09:11:42.331827+0000 mon.a (mon.0) 1386 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:11:42.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:42 smithi094 bash[59677]: cluster 2023-10-09T09:11:42.331864+0000 mon.a (mon.0) 1387 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3373/543 objects degraded (621.179%), 4 pgs degraded, 1 pg undersized) 2023-10-09T09:11:42.661 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:42 smithi016 bash[75179]: cluster 2023-10-09T09:11:42.331827+0000 mon.a (mon.0) 1386 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:11:42.661 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:42 smithi016 bash[75179]: cluster 2023-10-09T09:11:42.331864+0000 mon.a (mon.0) 1387 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3373/543 objects degraded (621.179%), 4 pgs degraded, 1 pg undersized) 2023-10-09T09:11:42.661 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:42 smithi016 bash[77892]: cluster 2023-10-09T09:11:42.331827+0000 mon.a (mon.0) 1386 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:11:42.661 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:42 smithi016 bash[77892]: cluster 2023-10-09T09:11:42.331864+0000 mon.a (mon.0) 1387 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3373/543 objects degraded (621.179%), 4 pgs degraded, 1 pg undersized) 2023-10-09T09:11:42.674 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:42.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:42.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:42.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:42.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:42.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:42.938 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:42.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:43.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:43.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:43.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:43.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:43.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:43.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:43.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:43.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:11:43.347 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:43.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:43.453 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:43.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:43.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:43.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:43 smithi016 bash[77892]: cluster 2023-10-09T09:11:42.034904+0000 mgr.y (mgr.25421) 1031 : cluster 0 pgmap v1100: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:43 smithi016 bash[75179]: cluster 2023-10-09T09:11:42.034904+0000 mgr.y (mgr.25421) 1031 : cluster 0 pgmap v1100: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:43.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:43.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:43.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:43.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:43 smithi094 bash[59677]: cluster 2023-10-09T09:11:42.034904+0000 mgr.y (mgr.25421) 1031 : cluster 0 pgmap v1100: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:43.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:43.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:43.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:43.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:43.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:43.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:43.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:44.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:44.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:44.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:44.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:44.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:44.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:44.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:44.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:44.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:44.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:44.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:11:44.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:44.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:44.802 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:44.827 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:44.931 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:44.962 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:44.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:45.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:45.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:45.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:45.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:45.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:45.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:45.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:45.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:45.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:45.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:45.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:45.486 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:45 smithi016 bash[75179]: cluster 2023-10-09T09:11:44.035631+0000 mgr.y (mgr.25421) 1032 : cluster 0 pgmap v1101: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:45.487 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:45 smithi016 bash[77892]: cluster 2023-10-09T09:11:44.035631+0000 mgr.y (mgr.25421) 1032 : cluster 0 pgmap v1101: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:45.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:45.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:45.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:45.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:45.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:45 smithi094 bash[59677]: cluster 2023-10-09T09:11:44.035631+0000 mgr.y (mgr.25421) 1032 : cluster 0 pgmap v1101: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:45.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:45.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:45.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:45.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:45.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:45.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:45.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:45.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:45.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:11:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:11:45] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.33.4" 2023-10-09T09:11:46.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:46.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:46.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:46.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:46.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:46.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:46.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:46.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:46.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:46.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:46.334 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:11:46.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:46.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:46.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:46.561 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:46.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:46.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:46.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:46.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:46 smithi016 bash[75179]: cluster 2023-10-09T09:11:46.389687+0000 mon.a (mon.0) 1388 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:11:46.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:46 smithi016 bash[77892]: cluster 2023-10-09T09:11:46.389687+0000 mon.a (mon.0) 1388 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:11:46.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:46.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:46.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:46 smithi094 bash[59677]: cluster 2023-10-09T09:11:46.389687+0000 mon.a (mon.0) 1388 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:11:46.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:46.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:46.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:46.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:46.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:46.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:47.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:47.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:47.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:47.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:47.381 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:47.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:47.446 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:47.446 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:47.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:47.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:47.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:47.647 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:47 smithi094 bash[59677]: cluster 2023-10-09T09:11:46.036564+0000 mgr.y (mgr.25421) 1033 : cluster 0 pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:47.674 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:47.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:47.791 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:47 smithi016 bash[75179]: cluster 2023-10-09T09:11:46.036564+0000 mgr.y (mgr.25421) 1033 : cluster 0 pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:47.792 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:47 smithi016 bash[77892]: cluster 2023-10-09T09:11:46.036564+0000 mgr.y (mgr.25421) 1033 : cluster 0 pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:47.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:47.803 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:11:47.803 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:11:47.804 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T09:11:47.804 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:11:47.804 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:11:47.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T09:11:47.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:47.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:47.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:47.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:48.032 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:48.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:48.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:48.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:48.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:48.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:48.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:48.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:48.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:48.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:48.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:48.459 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:11:48.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:48.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:48.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:48.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:48.612 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:48.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:48.716 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:48.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:48.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:48.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:48.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:48.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:48.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:48.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:48.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:49.004 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:49.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:49.107 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:49.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:49.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:49.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:49.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:49.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:49.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:49.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:49.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:49.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:49.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:49.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:49 smithi016 bash[75179]: cluster 2023-10-09T09:11:48.036994+0000 mgr.y (mgr.25421) 1034 : cluster 0 pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:49.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:49 smithi016 bash[75179]: audit 2023-10-09T09:11:49.250687+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:49.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:49 smithi016 bash[75179]: audit 2023-10-09T09:11:49.252066+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:49.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:49 smithi016 bash[77892]: cluster 2023-10-09T09:11:48.036994+0000 mgr.y (mgr.25421) 1034 : cluster 0 pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:49.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:49 smithi016 bash[77892]: audit 2023-10-09T09:11:49.250687+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:49.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:49 smithi016 bash[77892]: audit 2023-10-09T09:11:49.252066+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:49.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:49.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:49 smithi094 bash[59677]: cluster 2023-10-09T09:11:48.036994+0000 mgr.y (mgr.25421) 1034 : cluster 0 pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:49 smithi094 bash[59677]: audit 2023-10-09T09:11:49.250687+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:49.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:49 smithi094 bash[59677]: audit 2023-10-09T09:11:49.252066+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:49.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:49.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:49.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:49.876 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:49.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:49.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:49.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:50.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:50.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:50.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:50.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:50.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:50.248 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:11:50.248 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:11:50.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:50.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:50.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:50.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:50.442 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:50.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:50.546 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:50.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:50.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:50.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:50.700 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:50.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:50.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:50.860 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:50 smithi016 bash[77892]: audit 2023-10-09T09:11:49.684665+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2023-10-09T09:11:50.860 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:50 smithi016 bash[77892]: cluster 2023-10-09T09:11:50.037706+0000 mgr.y (mgr.25421) 1035 : cluster 0 pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:50.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:50 smithi016 bash[75179]: audit 2023-10-09T09:11:49.684665+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2023-10-09T09:11:50.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:50 smithi016 bash[75179]: cluster 2023-10-09T09:11:50.037706+0000 mgr.y (mgr.25421) 1035 : cluster 0 pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:50.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:50.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:50.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:50.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:50 smithi094 bash[59677]: audit 2023-10-09T09:11:49.684665+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2023-10-09T09:11:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:50 smithi094 bash[59677]: cluster 2023-10-09T09:11:50.037706+0000 mgr.y (mgr.25421) 1035 : cluster 0 pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:51.060 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:51.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:51.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:51.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:51.218 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:51.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:51.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:51.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:11:51.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:51.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:51.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:51.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:51.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:51.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:51.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:51.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:51.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:51.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:51.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:51.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:51.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[75179]: cluster 2023-10-09T09:11:50.689537+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[75179]: audit 2023-10-09T09:11:50.691275+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2023-10-09T09:11:51.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[75179]: cluster 2023-10-09T09:11:50.691382+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[75179]: cluster 2023-10-09T09:11:50.812799+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[75179]: cluster 2023-10-09T09:11:50.813188+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[75179]: audit 2023-10-09T09:11:51.087375+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[77892]: cluster 2023-10-09T09:11:50.689537+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[77892]: audit 2023-10-09T09:11:50.691275+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[77892]: cluster 2023-10-09T09:11:50.691382+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[77892]: cluster 2023-10-09T09:11:50.812799+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[77892]: cluster 2023-10-09T09:11:50.813188+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:51 smithi016 bash[77892]: audit 2023-10-09T09:11:51.087375+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:11:51.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:51.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:51.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:51.849 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:51 smithi094 bash[59677]: cluster 2023-10-09T09:11:50.689537+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:51 smithi094 bash[59677]: audit 2023-10-09T09:11:50.691275+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2023-10-09T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:51 smithi094 bash[59677]: cluster 2023-10-09T09:11:50.691382+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:51 smithi094 bash[59677]: cluster 2023-10-09T09:11:50.812799+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:51 smithi094 bash[59677]: cluster 2023-10-09T09:11:50.813188+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:51 smithi094 bash[59677]: audit 2023-10-09T09:11:51.087375+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:11:51.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:51.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:51.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:51.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:52.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:52.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:52.112 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:52.147 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:52.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:52.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:52.272 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:52.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:52.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:52.404 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:52.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:52.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:52.578 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:11:52.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:52.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:52.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:52.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[75179]: cluster 2023-10-09T09:11:51.689757+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[75179]: cluster 2023-10-09T09:11:51.692566+0000 mon.a (mon.0) 1395 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[75179]: cluster 2023-10-09T09:11:51.692643+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[75179]: cluster 2023-10-09T09:11:51.695366+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[75179]: cluster 2023-10-09T09:11:51.838113+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[75179]: cluster 2023-10-09T09:11:52.038365+0000 mgr.y (mgr.25421) 1036 : cluster 0 pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[77892]: cluster 2023-10-09T09:11:51.689757+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[77892]: cluster 2023-10-09T09:11:51.692566+0000 mon.a (mon.0) 1395 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[77892]: cluster 2023-10-09T09:11:51.692643+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[77892]: cluster 2023-10-09T09:11:51.695366+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[77892]: cluster 2023-10-09T09:11:51.838113+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:52 smithi016 bash[77892]: cluster 2023-10-09T09:11:52.038365+0000 mgr.y (mgr.25421) 1036 : cluster 0 pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T09:11:52.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:52.796 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:52.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:52.926 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:52.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:52 smithi094 bash[59677]: cluster 2023-10-09T09:11:51.689757+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:52 smithi094 bash[59677]: cluster 2023-10-09T09:11:51.692566+0000 mon.a (mon.0) 1395 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T09:11:52.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:52 smithi094 bash[59677]: cluster 2023-10-09T09:11:51.692643+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:52 smithi094 bash[59677]: cluster 2023-10-09T09:11:51.695366+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:52 smithi094 bash[59677]: cluster 2023-10-09T09:11:51.838113+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:52.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:52 smithi094 bash[59677]: cluster 2023-10-09T09:11:52.038365+0000 mgr.y (mgr.25421) 1036 : cluster 0 pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T09:11:53.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:53.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:53.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:53.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:53.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:53.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:53.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:53.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:53.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:53.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:53.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:53.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:53.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:53.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:53.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:53.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:53.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:53.853 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:53.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:53.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:53.961 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:53 smithi094 bash[59677]: cluster 2023-10-09T09:11:51.920092+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:53.961 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:53 smithi094 bash[59677]: cluster 2023-10-09T09:11:51.991310+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:53.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:53.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:53 smithi016 bash[77892]: cluster 2023-10-09T09:11:51.920092+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:53.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:53 smithi016 bash[77892]: cluster 2023-10-09T09:11:51.991310+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:53.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:53 smithi016 bash[75179]: cluster 2023-10-09T09:11:51.920092+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:53 smithi016 bash[75179]: cluster 2023-10-09T09:11:51.991310+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T09:11:54.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:54.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:54.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:54.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:54.229 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:54.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:54.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:54.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:54.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:54.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:54.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:54.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:54.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:54.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:54.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:54.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:54.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:54.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:54.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:54.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:54.761 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:54 smithi094 bash[59677]: cluster 2023-10-09T09:11:54.039184+0000 mgr.y (mgr.25421) 1037 : cluster 0 pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:54.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:54.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:54.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:54.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:54.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:54.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:54.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:54 smithi016 bash[77892]: cluster 2023-10-09T09:11:54.039184+0000 mgr.y (mgr.25421) 1037 : cluster 0 pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:55.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:54 smithi016 bash[75179]: cluster 2023-10-09T09:11:54.039184+0000 mgr.y (mgr.25421) 1037 : cluster 0 pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:55.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:55.048 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:55.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:55.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:55.288 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:55.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:55.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:55.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:55.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:55.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:55.577 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:55.631 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:11:55.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:55.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:55.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:55.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:55.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:55.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:55.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:56.021 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:11:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:11:55] "GET /metrics HTTP/1.1" 200 34051 "" "Prometheus/2.33.4" 2023-10-09T09:11:56.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:56.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:56.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:56.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:56.187 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:56.216 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:56.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:56.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:56.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:56.329 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:56 smithi016 bash[75179]: audit 2023-10-09T09:11:55.078987+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:56.329 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:56 smithi016 bash[77892]: audit 2023-10-09T09:11:55.078987+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:56.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:56.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:56.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:56.462 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:56 smithi094 bash[59677]: audit 2023-10-09T09:11:55.078987+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:56.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:56.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:56.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:56.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:56.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:56.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:56.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:56.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:56.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:56.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:56.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:56.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:57.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:57.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:57.118 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:57.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:11:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:11:57.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:57.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:57 smithi016 bash[75179]: cluster 2023-10-09T09:11:56.040222+0000 mgr.y (mgr.25421) 1038 : cluster 0 pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:57.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:57 smithi016 bash[77892]: cluster 2023-10-09T09:11:56.040222+0000 mgr.y (mgr.25421) 1038 : cluster 0 pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:57.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:57.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:57.389 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:57 smithi094 bash[59677]: cluster 2023-10-09T09:11:56.040222+0000 mgr.y (mgr.25421) 1038 : cluster 0 pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:57.430 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:57.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:57.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:11:57.529 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:11:57.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:57.552 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:11:57.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:57.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:57.578 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:11:57.602 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:11:57.631 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:11:57.653 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:11:57.662 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:57.672 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:11:57.690 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:57.793 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:57.802 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:11:57.803 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T09:11:57.803 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:11:57.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:57.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:57.921 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:57.946 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:57.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:57.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:58.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:58.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:58.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:58.191 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:58 smithi094 bash[59677]: audit 2023-10-09T09:11:57.479860+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.16:0/1322819320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:58.191 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:58 smithi094 bash[59677]: audit 2023-10-09T09:11:57.579571+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:58.191 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:58 smithi094 bash[59677]: audit 2023-10-09T09:11:57.586363+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:58.207 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:58.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:58 smithi016 bash[75179]: audit 2023-10-09T09:11:57.479860+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.16:0/1322819320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:58.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:58 smithi016 bash[75179]: audit 2023-10-09T09:11:57.579571+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:58.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:58 smithi016 bash[75179]: audit 2023-10-09T09:11:57.586363+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:58.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:58 smithi016 bash[77892]: audit 2023-10-09T09:11:57.479860+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.16:0/1322819320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:11:58.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:58 smithi016 bash[77892]: audit 2023-10-09T09:11:57.579571+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:58.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:58 smithi016 bash[77892]: audit 2023-10-09T09:11:57.586363+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:11:58.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:58.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:11:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:11:58.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:58.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:58.570 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:58.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:58.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:58.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:58.723 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:58.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:58.826 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:58.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:58.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:58.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:58.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:58.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:59.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:59.106 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:11:59.106 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_4' created 2023-10-09T09:11:59.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:11:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:59.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:11:59.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:59.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:11:59.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:59.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:11:59.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:59 smithi016 bash[75179]: cluster 2023-10-09T09:11:58.040798+0000 mgr.y (mgr.25421) 1039 : cluster 0 pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:59.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:59 smithi016 bash[75179]: audit 2023-10-09T09:11:58.397480+0000 mon.a (mon.0) 1399 : audit 1 from='client.? 172.21.15.16:0/695612901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:11:59.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:11:59 smithi016 bash[75179]: audit 2023-10-09T09:11:58.417938+0000 mon.a (mon.0) 1400 : audit 1 from='client.? 172.21.15.16:0/1866219757' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T09:11:59.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:59 smithi016 bash[77892]: cluster 2023-10-09T09:11:58.040798+0000 mgr.y (mgr.25421) 1039 : cluster 0 pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:59.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:59 smithi016 bash[77892]: audit 2023-10-09T09:11:58.397480+0000 mon.a (mon.0) 1399 : audit 1 from='client.? 172.21.15.16:0/695612901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:11:59.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:11:59 smithi016 bash[77892]: audit 2023-10-09T09:11:58.417938+0000 mon.a (mon.0) 1400 : audit 1 from='client.? 172.21.15.16:0/1866219757' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T09:11:59.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:59.361 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:59 smithi094 bash[59677]: cluster 2023-10-09T09:11:58.040798+0000 mgr.y (mgr.25421) 1039 : cluster 0 pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:11:59.361 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:59 smithi094 bash[59677]: audit 2023-10-09T09:11:58.397480+0000 mon.a (mon.0) 1399 : audit 1 from='client.? 172.21.15.16:0/695612901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:11:59.361 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:11:59 smithi094 bash[59677]: audit 2023-10-09T09:11:58.417938+0000 mon.a (mon.0) 1400 : audit 1 from='client.? 172.21.15.16:0/1866219757' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T09:11:59.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:11:59.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:59.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:11:59.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:59.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:11:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:59.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:11:59.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:59.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:11:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:59.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:11:59.742 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:59.768 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:59.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:59.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:11:59.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:59.900 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:11:59.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:11:59.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:00.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:00.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:00.136 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:00.160 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:00.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[75179]: cluster 2023-10-09T09:11:59.087028+0000 mon.a (mon.0) 1401 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[75179]: audit 2023-10-09T09:11:59.098684+0000 mon.a (mon.0) 1402 : audit 1 from='client.? 172.21.15.16:0/695612901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[75179]: audit 2023-10-09T09:11:59.098817+0000 mon.a (mon.0) 1403 : audit 1 from='client.? 172.21.15.16:0/1866219757' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[75179]: cluster 2023-10-09T09:11:59.098929+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[75179]: cluster 2023-10-09T09:11:59.105080+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[75179]: cluster 2023-10-09T09:11:59.105199+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[77892]: cluster 2023-10-09T09:11:59.087028+0000 mon.a (mon.0) 1401 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:12:00.277 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[77892]: audit 2023-10-09T09:11:59.098684+0000 mon.a (mon.0) 1402 : audit 1 from='client.? 172.21.15.16:0/695612901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:12:00.278 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[77892]: audit 2023-10-09T09:11:59.098817+0000 mon.a (mon.0) 1403 : audit 1 from='client.? 172.21.15.16:0/1866219757' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T09:12:00.278 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[77892]: cluster 2023-10-09T09:11:59.098929+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T09:12:00.278 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[77892]: cluster 2023-10-09T09:11:59.105080+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T09:12:00.278 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:00 smithi016 bash[77892]: cluster 2023-10-09T09:11:59.105199+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T09:12:00.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:00.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:00.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:00.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:00 smithi094 bash[59677]: cluster 2023-10-09T09:11:59.087028+0000 mon.a (mon.0) 1401 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:12:00.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:00 smithi094 bash[59677]: audit 2023-10-09T09:11:59.098684+0000 mon.a (mon.0) 1402 : audit 1 from='client.? 172.21.15.16:0/695612901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:12:00.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:00 smithi094 bash[59677]: audit 2023-10-09T09:11:59.098817+0000 mon.a (mon.0) 1403 : audit 1 from='client.? 172.21.15.16:0/1866219757' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T09:12:00.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:00 smithi094 bash[59677]: cluster 2023-10-09T09:11:59.098929+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T09:12:00.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:00 smithi094 bash[59677]: cluster 2023-10-09T09:11:59.105080+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T09:12:00.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:00 smithi094 bash[59677]: cluster 2023-10-09T09:11:59.105199+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T09:12:00.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:00.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:00.532 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:00.559 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:00.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:00.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T09:12:00.662 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:00.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:00.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:00.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:00.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:00.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:00.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:01.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:01.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:01.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:01.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:01.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:01.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:01.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:01 smithi094 bash[59677]: cluster 2023-10-09T09:12:00.041235+0000 mgr.y (mgr.25421) 1040 : cluster 0 pgmap v1112: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:01 smithi094 bash[59677]: cluster 2023-10-09T09:12:00.096390+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:01 smithi094 bash[59677]: cluster 2023-10-09T09:12:00.098573+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:01 smithi094 bash[59677]: cluster 2023-10-09T09:12:00.098722+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T09:12:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:01 smithi094 bash[59677]: cluster 2023-10-09T09:12:00.098782+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:01 smithi094 bash[59677]: cluster 2023-10-09T09:12:00.099729+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:01 smithi094 bash[59677]: cluster 2023-10-09T09:12:00.100397+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:12:01.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:01.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[75179]: cluster 2023-10-09T09:12:00.041235+0000 mgr.y (mgr.25421) 1040 : cluster 0 pgmap v1112: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:01.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[75179]: cluster 2023-10-09T09:12:00.096390+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[75179]: cluster 2023-10-09T09:12:00.098573+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[75179]: cluster 2023-10-09T09:12:00.098722+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T09:12:01.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[75179]: cluster 2023-10-09T09:12:00.098782+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[75179]: cluster 2023-10-09T09:12:00.099729+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[75179]: cluster 2023-10-09T09:12:00.100397+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[77892]: cluster 2023-10-09T09:12:00.041235+0000 mgr.y (mgr.25421) 1040 : cluster 0 pgmap v1112: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:01.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[77892]: cluster 2023-10-09T09:12:00.096390+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[77892]: cluster 2023-10-09T09:12:00.098573+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[77892]: cluster 2023-10-09T09:12:00.098722+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T09:12:01.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[77892]: cluster 2023-10-09T09:12:00.098782+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[77892]: cluster 2023-10-09T09:12:00.099729+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:01 smithi016 bash[77892]: cluster 2023-10-09T09:12:00.100397+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T09:12:01.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:01.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:01.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:01.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:01.489 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:12:01.489 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:12:01.489 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:12:01.489 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:12:01.489 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:12:01.493 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:01.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:01.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:01.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:01.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:01.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:01.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:01.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:01.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:01.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:01.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:01.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:01.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:02.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:02.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:02.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:02.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:02.159 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:02 smithi094 bash[59677]: cluster 2023-10-09T09:12:01.098634+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T09:12:02.159 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:02 smithi094 bash[59677]: cluster 2023-10-09T09:12:01.101384+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T09:12:02.159 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:02 smithi094 bash[59677]: cluster 2023-10-09T09:12:01.793777+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T09:12:02.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:02.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:02.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:02.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:02.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:02 smithi016 bash[75179]: cluster 2023-10-09T09:12:01.098634+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T09:12:02.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:02 smithi016 bash[75179]: cluster 2023-10-09T09:12:01.101384+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T09:12:02.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:02 smithi016 bash[75179]: cluster 2023-10-09T09:12:01.793777+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T09:12:02.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:02 smithi016 bash[77892]: cluster 2023-10-09T09:12:01.098634+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T09:12:02.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:02 smithi016 bash[77892]: cluster 2023-10-09T09:12:01.101384+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T09:12:02.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:02 smithi016 bash[77892]: cluster 2023-10-09T09:12:01.793777+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T09:12:02.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:02.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:02.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:02.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:12:02.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:02.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:02.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:02.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:02.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:02.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:02.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:12:02.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:02.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:02.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:02.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:02.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:02.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:02.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:02.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:02.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:03.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:03.125 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:03.160 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:03.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:03.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:03.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:03.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:03 smithi094 bash[59677]: cluster 2023-10-09T09:12:02.041689+0000 mgr.y (mgr.25421) 1041 : cluster 0 pgmap v1115: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:03.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:03 smithi094 bash[59677]: audit 2023-10-09T09:12:02.250425+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:03 smithi094 bash[59677]: audit 2023-10-09T09:12:02.258060+0000 mon.a (mon.0) 1408 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:03 smithi094 bash[59677]: audit 2023-10-09T09:12:02.781940+0000 mon.a (mon.0) 1409 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:03.414 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:03 smithi094 bash[59677]: audit 2023-10-09T09:12:02.783693+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:03.415 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:03 smithi094 bash[59677]: audit 2023-10-09T09:12:02.792414+0000 mon.a (mon.0) 1411 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:03.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[75179]: cluster 2023-10-09T09:12:02.041689+0000 mgr.y (mgr.25421) 1041 : cluster 0 pgmap v1115: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:03.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[75179]: audit 2023-10-09T09:12:02.250425+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[75179]: audit 2023-10-09T09:12:02.258060+0000 mon.a (mon.0) 1408 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[75179]: audit 2023-10-09T09:12:02.781940+0000 mon.a (mon.0) 1409 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:03.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[75179]: audit 2023-10-09T09:12:02.783693+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:03.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[75179]: audit 2023-10-09T09:12:02.792414+0000 mon.a (mon.0) 1411 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[77892]: cluster 2023-10-09T09:12:02.041689+0000 mgr.y (mgr.25421) 1041 : cluster 0 pgmap v1115: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:03.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[77892]: audit 2023-10-09T09:12:02.250425+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[77892]: audit 2023-10-09T09:12:02.258060+0000 mon.a (mon.0) 1408 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[77892]: audit 2023-10-09T09:12:02.781940+0000 mon.a (mon.0) 1409 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:03.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[77892]: audit 2023-10-09T09:12:02.783693+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:03.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:03 smithi016 bash[77892]: audit 2023-10-09T09:12:02.792414+0000 mon.a (mon.0) 1411 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:03.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:03.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:03.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:03.673 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:03.707 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:03.810 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:03.842 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:03.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:03.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:03.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:04.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:04.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:04.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:04.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:04.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:04.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:04.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:04.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:04.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:04.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:04 smithi094 bash[59677]: audit 2023-10-09T09:12:04.245907+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:04.495 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:04.526 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:04.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:04.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:04.549 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:12:04.549 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:12:04.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:04.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:04 smithi016 bash[75179]: audit 2023-10-09T09:12:04.245907+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:04.644 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:04 smithi016 bash[77892]: audit 2023-10-09T09:12:04.245907+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:04.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:04.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:04.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:04.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:04.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:04.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:04.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:04.940 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:04.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:05.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:05.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:05.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:05.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:05.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:05.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:05.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:05.217 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:05.218 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:05.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:05.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:05.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:05.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:05.463 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:05 smithi094 bash[59677]: cluster 2023-10-09T09:12:04.042529+0000 mgr.y (mgr.25421) 1042 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:05.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:05.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:05.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:05.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:05 smithi016 bash[75179]: cluster 2023-10-09T09:12:04.042529+0000 mgr.y (mgr.25421) 1042 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:05.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:05 smithi016 bash[77892]: cluster 2023-10-09T09:12:04.042529+0000 mgr.y (mgr.25421) 1042 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:05.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:05.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:05.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:05.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:05.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:05.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:05.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:05.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:05.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:05.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:05.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:12:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:12:05] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.33.4" 2023-10-09T09:12:06.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:06.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:06.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:06.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:06.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:06.252 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:06.288 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:06.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:06.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:06.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:06.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:06.557 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:06.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:06.661 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:06.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:06.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:06.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:06.828 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:06.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:06.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:06.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:07.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:07.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:07.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:07.197 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:12:07.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:07.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:07.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:07.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:07.248 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:07.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:07.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:07.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:07.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:07.475 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:07 smithi094 bash[59677]: cluster 2023-10-09T09:12:06.043560+0000 mgr.y (mgr.25421) 1043 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:07.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:07.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:07.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:07.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:07.603 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:07 smithi016 bash[77892]: cluster 2023-10-09T09:12:06.043560+0000 mgr.y (mgr.25421) 1043 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:07.603 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:07 smithi016 bash[75179]: cluster 2023-10-09T09:12:06.043560+0000 mgr.y (mgr.25421) 1043 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:07.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:07.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:07.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:07.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:07.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:07.858 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:07.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:07.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:07.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:08.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:08.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:08.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:08.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:08.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:08.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:08.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:08.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:08.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:08.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:08.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:08.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:08.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:08.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:08.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:08.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:08.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:08.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:08.910 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:08.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:08.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:08.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:09.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:09.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:09.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:09.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:09.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:09.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:09.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:09.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:09.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:09.445 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:09 smithi094 bash[59677]: cluster 2023-10-09T09:12:08.044069+0000 mgr.y (mgr.25421) 1044 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:09.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:09.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:09.576 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:09 smithi016 bash[75179]: cluster 2023-10-09T09:12:08.044069+0000 mgr.y (mgr.25421) 1044 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:09.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:09 smithi016 bash[77892]: cluster 2023-10-09T09:12:08.044069+0000 mgr.y (mgr.25421) 1044 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:09.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:09.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:09.692 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:09.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:09.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:09.825 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:09.857 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:09.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:09.990 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:09.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:09.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:10.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:10.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:10.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:10.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:10.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:10.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:10.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:10.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:10.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:10.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:10.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:10.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:10.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:10.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:10.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:10.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:10.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:10.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:10.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:10.836 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:10.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:10.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:10.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:10.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:11.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:11.094 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:11.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:11.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:11.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:11.226 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:11.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:11.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:11.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:11 smithi016 bash[77892]: cluster 2023-10-09T09:12:10.044627+0000 mgr.y (mgr.25421) 1045 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:11.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:11 smithi016 bash[77892]: audit 2023-10-09T09:12:10.094302+0000 mon.a (mon.0) 1413 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:11.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:11 smithi016 bash[75179]: cluster 2023-10-09T09:12:10.044627+0000 mgr.y (mgr.25421) 1045 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:11.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:11 smithi016 bash[75179]: audit 2023-10-09T09:12:10.094302+0000 mon.a (mon.0) 1413 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:11.304 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:12:11.304 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T09:12:11.304 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T09:12:11.304 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:12:11.304 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.304 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:145580 9.9 e231/229 9:96238d86:::benchmark_data_smithi016_172771_object48994:head v 231'9076, mlcod=231'9074)", 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:49.546824+0000", 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 141.75219968100001, 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.032976852000000001, 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.546824+0000", 2023-10-09T09:12:11.305 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.546824+0000", 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.546823+0000", 2023-10-09T09:12:11.306 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.9929999999999995e-06 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.546831+0000", 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0866000000000001e-05 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.546842+0000", 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0680000000000002e-06 2023-10-09T09:12:11.307 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.546848+0000", 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0037915319999999998 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550640+0000", 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1750999999999999e-05 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.308 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550661+0000", 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.028912957 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.579574+0000", 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00022647800000000001 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.309 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.579801+0000", 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.032976852000000001 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:145583 9.3 e231/229 9:c75ef31f:::benchmark_data_smithi016_172771_object48997:head v 231'9206, mlcod=231'9204)", 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:49.547483+0000", 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 141.751540904, 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.028927277000000001, 2023-10-09T09:12:11.310 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547483+0000", 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547483+0000", 2023-10-09T09:12:11.311 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:12:11.312 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.312 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.312 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.312 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547482+0000", 2023-10-09T09:12:11.312 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9240000000000001e-06 2023-10-09T09:12:11.313 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.313 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.313 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.313 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547489+0000", 2023-10-09T09:12:11.313 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8270000000000004e-06 2023-10-09T09:12:11.313 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.313 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.314 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.314 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547498+0000", 2023-10-09T09:12:11.314 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.2869999999999998e-06 2023-10-09T09:12:11.314 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.314 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.314 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547502+0000", 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3532000000000003e-05 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547546+0000", 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4416000000000001e-05 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.315 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.547560+0000", 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0049502950000000004 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.552510+0000", 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.023899688999999998 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.316 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.576410+0000", 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.028927277000000001 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:145582 9.1 e231/230 9:884f4f68:::benchmark_data_smithi016_172771_object48996:head v 231'9209, mlcod=231'9207)", 2023-10-09T09:12:11.317 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:49.550143+0000", 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 141.74888054600001, 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.036256495, 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550143+0000", 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.318 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550143+0000", 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550142+0000", 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4977000000000001e-05 2023-10-09T09:12:11.319 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550157+0000", 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1090999999999999e-05 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550168+0000", 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.285e-06 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.320 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550174+0000", 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00055491099999999995 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550729+0000", 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.067e-05 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.321 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550750+0000", 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.035615742999999998 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.586366+0000", 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3787e-05 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.322 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.586400+0000", 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.036256495 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:145581 9.2 e231/230 9:4da16012:::benchmark_data_smithi016_172771_object48995:head v 231'9040, mlcod=231'9037)", 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:49.550622+0000", 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 141.74840113499999, 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.035491100999999997, 2023-10-09T09:12:11.323 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550622+0000", 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550622+0000", 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:12:11.324 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550622+0000", 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5809999999999999e-06 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550629+0000", 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1681e-05 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.325 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550641+0000", 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5100000000000004e-06 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.550648+0000", 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.025686568 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.326 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.576334+0000", 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.4000999999999999e-05 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.576358+0000", 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0096463790000000001 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.586004+0000", 2023-10-09T09:12:11.327 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000109079 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.586114+0000", 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.035491100999999997 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.328 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:145570 9.d e231/230 9:b65dabba:::benchmark_data_smithi016_172771_object48984:head v 231'8922, mlcod=219'3033)", 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:49.555795+0000", 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 141.74322846699999, 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.032184548, 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.555795+0000", 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.329 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.555795+0000", 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.555794+0000", 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.5150000000000006e-06 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.330 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.555803+0000", 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.020261675 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.576064+0000", 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7121000000000001e-05 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.331 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.576081+0000", 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015470100000000001 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.576236+0000", 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.4409e-05 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.332 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.576261+0000", 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.011640767999999999 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.587901+0000", 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8363999999999997e-05 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.333 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:49.587980+0000", 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.032184548 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:146793 9.2 e231/230 9:49c4ea52:::benchmark_data_smithi016_172771_object50207:head v 231'9100, mlcod=231'9097)", 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:51.956530+0000", 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 139.34249384, 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.026989425000000001, 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.334 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.956530+0000", 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.956530+0000", 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.335 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.956529+0000", 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6089000000000001e-05 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.956545+0000", 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.6780000000000006e-06 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.336 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.956554+0000", 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.8400000000000002e-06 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.956558+0000", 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00047686099999999997 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.957035+0000", 2023-10-09T09:12:11.337 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3299e-05 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.957079+0000", 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.026400499000000001 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.983479+0000", 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.0095999999999997e-05 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.338 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.983519+0000", 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.026989425000000001 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:146794 9.1 e231/230 9:86edcddb:::benchmark_data_smithi016_172771_object50208:head v 231'9274, mlcod=231'9272)", 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:51.958658+0000", 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 139.340365617, 2023-10-09T09:12:11.339 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027957691999999999, 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.958658+0000", 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.340 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.958658+0000", 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.958657+0000", 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.4170000000000003e-06 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.958665+0000", 2023-10-09T09:12:11.341 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5569e-05 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.958680+0000", 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9580000000000001e-06 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.958687+0000", 2023-10-09T09:12:11.342 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.018014295999999999 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.976701+0000", 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1467e-05 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.976723+0000", 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0097882009999999998 2023-10-09T09:12:11.343 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.986511+0000", 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010452 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:51.986616+0000", 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027957691999999999 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.344 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:146822 9.3 e231/229 9:cfb1d94e:::benchmark_data_smithi016_172771_object50236:head v 231'9293, mlcod=231'9289)", 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:52.018663+0000", 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 139.280360492, 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.029249384, 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.345 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.018663+0000", 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.018663+0000", 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.346 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.018662+0000", 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6499999999999996e-06 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.018670+0000", 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1627999999999999e-05 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.018682+0000", 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0820000000000004e-06 2023-10-09T09:12:11.347 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.018688+0000", 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0072995409999999997 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.025987+0000", 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0606999999999997e-05 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.348 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.026058+0000", 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.021758422999999999 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.047816+0000", 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.6098e-05 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.349 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:52.047912+0000", 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.029249384 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.350 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:148146 9.9 e231/229 9:98873aa1:::benchmark_data_smithi016_172771_object51560:head v 231'9229, mlcod=231'9226)", 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:54.540525+0000", 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 136.75849877600001, 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.029125135999999999, 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540525+0000", 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.351 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540525+0000", 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540524+0000", 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.3899999999999993e-06 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.352 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540532+0000", 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5166e-05 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540548+0000", 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8900000000000001e-06 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.353 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540554+0000", 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4465000000000002e-05 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540599+0000", 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0848000000000001e-05 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.354 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.540620+0000", 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.028908024000000001 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.569528+0000", 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012218099999999999 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.355 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:54.569650+0000", 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.029125135999999999 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:148410 9.2 e231/230 9:4084a090:::benchmark_data_smithi016_172771_object51824:head v 231'9201, mlcod=231'9199)", 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:55.038914+0000", 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 136.26010999900001, 2023-10-09T09:12:11.356 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.03109437, 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.038914+0000", 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.038914+0000", 2023-10-09T09:12:11.357 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.038913+0000", 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9319999999999999e-06 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.038920+0000", 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4069999999999993e-06 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.358 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.038928+0000", 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.417e-06 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.038934+0000", 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0037656289999999999 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.042699+0000", 2023-10-09T09:12:11.359 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9047000000000005e-05 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.042768+0000", 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027114533 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.069883+0000", 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012519199999999999 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.360 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.070008+0000", 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.03109437 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:148411 9.2 e231/230 9:45112443:::benchmark_data_smithi016_172771_object51825:head v 231'9202, mlcod=231'9199)", 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:09:55.043125+0000", 2023-10-09T09:12:11.361 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 136.255898806, 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027023518999999999, 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.043125+0000", 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.362 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.043125+0000", 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.043124+0000", 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.4809999999999995e-06 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.043133+0000", 2023-10-09T09:12:11.363 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2955999999999999e-05 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.043146+0000", 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.246e-06 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.043152+0000", 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0080701579999999992 2023-10-09T09:12:11.364 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.051223+0000", 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.8011999999999999e-05 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.051251+0000", 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.018760237999999999 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.365 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.070011+0000", 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013742800000000001 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:09:55.070148+0000", 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027023518999999999 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.366 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:154306 9.1 e233/232 9:8245a999:::benchmark_data_smithi016_172771_object57720:head v 233'9776, mlcod=233'9772)", 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:07.523200+0000", 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 123.77582357999999, 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.031481724000000003, 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.367 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.523200+0000", 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.523200+0000", 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.368 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.523199+0000", 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6180999999999997e-05 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.523245+0000", 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.2536999999999997e-05 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.523278+0000", 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5660000000000001e-06 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.369 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.523285+0000", 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0039824429999999996 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.527268+0000", 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9533000000000001e-05 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.370 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.527287+0000", 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027341529999999999 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.554629+0000", 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3056999999999997e-05 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.371 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:07.554682+0000", 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.031481724000000003 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:155082 9.e e233/230 9:70bb98ab:::benchmark_data_smithi016_172771_object58496:head v 233'9728, mlcod=219'3671)", 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:08.942703+0000", 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 122.35632104699999, 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027384736999999999, 2023-10-09T09:12:11.372 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942703+0000", 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.373 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942703+0000", 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942702+0000", 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.5360000000000003e-06 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942707+0000", 2023-10-09T09:12:11.374 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5620000000000003e-06 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942712+0000", 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.7189999999999999e-06 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942716+0000", 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000166222 2023-10-09T09:12:11.375 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942882+0000", 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1767e-05 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.942904+0000", 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027141826000000001 2023-10-09T09:12:11.376 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.970046+0000", 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.1403999999999999e-05 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:08.970087+0000", 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027384736999999999 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.377 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:167311 9.7 e237/229 9:e9de203d:::benchmark_data_smithi016_172771_object70725:head v 237'10208, mlcod=237'10206)", 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:33.766634+0000", 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 97.532389326000001, 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.037623665000000001, 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.378 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766634+0000", 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766634+0000", 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766634+0000", 2023-10-09T09:12:11.379 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.5609999999999999e-06 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766637+0000", 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.036e-06 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766642+0000", 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.65e-06 2023-10-09T09:12:11.380 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766644+0000", 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.1136000000000001e-05 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766685+0000", 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.2802e-05 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.381 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.766708+0000", 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.037504793000000002 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.804213+0000", 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.5028000000000003e-05 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.382 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:33.804258+0000", 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.037623665000000001 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:167505 9.a e238/229 9:59d02ae0:::benchmark_data_smithi016_172771_object70919:head v 238'10440, mlcod=237'10438)", 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.445620+0000", 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 96.853403205999996, 2023-10-09T09:12:11.383 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.067116268000000007, 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445620+0000", 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.384 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445620+0000", 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445619+0000", 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2678e-05 2023-10-09T09:12:11.385 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445632+0000", 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1967000000000002e-05 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445654+0000", 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7029999999999995e-06 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.386 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445662+0000", 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3336999999999993e-05 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445726+0000", 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.8487999999999999e-05 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.387 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.445754+0000", 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.066888036999999997 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.512642+0000", 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.4338000000000004e-05 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.512737+0000", 2023-10-09T09:12:11.388 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.067116268000000007 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167510 9.13 9.93121913 (undecoded) ondisk+write+known_if_redirected e238)", 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.562501+0000", 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 96.736522686000001, 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.11602164, 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.389 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167510 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562501+0000", 2023-10-09T09:12:11.390 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562501+0000", 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562499+0000", 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.3100000000000006e-06 2023-10-09T09:12:11.391 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562508+0000", 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0609e-05 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562519+0000", 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4989999999999994e-06 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.392 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562527+0000", 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9993e-05 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562587+0000", 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8379999999999997e-06 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.393 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562597+0000", 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1159125089999999 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.678509+0000", 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3142000000000001e-05 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.394 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.678523+0000", 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.11602164 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:190684 9.11 e243/238 9:8aa0ccfb:::benchmark_data_smithi016_172771_object94098:head v 243'11327, mlcod=243'11325)", 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:11:18.963349+0000", 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 52.335674722999997, 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.030915483000000001, 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.395 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963349+0000", 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963349+0000", 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.396 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963348+0000", 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.3969999999999994e-06 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963356+0000", 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4626e-05 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.397 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963371+0000", 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6720000000000004e-06 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963379+0000", 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8296999999999999e-05 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.398 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963457+0000", 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0483e-05 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.963477+0000", 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.030763623 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.399 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.994241+0000", 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.3215000000000001e-05 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.994264+0000", 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.030915483000000001 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.400 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:190686 9.1 e243/238 9:8519e257:::benchmark_data_smithi016_172771_object94100:head v 243'11339, mlcod=243'11337)", 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:11:18.964600+0000", 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 52.334423786000002, 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.029636453, 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.964600+0000", 2023-10-09T09:12:11.401 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.964600+0000", 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.964599+0000", 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0644e-05 2023-10-09T09:12:11.402 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.964609+0000", 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1946e-05 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.964631+0000", 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0186999999999999e-05 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.403 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.964642+0000", 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0017564589999999999 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.966398+0000", 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.2195999999999999e-05 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.404 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.966420+0000", 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027769760000000001 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.994190+0000", 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.6249999999999999e-05 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:18.994236+0000", 2023-10-09T09:12:11.405 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.029636453 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:190849 9.a e243/229 9:5a3f1c4c:::benchmark_data_smithi016_172771_object94263:head v 243'11932, mlcod=243'11929)", 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:11:19.210526+0000", 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 52.088498063000003, 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.032041826000000002, 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.406 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210526+0000", 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210526+0000", 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.407 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210525+0000", 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6029999999999999e-06 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210532+0000", 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0710999999999999e-05 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.408 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210553+0000", 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.9160000000000007e-06 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210562+0000", 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.4878999999999999e-05 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210637+0000", 2023-10-09T09:12:11.409 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7257000000000001e-05 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.210654+0000", 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.031876212000000001 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.242530+0000", 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.7110999999999998e-05 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.410 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:19.242567+0000", 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.032041826000000002 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_repop(client.45290.0:191556 9.a e243/229 9:5f6987c9:::benchmark_data_smithi016_172771_object94970:head v 243'11985, mlcod=243'11982)", 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:11:20.364517+0000", 2023-10-09T09:12:11.411 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 50.934506802000001, 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.028090045000000001, 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364517+0000", 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:12:11.412 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364517+0000", 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364516+0000", 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1491000000000001e-05 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364527+0000", 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3322000000000001e-05 2023-10-09T09:12:11.413 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364541+0000", 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9600000000000003e-06 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364547+0000", 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7410000000000004e-05 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.414 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364645+0000", 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1413999999999999e-05 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.364666+0000", 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.027918681000000001 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.415 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.392585+0000", 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1821e-05 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:11:20.392607+0000", 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.028090045000000001 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.416 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.417 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.417 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:12:11.417 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:12:11.417 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:12:11.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:11.418 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:11 smithi094 bash[59677]: cluster 2023-10-09T09:12:10.044627+0000 mgr.y (mgr.25421) 1045 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:11.418 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:11 smithi094 bash[59677]: audit 2023-10-09T09:12:10.094302+0000 mon.a (mon.0) 1413 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:11.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:11.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:11.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:11.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:11.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:11.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:11.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:11.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:11.709 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:11.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:11.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:11.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:11.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:11.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:11.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:11.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:12.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:12.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:12.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:12.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:12.230 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:12.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:12.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:12:12.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:12.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:12.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:12.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:12.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:12.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:12.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:12.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:12.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:12.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:12.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:12.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:12.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:12.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:12.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:12.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:12.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.004 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:13.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:13.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:13.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:13.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:13.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:13.382 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:13 smithi094 bash[59677]: cluster 2023-10-09T09:12:12.045706+0000 mgr.y (mgr.25421) 1046 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:13.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:13.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:13.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:13 smithi016 bash[75179]: cluster 2023-10-09T09:12:12.045706+0000 mgr.y (mgr.25421) 1046 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:13.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:13 smithi016 bash[77892]: cluster 2023-10-09T09:12:12.045706+0000 mgr.y (mgr.25421) 1046 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:13.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:13.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:13.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:13.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:13.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:13.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:13.896 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:13.929 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:13.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:13.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:14.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:14.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:14.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:14.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:14.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:14.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:14.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:14.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:14.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:14.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:14.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:14.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:14.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:14.725 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:14.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:14.859 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:14.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:14.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:14.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:14.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:14.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:15.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:15.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:15.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:15.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:15.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:15.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:15 smithi016 bash[75179]: cluster 2023-10-09T09:12:14.046613+0000 mgr.y (mgr.25421) 1047 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:15.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:15 smithi016 bash[77892]: cluster 2023-10-09T09:12:14.046613+0000 mgr.y (mgr.25421) 1047 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:15.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:15.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:15.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:15.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:15.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:15.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:15.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:15 smithi094 bash[59677]: cluster 2023-10-09T09:12:14.046613+0000 mgr.y (mgr.25421) 1047 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:15.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:15.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:15.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:15.655 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:15.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:15.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:15.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:15.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:15.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:15.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:15.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:12:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:12:15] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.33.4" 2023-10-09T09:12:15.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:15.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:16.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:16.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:16.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:16.114 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T09:12:16.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:16.152 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:16 smithi016 bash[75179]: audit 2023-10-09T09:12:16.067698+0000 mon.a (mon.0) 1414 : audit 1 from='client.? 172.21.15.16:0/2199181756' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:12:16.152 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:16 smithi016 bash[77892]: audit 2023-10-09T09:12:16.067698+0000 mon.a (mon.0) 1414 : audit 1 from='client.? 172.21.15.16:0/2199181756' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:12:16.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:16.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:16.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:16.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:16.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:16.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:16 smithi094 bash[59677]: audit 2023-10-09T09:12:16.067698+0000 mon.a (mon.0) 1414 : audit 1 from='client.? 172.21.15.16:0/2199181756' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:12:16.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:16.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:16.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:16.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:16.701 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:16.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:16.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:16.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:16.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:16.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:16.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:16.965 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:12:16.988 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:12:17.007 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:12:17.028 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:12:17.051 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:12:17.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:17.071 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:12:17.093 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:12:17.098 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:17.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:17.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:17.115 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:12:17.117 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.117 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.117 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.117 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.117 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.117 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.118 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.118 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:12:17.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:17.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:17.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:17 smithi094 bash[59677]: cluster 2023-10-09T09:12:16.047225+0000 mgr.y (mgr.25421) 1048 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:17 smithi094 bash[59677]: cluster 2023-10-09T09:12:16.107688+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:17 smithi094 bash[59677]: audit 2023-10-09T09:12:16.110087+0000 mon.a (mon.0) 1415 : audit 1 from='client.? 172.21.15.16:0/2199181756' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:12:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:17 smithi094 bash[59677]: cluster 2023-10-09T09:12:16.110144+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T09:12:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:17 smithi094 bash[59677]: cluster 2023-10-09T09:12:16.289623+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:17 smithi094 bash[59677]: cluster 2023-10-09T09:12:16.831570+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:17 smithi094 bash[59677]: audit 2023-10-09T09:12:16.943964+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.16:0/2074043291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:12:17.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:17.360 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:17.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:17.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:17.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:17.475 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[75179]: cluster 2023-10-09T09:12:16.047225+0000 mgr.y (mgr.25421) 1048 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[75179]: cluster 2023-10-09T09:12:16.107688+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[75179]: audit 2023-10-09T09:12:16.110087+0000 mon.a (mon.0) 1415 : audit 1 from='client.? 172.21.15.16:0/2199181756' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[75179]: cluster 2023-10-09T09:12:16.110144+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[75179]: cluster 2023-10-09T09:12:16.289623+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[75179]: cluster 2023-10-09T09:12:16.831570+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[75179]: audit 2023-10-09T09:12:16.943964+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.16:0/2074043291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[77892]: cluster 2023-10-09T09:12:16.047225+0000 mgr.y (mgr.25421) 1048 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[77892]: cluster 2023-10-09T09:12:16.107688+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[77892]: audit 2023-10-09T09:12:16.110087+0000 mon.a (mon.0) 1415 : audit 1 from='client.? 172.21.15.16:0/2199181756' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:12:17.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[77892]: cluster 2023-10-09T09:12:16.110144+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T09:12:17.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[77892]: cluster 2023-10-09T09:12:16.289623+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[77892]: cluster 2023-10-09T09:12:16.831570+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:17.477 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:17 smithi016 bash[77892]: audit 2023-10-09T09:12:16.943964+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.16:0/2074043291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:12:17.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:17.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:17.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:17.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:17.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:17.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:17.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:17.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:17.860 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:17.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:17.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:17.990 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:18.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:18.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:18.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:18.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:18 smithi094 bash[59677]: cluster 2023-10-09T09:12:16.462899+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:18.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:18.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:18.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:18.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:18.397 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:18 smithi016 bash[75179]: cluster 2023-10-09T09:12:16.462899+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:18.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:18 smithi016 bash[77892]: cluster 2023-10-09T09:12:16.462899+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T09:12:18.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:18.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:18.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:18.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:18.650 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:18.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:18.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:18.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:18.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:18.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:18.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:18.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:18.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:19.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:19.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:19.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:19.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:19.210 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:19 smithi016 bash[75179]: cluster 2023-10-09T09:12:18.047942+0000 mgr.y (mgr.25421) 1049 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:19.210 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:19 smithi016 bash[77892]: cluster 2023-10-09T09:12:18.047942+0000 mgr.y (mgr.25421) 1049 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:19.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:19.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:19.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:19.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:19.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:19 smithi094 bash[59677]: cluster 2023-10-09T09:12:18.047942+0000 mgr.y (mgr.25421) 1049 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:19.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:19.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:19.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:19.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:19.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:19.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:19.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:19.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:19.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:19.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:19.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:20.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:20.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:20.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:20.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:20.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:20.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:20.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:20.264 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:20.361 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:20.374 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:20 smithi094 bash[59677]: audit 2023-10-09T09:12:19.245984+0000 mon.a (mon.0) 1417 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:20.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:20.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:20 smithi016 bash[75179]: audit 2023-10-09T09:12:19.245984+0000 mon.a (mon.0) 1417 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:20.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:20 smithi016 bash[77892]: audit 2023-10-09T09:12:19.245984+0000 mon.a (mon.0) 1417 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:20.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:20.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:20.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:20.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:20.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:20.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:20.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:20.756 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T09:12:20.757 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T09:12:20.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:12:20.759 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:12:20.759 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:12:20.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:20.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:20.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:20.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:20.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:20.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:20.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:20.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:20.983 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-10-09T09:12:20.983 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:12:20.983 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_193276 2023-10-09T09:12:20.984 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:12:20.984 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:12:21.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:21.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:21.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:21.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:21.140 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:12:21.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:21.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:21.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:21.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:21 smithi094 bash[59677]: cluster 2023-10-09T09:12:20.048729+0000 mgr.y (mgr.25421) 1050 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:21.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:21 smithi094 bash[59677]: audit 2023-10-09T09:12:20.374541+0000 mon.b (mon.1) 23 : audit 1 from='client.? 172.21.15.16:0/3751852781' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:12:21.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:21 smithi094 bash[59677]: audit 2023-10-09T09:12:20.375570+0000 mon.a (mon.0) 1418 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:12:21.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:21.349 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:21 smithi016 bash[75179]: cluster 2023-10-09T09:12:20.048729+0000 mgr.y (mgr.25421) 1050 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:21.349 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:21 smithi016 bash[75179]: audit 2023-10-09T09:12:20.374541+0000 mon.b (mon.1) 23 : audit 1 from='client.? 172.21.15.16:0/3751852781' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:12:21.350 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:21 smithi016 bash[75179]: audit 2023-10-09T09:12:20.375570+0000 mon.a (mon.0) 1418 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:12:21.350 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:21 smithi016 bash[77892]: cluster 2023-10-09T09:12:20.048729+0000 mgr.y (mgr.25421) 1050 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:21.350 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:21 smithi016 bash[77892]: audit 2023-10-09T09:12:20.374541+0000 mon.b (mon.1) 23 : audit 1 from='client.? 172.21.15.16:0/3751852781' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:12:21.350 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:21 smithi016 bash[77892]: audit 2023-10-09T09:12:20.375570+0000 mon.a (mon.0) 1418 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:12:21.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:21.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:21.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:21.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:21.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:21.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:21.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:21.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:21.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:21.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:21.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:21.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:21.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:12:21.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:21.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:21.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:21.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:21.984 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2711 2695 168.428 168.438 0.0437249 0.00591833 2023-10-09T09:12:22.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:22.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:22.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:22.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[75179]: cluster 2023-10-09T09:12:21.120771+0000 mon.a (mon.0) 1419 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:12:22.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[75179]: cluster 2023-10-09T09:12:21.120833+0000 mon.a (mon.0) 1420 : cluster 1 Cluster is now healthy 2023-10-09T09:12:22.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[75179]: cluster 2023-10-09T09:12:21.129441+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T09:12:22.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[75179]: audit 2023-10-09T09:12:21.134057+0000 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:12:22.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[75179]: cluster 2023-10-09T09:12:21.134355+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-10-09T09:12:22.146 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[75179]: cluster 2023-10-09T09:12:21.144847+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T09:12:22.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[77892]: cluster 2023-10-09T09:12:21.120771+0000 mon.a (mon.0) 1419 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:12:22.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[77892]: cluster 2023-10-09T09:12:21.120833+0000 mon.a (mon.0) 1420 : cluster 1 Cluster is now healthy 2023-10-09T09:12:22.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[77892]: cluster 2023-10-09T09:12:21.129441+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T09:12:22.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[77892]: audit 2023-10-09T09:12:21.134057+0000 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:12:22.147 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[77892]: cluster 2023-10-09T09:12:21.134355+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-10-09T09:12:22.148 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:22 smithi016 bash[77892]: cluster 2023-10-09T09:12:21.144847+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T09:12:22.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:22.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:22.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:22.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:22.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:22.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:22 smithi094 bash[59677]: cluster 2023-10-09T09:12:21.120771+0000 mon.a (mon.0) 1419 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:12:22.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:22 smithi094 bash[59677]: cluster 2023-10-09T09:12:21.120833+0000 mon.a (mon.0) 1420 : cluster 1 Cluster is now healthy 2023-10-09T09:12:22.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:22 smithi094 bash[59677]: cluster 2023-10-09T09:12:21.129441+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T09:12:22.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:22 smithi094 bash[59677]: audit 2023-10-09T09:12:21.134057+0000 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:12:22.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:22 smithi094 bash[59677]: cluster 2023-10-09T09:12:21.134355+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-10-09T09:12:22.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:22 smithi094 bash[59677]: cluster 2023-10-09T09:12:21.144847+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T09:12:22.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:22.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:22.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:22.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:22.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:22.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:22.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:22.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:22.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:22.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:22.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5070 5054 157.925 147.438 0.00831109 0.00631854 2023-10-09T09:12:23.068 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:23.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:23.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:23.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:23.209 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:23 smithi094 bash[59677]: cluster 2023-10-09T09:12:22.049147+0000 mgr.y (mgr.25421) 1051 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:23.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:23.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:23.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:23.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:23.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:23.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:23.493 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:23 smithi016 bash[75179]: cluster 2023-10-09T09:12:22.049147+0000 mgr.y (mgr.25421) 1051 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:23.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:23 smithi016 bash[77892]: cluster 2023-10-09T09:12:22.049147+0000 mgr.y (mgr.25421) 1051 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T09:12:23.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:23.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:23.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:23.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:23.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:23.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:23.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:23.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:23.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:23.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:23.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:23.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:23.984 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7509 7493 156.09 152.438 0.00459922 0.00639641 2023-10-09T09:12:24.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:24.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:24.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:24.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:24.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:24.306 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:24.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:24.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:24.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:24.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:24.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:24.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:24.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:24.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:24.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:24.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:24.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:24.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:24.984 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9203 9187 143.533 105.875 0.00427683 0.00695677 2023-10-09T09:12:25.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:25.104 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:25.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:25.136 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:25 smithi016 bash[75179]: cluster 2023-10-09T09:12:24.049785+0000 mgr.y (mgr.25421) 1052 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 73 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 9.1 MiB/s wr, 145 op/s 2023-10-09T09:12:25.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:25.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:25.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:25.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:25.344 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:25.353 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:25 smithi094 bash[59677]: cluster 2023-10-09T09:12:24.049785+0000 mgr.y (mgr.25421) 1052 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 73 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 9.1 MiB/s wr, 145 op/s 2023-10-09T09:12:25.373 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:25.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:25.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:25.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:25.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:25 smithi016 bash[77892]: cluster 2023-10-09T09:12:24.049785+0000 mgr.y (mgr.25421) 1052 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 73 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 9.1 MiB/s wr, 145 op/s 2023-10-09T09:12:25.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:25.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:25.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:25.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:25.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:12:25.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:25.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:25.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:25.740 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:12:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:12:25] "GET /metrics HTTP/1.1" 200 36378 "" "Prometheus/2.33.4" 2023-10-09T09:12:25.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:25.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:25.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:25.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:25.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:25.913 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:25.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:25.984 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 11518 11502 143.761 144.688 0.0075392 0.00694818 2023-10-09T09:12:26.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:26.048 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:26.128 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:26.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:26.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:26.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:26.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:26.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:26.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:26.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:26.542 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:26.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:26.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:26.702 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:26.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:26.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:26.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:26.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:26.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:26.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:26.984 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 13432 13416 139.736 119.625 0.00837789 0.00714823 2023-10-09T09:12:27.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:27.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:27.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:27.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:27.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:27.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:27.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:27.336 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:27.345 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:27 smithi094 bash[59677]: cluster 2023-10-09T09:12:26.050637+0000 mgr.y (mgr.25421) 1053 : cluster 0 pgmap v1129: 89 pgs: 89 active+clean; 564 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T09:12:27.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:27.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:27.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:27.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:27.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:27 smithi016 bash[75179]: cluster 2023-10-09T09:12:26.050637+0000 mgr.y (mgr.25421) 1053 : cluster 0 pgmap v1129: 89 pgs: 89 active+clean; 564 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T09:12:27.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:27 smithi016 bash[77892]: cluster 2023-10-09T09:12:26.050637+0000 mgr.y (mgr.25421) 1053 : cluster 0 pgmap v1129: 89 pgs: 89 active+clean; 564 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T09:12:27.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:27.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:27.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:27.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:27.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:27.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:27.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:27.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:27.868 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:27.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:27.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:27.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 15523 15507 138.442 130.688 0.00743432 0.00721445 2023-10-09T09:12:28.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:28.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:28.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:28.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:28.164 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:28.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:28.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:28.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:28.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:28.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:28.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:28.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:28.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:28.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:28.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:28.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:28.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:28.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:28.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:28.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:28.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:28.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:28.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 16959 16943 132.354 89.75 0.0119425 0.00754853 2023-10-09T09:12:29.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:29.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:29.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:29.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:29.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:29.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:29.326 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:29.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:29.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:29.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:29.431 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:29 smithi016 bash[75179]: cluster 2023-10-09T09:12:28.051032+0000 mgr.y (mgr.25421) 1054 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 564 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T09:12:29.432 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:29 smithi016 bash[77892]: cluster 2023-10-09T09:12:28.051032+0000 mgr.y (mgr.25421) 1054 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 564 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T09:12:29.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:29.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:29 smithi094 bash[59677]: cluster 2023-10-09T09:12:28.051032+0000 mgr.y (mgr.25421) 1054 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 564 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T09:12:29.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:29.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:29.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:29.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:29.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:29.682 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:12:29.683 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:12:29.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:29.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:29.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:29.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:29.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:29.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:29.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 18884 18868 131.015 120.312 0.00536735 0.00762454 2023-10-09T09:12:29.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:30.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:30.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:30.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:30.117 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:30.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:30.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:30.207 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:30.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:30.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:30.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:30.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:30.412 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:30.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:30.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:30.699 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:30 smithi094 bash[59677]: cluster 2023-10-09T09:12:30.051586+0000 mgr.y (mgr.25421) 1055 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 681 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:12:30.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:30.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:30.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:30.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:30.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:30.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:30.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:30.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:30.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 20721 20705 129.394 114.812 0.00623362 0.00772175 2023-10-09T09:12:30.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:30 smithi016 bash[75179]: cluster 2023-10-09T09:12:30.051586+0000 mgr.y (mgr.25421) 1055 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 681 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:12:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:30 smithi016 bash[77892]: cluster 2023-10-09T09:12:30.051586+0000 mgr.y (mgr.25421) 1055 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 681 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:12:31.001 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:31.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:31.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:31.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:31.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:31.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:31.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:31.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:31.489 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:31.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:31.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:31.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:31.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:31.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:31.770 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:31.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:31.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:31.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:31.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:31.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 22589 22573 128.243 116.75 0.00603897 0.0077917 2023-10-09T09:12:32.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:32.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:32.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:32.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:32.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:32.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:32.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:32.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:32.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:32.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:32.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:32.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:32.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:32.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:32.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:32.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:32.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 24378 24362 126.873 111.812 0.00324702 0.00787475 2023-10-09T09:12:33.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:33.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:33.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:33.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:33.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:33.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:33.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:33.357 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:33 smithi016 bash[75179]: cluster 2023-10-09T09:12:32.052374+0000 mgr.y (mgr.25421) 1056 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T09:12:33.357 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:33 smithi016 bash[77892]: cluster 2023-10-09T09:12:32.052374+0000 mgr.y (mgr.25421) 1056 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T09:12:33.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:33.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:33.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:33.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:33 smithi094 bash[59677]: cluster 2023-10-09T09:12:32.052374+0000 mgr.y (mgr.25421) 1056 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T09:12:33.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:33.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:33.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:33.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:33.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:33.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:33.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:33.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:33.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:33.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:33.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:33.903 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:33.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:33.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 26067 26051 125.233 105.562 0.00418529 0.00797781 2023-10-09T09:12:34.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:34.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:34.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:34.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:34.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:34.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:34.286 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:34.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:34.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:34.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:34.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:34.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:34.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:34.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:34.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:34.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:34.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:34.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:34.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:34.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:34.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 27598 27582 123.122 95.6875 0.0106236 0.00811514 2023-10-09T09:12:35.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:35.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:35.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:35.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:35.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:35.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:35.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:35.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:35.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:35.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:35.461 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:35 smithi094 bash[59677]: cluster 2023-10-09T09:12:34.053101+0000 mgr.y (mgr.25421) 1057 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:12:35.461 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:35 smithi094 bash[59677]: audit 2023-10-09T09:12:34.251860+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:35.461 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:35 smithi094 bash[59677]: audit 2023-10-09T09:12:34.253609+0000 mon.a (mon.0) 1424 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:35.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:35.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:35.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:35 smithi016 bash[75179]: cluster 2023-10-09T09:12:34.053101+0000 mgr.y (mgr.25421) 1057 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:12:35.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:35 smithi016 bash[75179]: audit 2023-10-09T09:12:34.251860+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:35.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:35 smithi016 bash[75179]: audit 2023-10-09T09:12:34.253609+0000 mon.a (mon.0) 1424 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:35.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:35 smithi016 bash[77892]: cluster 2023-10-09T09:12:34.053101+0000 mgr.y (mgr.25421) 1057 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:12:35.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:35 smithi016 bash[77892]: audit 2023-10-09T09:12:34.251860+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:12:35.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:35 smithi016 bash[77892]: audit 2023-10-09T09:12:34.253609+0000 mon.a (mon.0) 1424 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:35.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:35.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:35.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:35.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:35.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:35.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:35.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:35.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:35.856 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:12:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:12:35] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.33.4" 2023-10-09T09:12:35.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:35.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:35.977 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:35.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 29116 29100 121.239 94.875 0.0083009 0.00824107 2023-10-09T09:12:36.005 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:36.109 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:36.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:36.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:36.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:12:36.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:36.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:36.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:36.379 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:36.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:36.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:36.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:36.541 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:36.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:36.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:36.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:36.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:36.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:36.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:36.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:36.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:36.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 30750 30734 120.043 102.125 0.00514001 0.00832414 2023-10-09T09:12:37.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:37.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:37.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:37.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:37.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:37.207 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:37.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:37.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:37.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:37.322 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:37 smithi016 bash[75179]: cluster 2023-10-09T09:12:36.053890+0000 mgr.y (mgr.25421) 1058 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 137 MiB/s wr, 2.18k op/s 2023-10-09T09:12:37.322 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:37 smithi016 bash[77892]: cluster 2023-10-09T09:12:36.053890+0000 mgr.y (mgr.25421) 1058 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 137 MiB/s wr, 2.18k op/s 2023-10-09T09:12:37.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:37.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:37.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:37.451 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:37 smithi094 bash[59677]: cluster 2023-10-09T09:12:36.053890+0000 mgr.y (mgr.25421) 1058 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 137 MiB/s wr, 2.18k op/s 2023-10-09T09:12:37.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:37.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:37.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:37.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:37.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:37.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:37.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:37.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:37.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:37.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:37.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:37.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:37.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:37.985 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 32471 32455 119.309 107.562 0.00818045 0.00837511 2023-10-09T09:12:37.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:38.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:38.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:38.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:38.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:38.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:38.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:38.406 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:38.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:38.540 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:38.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:38.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:38.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:38.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:38.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:38.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:38.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:38.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:38.938 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:38.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:38.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:38.986 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 34113 34097 118.381 102.625 0.0118421 0.00844168 2023-10-09T09:12:39.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:39.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:39.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:39.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:39.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:39.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:39.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:39.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:39.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:39.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:39.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:39.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:39.452 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:39 smithi016 bash[75179]: cluster 2023-10-09T09:12:38.054365+0000 mgr.y (mgr.25421) 1059 : cluster 0 pgmap v1135: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T09:12:39.453 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:39 smithi016 bash[77892]: cluster 2023-10-09T09:12:38.054365+0000 mgr.y (mgr.25421) 1059 : cluster 0 pgmap v1135: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T09:12:39.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:39.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:39.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:39.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:39.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:39.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:39.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:39.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:39.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:39 smithi094 bash[59677]: cluster 2023-10-09T09:12:38.054365+0000 mgr.y (mgr.25421) 1059 : cluster 0 pgmap v1135: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T09:12:39.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:39.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:39.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:39.972 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:39.986 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 35457 35441 116.571 84 0.00681245 0.00855104 2023-10-09T09:12:40.004 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:40.108 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:40.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:40.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:40.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:40.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:40.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:40.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:40.548 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:40.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:40.681 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:40.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:40.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:40 smithi016 bash[77892]: cluster 2023-10-09T09:12:40.055107+0000 mgr.y (mgr.25421) 1060 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T09:12:40.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:40.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:40.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:40.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:40.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:40.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:40.986 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:12:40.986005+0000 min lat: 0.00132275 max lat: 0.323328 avg lat: 0.00866012 2023-10-09T09:12:40.986 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:12:40.986 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 36942 36926 115.383 92.8125 0.0101918 0.00866012 2023-10-09T09:12:40.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:40 smithi094 bash[59677]: cluster 2023-10-09T09:12:40.055107+0000 mgr.y (mgr.25421) 1060 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T09:12:41.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:41.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:40 smithi016 bash[75179]: cluster 2023-10-09T09:12:40.055107+0000 mgr.y (mgr.25421) 1060 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T09:12:41.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:41.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:41.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:41.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:41.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:41.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:41.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:41.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:41.358 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:41.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:41.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:41.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:41.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:41.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:41.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:41.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:41.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:41.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:41.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:41.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:41.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:41.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:41.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:41.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:41.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:41.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:41.986 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 38385 38369 114.182 90.1875 0.0172539 0.0087519 2023-10-09T09:12:41.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:42.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:42.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:42.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:42.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:42.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:42.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:42.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:42.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:42.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:42.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:42.522 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:42.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:42.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:42.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:42.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:42.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:42.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:42.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:42.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:42.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:42.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:42.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:42.986 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 39838 39822 113.12 90.8125 0.00849467 0.00883465 2023-10-09T09:12:43.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:43.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:43.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:43.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:43.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:43.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:43.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:43.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:43 smithi016 bash[75179]: cluster 2023-10-09T09:12:42.055825+0000 mgr.y (mgr.25421) 1061 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T09:12:43.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:43 smithi016 bash[77892]: cluster 2023-10-09T09:12:42.055825+0000 mgr.y (mgr.25421) 1061 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T09:12:43.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:43.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:43.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:43.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:43.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:43.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:43.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:43.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:43.492 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:12:43.494 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:43 smithi094 bash[59677]: cluster 2023-10-09T09:12:42.055825+0000 mgr.y (mgr.25421) 1061 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T09:12:43.500 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:12:43.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:43.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:43.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:43.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:43.734 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:43.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:43.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:43.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:43.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:43.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:43.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 41274 41258 112.09 89.75 0.0170102 0.00891247 2023-10-09T09:12:43.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:43.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:44.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:44.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:44.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:44.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:44 smithi016 bash[75179]: audit 2023-10-09T09:12:43.470836+0000 mgr.y (mgr.25421) 1062 : audit 0 from='client.55210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:12:44.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:44 smithi016 bash[77892]: audit 2023-10-09T09:12:43.470836+0000 mgr.y (mgr.25421) 1062 : audit 0 from='client.55210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:12:44.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:44.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:44.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:44.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:44 smithi094 bash[59677]: audit 2023-10-09T09:12:43.470836+0000 mgr.y (mgr.25421) 1062 : audit 0 from='client.55210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:12:44.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:44.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:44.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:44.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:44.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:44.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:44.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:44.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:44.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:44.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:44.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:44.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 42304 42288 110.102 64.375 0.00261161 0.00904009 2023-10-09T09:12:45.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:45.053 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:45.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:45.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:45.286 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:45.295 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:45 smithi094 bash[59677]: cluster 2023-10-09T09:12:44.056421+0000 mgr.y (mgr.25421) 1063 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T09:12:45.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:45.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:45.416 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:45.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:45.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:45 smithi016 bash[75179]: cluster 2023-10-09T09:12:44.056421+0000 mgr.y (mgr.25421) 1063 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T09:12:45.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:45 smithi016 bash[77892]: cluster 2023-10-09T09:12:44.056421+0000 mgr.y (mgr.25421) 1063 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T09:12:45.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:45.584 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:45.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:45.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:45.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:45.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:45.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:45.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:45.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:45.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:45.970 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:12:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:12:45] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.33.4" 2023-10-09T09:12:45.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:45.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:45.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 43564 43548 108.848 78.75 0.0116851 0.00918009 2023-10-09T09:12:46.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:46.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:46.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:46.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:46.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:46.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:46.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:46.264 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:12:46.295 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:12:46.336 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:12:46.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:46.376 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:12:46.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:46.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:46.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:46.407 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:12:46.442 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:12:46.481 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:12:46.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:46.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:46.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:46.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:46.519 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:12:46.522 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.522 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.522 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.522 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.522 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.522 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.522 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.523 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:12:46.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:46.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:46.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:46.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:46.788 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:46.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:46.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:46.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:46.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:46.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:46.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 44962 44946 108.022 87.375 0.00612086 0.00925108 2023-10-09T09:12:47.024 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:47.053 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:47.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:47.157 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1137,"stamp":"2023-10-09T09:12:42.055300+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2305103776,"num_objects":35188,"num_object_clones":0,"num_object_copies":105564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35188,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":35424,"num_write_kb":2252941,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":2970226,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35358,"ondisk_log_size":35358,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12117492,"kb_used_data":6964668,"kb_used_omap":117,"kb_used_meta":5152650,"kb_avail":644098572,"statfs":{"total":671965249536,"available":659556937728,"internally_reserved":0,"allocated":7131820032,"data_stored":7126585604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":5276314258},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1590689792,"num_objects":24272,"num_object_clones":0,"num_object_copies":72816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":24272,"num_write_kb":1553408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004100"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":114,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112167+0000","last_change":"2023-10-09T09:02:02.645995+0000","last_active":"2023-10-09T09:12:16.112167+0000","last_peered":"2023-10-09T09:12:16.112167+0000","last_clean":"2023-10-09T09:12:16.112167+0000","last_became_active":"2023-10-09T09:02:02.645285+0000","last_became_peered":"2023-10-09T09:02:02.645285+0000","last_unstale":"2023-10-09T09:12:16.112167+0000","last_undegraded":"2023-10-09T09:12:16.112167+0000","last_fullsized":"2023-10-09T09:12:16.112167+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112544+0000","last_change":"2023-10-09T09:09:06.663045+0000","last_active":"2023-10-09T09:12:16.112544+0000","last_peered":"2023-10-09T09:12:16.112544+0000","last_clean":"2023-10-09T09:12:16.112544+0000","last_became_active":"2023-10-09T09:09:06.662681+0000","last_became_peered":"2023-10-09T09:09:06.662681+0000","last_unstale":"2023-10-09T09:12:16.112544+0000","last_undegraded":"2023-10-09T09:12:16.112544+0000","last_fullsized":"2023-10-09T09:12:16.112544+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":173,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.111945+0000","last_change":"2023-10-09T09:09:06.662154+0000","last_active":"2023-10-09T09:12:16.111945+0000","last_peered":"2023-10-09T09:12:16.111945+0000","last_clean":"2023-10-09T09:12:16.111945+0000","last_became_active":"2023-10-09T09:09:06.661824+0000","last_became_peered":"2023-10-09T09:09:06.661824+0000","last_unstale":"2023-10-09T09:12:16.111945+0000","last_undegraded":"2023-10-09T09:12:16.111945+0000","last_fullsized":"2023-10-09T09:12:16.111945+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":285,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264326+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:12:16.264326+0000","last_peered":"2023-10-09T09:12:16.264326+0000","last_clean":"2023-10-09T09:12:16.264326+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:12:16.264326+0000","last_undegraded":"2023-10-09T09:12:16.264326+0000","last_fullsized":"2023-10-09T09:12:16.264326+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":211,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264213+0000","last_change":"2023-10-09T09:09:06.657019+0000","last_active":"2023-10-09T09:12:16.264213+0000","last_peered":"2023-10-09T09:12:16.264213+0000","last_clean":"2023-10-09T09:12:16.264213+0000","last_became_active":"2023-10-09T09:09:06.656605+0000","last_became_peered":"2023-10-09T09:09:06.656605+0000","last_unstale":"2023-10-09T09:12:16.264213+0000","last_undegraded":"2023-10-09T09:12:16.264213+0000","last_fullsized":"2023-10-09T09:12:16.264213+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:08.279835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":137,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.836787+0000","last_change":"2023-10-09T09:08:24.652386+0000","last_active":"2023-10-09T09:12:16.836787+0000","last_peered":"2023-10-09T09:12:16.836787+0000","last_clean":"2023-10-09T09:12:16.836787+0000","last_became_active":"2023-10-09T09:08:24.651982+0000","last_became_peered":"2023-10-09T09:08:24.651982+0000","last_unstale":"2023-10-09T09:12:16.836787+0000","last_undegraded":"2023-10-09T09:12:16.836787+0000","last_fullsized":"2023-10-09T09:12:16.836787+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":238,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:21.148892+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:12:21.148892+0000","last_peered":"2023-10-09T09:12:21.148892+0000","last_clean":"2023-10-09T09:12:21.148892+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:12:21.148892+0000","last_undegraded":"2023-10-09T09:12:21.148892+0000","last_fullsized":"2023-10-09T09:12:21.148892+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":221,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.837133+0000","last_change":"2023-10-09T09:09:06.662705+0000","last_active":"2023-10-09T09:12:16.837133+0000","last_peered":"2023-10-09T09:12:16.837133+0000","last_clean":"2023-10-09T09:12:16.837133+0000","last_became_active":"2023-10-09T09:09:06.662304+0000","last_became_peered":"2023-10-09T09:09:06.662304+0000","last_unstale":"2023-10-09T09:12:16.837133+0000","last_undegraded":"2023-10-09T09:12:16.837133+0000","last_fullsized":"2023-10-09T09:12:16.837133+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":48,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.466579+0000","last_change":"2023-10-09T09:09:44.365998+0000","last_active":"2023-10-09T09:12:16.466579+0000","last_peered":"2023-10-09T09:12:16.466579+0000","last_clean":"2023-10-09T09:12:16.466579+0000","last_became_active":"2023-10-09T09:09:44.365450+0000","last_became_peered":"2023-10-09T09:09:44.365450+0000","last_unstale":"2023-10-09T09:12:16.466579+0000","last_undegraded":"2023-10-09T09:12:16.466579+0000","last_fullsized":"2023-10-09T09:12:16.466579+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.294719+0000","last_change":"2023-10-09T09:09:44.364273+0000","last_active":"2023-10-09T09:12:16.294719+0000","last_peered":"2023-10-09T09:12:16.294719+0000","last_clean":"2023-10-09T09:12:16.294719+0000","last_became_active":"2023-10-09T09:09:44.363921+0000","last_became_peered":"2023-10-09T09:09:44.363921+0000","last_unstale":"2023-10-09T09:12:16.294719+0000","last_undegraded":"2023-10-09T09:12:16.294719+0000","last_fullsized":"2023-10-09T09:12:16.294719+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":38,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.294965+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:12:16.294965+0000","last_peered":"2023-10-09T09:12:16.294965+0000","last_clean":"2023-10-09T09:12:16.294965+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:12:16.294965+0000","last_undegraded":"2023-10-09T09:12:16.294965+0000","last_fullsized":"2023-10-09T09:12:16.294965+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":211,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.837288+0000","last_change":"2023-10-09T09:02:02.637290+0000","last_active":"2023-10-09T09:12:16.837288+0000","last_peered":"2023-10-09T09:12:16.837288+0000","last_clean":"2023-10-09T09:12:16.837288+0000","last_became_active":"2023-10-09T09:02:02.636517+0000","last_became_peered":"2023-10-09T09:02:02.636517+0000","last_unstale":"2023-10-09T09:12:16.837288+0000","last_undegraded":"2023-10-09T09:12:16.837288+0000","last_fullsized":"2023-10-09T09:12:16.837288+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":153,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112939+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:12:16.112939+0000","last_peered":"2023-10-09T09:12:16.112939+0000","last_clean":"2023-10-09T09:12:16.112939+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:12:16.112939+0000","last_undegraded":"2023-10-09T09:12:16.112939+0000","last_fullsized":"2023-10-09T09:12:16.112939+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":132,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112068+0000","last_change":"2023-10-09T09:08:24.651923+0000","last_active":"2023-10-09T09:12:16.112068+0000","last_peered":"2023-10-09T09:12:16.112068+0000","last_clean":"2023-10-09T09:12:16.112068+0000","last_became_active":"2023-10-09T09:08:24.650836+0000","last_became_peered":"2023-10-09T09:08:24.650836+0000","last_unstale":"2023-10-09T09:12:16.112068+0000","last_undegraded":"2023-10-09T09:12:16.112068+0000","last_fullsized":"2023-10-09T09:12:16.112068+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":38,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.294599+0000","last_change":"2023-10-09T09:09:44.587384+0000","last_active":"2023-10-09T09:12:16.294599+0000","last_peered":"2023-10-09T09:12:16.294599+0000","last_clean":"2023-10-09T09:12:16.294599+0000","last_became_active":"2023-10-09T09:09:44.355040+0000","last_became_peered":"2023-10-09T09:09:44.355040+0000","last_unstale":"2023-10-09T09:12:16.294599+0000","last_undegraded":"2023-10-09T09:12:16.294599+0000","last_fullsized":"2023-10-09T09:12:16.294599+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.295182+0000","last_change":"2023-10-09T09:09:44.365312+0000","last_active":"2023-10-09T09:12:16.295182+0000","last_peered":"2023-10-09T09:12:16.295182+0000","last_clean":"2023-10-09T09:12:16.295182+0000","last_became_active":"2023-10-09T09:09:44.364936+0000","last_became_peered":"2023-10-09T09:09:44.364936+0000","last_unstale":"2023-10-09T09:12:16.295182+0000","last_undegraded":"2023-10-09T09:12:16.295182+0000","last_fullsized":"2023-10-09T09:12:16.295182+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":265,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264062+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:12:16.264062+0000","last_peered":"2023-10-09T09:12:16.264062+0000","last_clean":"2023-10-09T09:12:16.264062+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:12:16.264062+0000","last_undegraded":"2023-10-09T09:12:16.264062+0000","last_fullsized":"2023-10-09T09:12:16.264062+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":49,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.466530+0000","last_change":"2023-10-09T09:09:44.363249+0000","last_active":"2023-10-09T09:12:16.466530+0000","last_peered":"2023-10-09T09:12:16.466530+0000","last_clean":"2023-10-09T09:12:16.466530+0000","last_became_active":"2023-10-09T09:09:44.362838+0000","last_became_peered":"2023-10-09T09:09:44.362838+0000","last_unstale":"2023-10-09T09:12:16.466530+0000","last_undegraded":"2023-10-09T09:12:16.466530+0000","last_fullsized":"2023-10-09T09:12:16.466530+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.294318+0000","last_change":"2023-10-09T09:09:44.352696+0000","last_active":"2023-10-09T09:12:16.294318+0000","last_peered":"2023-10-09T09:12:16.294318+0000","last_clean":"2023-10-09T09:12:16.294318+0000","last_became_active":"2023-10-09T09:09:44.351809+0000","last_became_peered":"2023-10-09T09:09:44.351809+0000","last_unstale":"2023-10-09T09:12:16.294318+0000","last_undegraded":"2023-10-09T09:12:16.294318+0000","last_fullsized":"2023-10-09T09:12:16.294318+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":238,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:21.149586+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:12:21.149586+0000","last_peered":"2023-10-09T09:12:21.149586+0000","last_clean":"2023-10-09T09:12:21.149586+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:12:21.149586+0000","last_undegraded":"2023-10-09T09:12:21.149586+0000","last_fullsized":"2023-10-09T09:12:21.149586+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":149,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.835984+0000","last_change":"2023-10-09T09:08:24.741108+0000","last_active":"2023-10-09T09:12:16.835984+0000","last_peered":"2023-10-09T09:12:16.835984+0000","last_clean":"2023-10-09T09:12:16.835984+0000","last_became_active":"2023-10-09T09:08:24.651865+0000","last_became_peered":"2023-10-09T09:08:24.651865+0000","last_unstale":"2023-10-09T09:12:16.835984+0000","last_undegraded":"2023-10-09T09:12:16.835984+0000","last_fullsized":"2023-10-09T09:12:16.835984+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.295609+0000","last_change":"2023-10-09T09:09:44.460257+0000","last_active":"2023-10-09T09:12:16.295609+0000","last_peered":"2023-10-09T09:12:16.295609+0000","last_clean":"2023-10-09T09:12:16.295609+0000","last_became_active":"2023-10-09T09:09:44.459713+0000","last_became_peered":"2023-10-09T09:09:44.459713+0000","last_unstale":"2023-10-09T09:12:16.295609+0000","last_undegraded":"2023-10-09T09:12:16.295609+0000","last_fullsized":"2023-10-09T09:12:16.295609+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"252'2221","reported_seq":2233,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.174820+0000","last_change":"2023-10-09T09:12:00.109899+0000","last_active":"2023-10-09T09:12:40.174820+0000","last_peered":"2023-10-09T09:12:40.174820+0000","last_clean":"2023-10-09T09:12:40.174820+0000","last_became_active":"2023-10-09T09:12:00.109613+0000","last_became_peered":"2023-10-09T09:12:00.109613+0000","last_unstale":"2023-10-09T09:12:40.174820+0000","last_undegraded":"2023-10-09T09:12:40.174820+0000","last_fullsized":"2023-10-09T09:12:40.174820+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2221,"log_dups_size":0,"ondisk_log_size":2221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145555456,"num_objects":2221,"num_object_clones":0,"num_object_copies":6663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2221,"num_write_kb":142144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":36,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.295499+0000","last_change":"2023-10-09T09:09:45.088513+0000","last_active":"2023-10-09T09:12:16.295499+0000","last_peered":"2023-10-09T09:12:16.295499+0000","last_clean":"2023-10-09T09:12:16.295499+0000","last_became_active":"2023-10-09T09:09:44.459170+0000","last_became_peered":"2023-10-09T09:09:44.459170+0000","last_unstale":"2023-10-09T09:12:16.295499+0000","last_undegraded":"2023-10-09T09:12:16.295499+0000","last_fullsized":"2023-10-09T09:12:16.295499+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.466474+0000","last_change":"2023-10-09T09:09:44.355469+0000","last_active":"2023-10-09T09:12:16.466474+0000","last_peered":"2023-10-09T09:12:16.466474+0000","last_clean":"2023-10-09T09:12:16.466474+0000","last_became_active":"2023-10-09T09:09:44.354880+0000","last_became_peered":"2023-10-09T09:09:44.354880+0000","last_unstale":"2023-10-09T09:12:16.466474+0000","last_undegraded":"2023-10-09T09:12:16.466474+0000","last_fullsized":"2023-10-09T09:12:16.466474+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"252'2209","reported_seq":2221,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.126438+0000","last_change":"2023-10-09T09:12:00.136662+0000","last_active":"2023-10-09T09:12:40.126438+0000","last_peered":"2023-10-09T09:12:40.126438+0000","last_clean":"2023-10-09T09:12:40.126438+0000","last_became_active":"2023-10-09T09:12:00.135779+0000","last_became_peered":"2023-10-09T09:12:00.135779+0000","last_unstale":"2023-10-09T09:12:40.126438+0000","last_undegraded":"2023-10-09T09:12:40.126438+0000","last_fullsized":"2023-10-09T09:12:40.126438+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2209,"log_dups_size":0,"ondisk_log_size":2209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144769024,"num_objects":2209,"num_object_clones":0,"num_object_copies":6627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2209,"num_write_kb":141376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":36,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.295572+0000","last_change":"2023-10-09T09:09:45.000846+0000","last_active":"2023-10-09T09:12:16.295572+0000","last_peered":"2023-10-09T09:12:16.295572+0000","last_clean":"2023-10-09T09:12:16.295572+0000","last_became_active":"2023-10-09T09:09:44.404839+0000","last_became_peered":"2023-10-09T09:09:44.404839+0000","last_unstale":"2023-10-09T09:12:16.295572+0000","last_undegraded":"2023-10-09T09:12:16.295572+0000","last_fullsized":"2023-10-09T09:12:16.295572+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":132,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112182+0000","last_change":"2023-10-09T09:03:19.557931+0000","last_active":"2023-10-09T09:12:16.112182+0000","last_peered":"2023-10-09T09:12:16.112182+0000","last_clean":"2023-10-09T09:12:16.112182+0000","last_became_active":"2023-10-09T09:03:19.557554+0000","last_became_peered":"2023-10-09T09:03:19.557554+0000","last_unstale":"2023-10-09T09:12:16.112182+0000","last_undegraded":"2023-10-09T09:12:16.112182+0000","last_fullsized":"2023-10-09T09:12:16.112182+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"252'2005","reported_seq":2018,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:38.353251+0000","last_change":"2023-10-09T09:12:00.136187+0000","last_active":"2023-10-09T09:12:38.353251+0000","last_peered":"2023-10-09T09:12:38.353251+0000","last_clean":"2023-10-09T09:12:38.353251+0000","last_became_active":"2023-10-09T09:12:00.135851+0000","last_became_peered":"2023-10-09T09:12:00.135851+0000","last_unstale":"2023-10-09T09:12:38.353251+0000","last_undegraded":"2023-10-09T09:12:38.353251+0000","last_fullsized":"2023-10-09T09:12:38.353251+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2005,"log_dups_size":0,"ondisk_log_size":2005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:25.402310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131399680,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2005,"num_write_kb":128320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":36,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.295230+0000","last_change":"2023-10-09T09:09:44.913374+0000","last_active":"2023-10-09T09:12:16.295230+0000","last_peered":"2023-10-09T09:12:16.295230+0000","last_clean":"2023-10-09T09:12:16.295230+0000","last_became_active":"2023-10-09T09:09:44.369058+0000","last_became_peered":"2023-10-09T09:09:44.369058+0000","last_unstale":"2023-10-09T09:12:16.295230+0000","last_undegraded":"2023-10-09T09:12:16.295230+0000","last_fullsized":"2023-10-09T09:12:16.295230+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":146,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.835975+0000","last_change":"2023-10-09T09:09:44.359930+0000","last_active":"2023-10-09T09:12:16.835975+0000","last_peered":"2023-10-09T09:12:16.835975+0000","last_clean":"2023-10-09T09:12:16.835975+0000","last_became_active":"2023-10-09T09:09:44.359475+0000","last_became_peered":"2023-10-09T09:09:44.359475+0000","last_unstale":"2023-10-09T09:12:16.835975+0000","last_undegraded":"2023-10-09T09:12:16.835975+0000","last_fullsized":"2023-10-09T09:12:16.835975+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"252'2122","reported_seq":2135,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:38.596476+0000","last_change":"2023-10-09T09:12:00.136051+0000","last_active":"2023-10-09T09:12:38.596476+0000","last_peered":"2023-10-09T09:12:38.596476+0000","last_clean":"2023-10-09T09:12:38.596476+0000","last_became_active":"2023-10-09T09:12:00.135704+0000","last_became_peered":"2023-10-09T09:12:00.135704+0000","last_unstale":"2023-10-09T09:12:38.596476+0000","last_undegraded":"2023-10-09T09:12:38.596476+0000","last_fullsized":"2023-10-09T09:12:38.596476+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2122,"log_dups_size":0,"ondisk_log_size":2122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:43.169214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139067392,"num_objects":2122,"num_object_clones":0,"num_object_copies":6366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2122,"num_write_kb":135808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":40,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.467388+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:12:16.467388+0000","last_peered":"2023-10-09T09:12:16.467388+0000","last_clean":"2023-10-09T09:12:16.467388+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:12:16.467388+0000","last_undegraded":"2023-10-09T09:12:16.467388+0000","last_fullsized":"2023-10-09T09:12:16.467388+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":247,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.263624+0000","last_change":"2023-10-09T09:02:02.645869+0000","last_active":"2023-10-09T09:12:16.263624+0000","last_peered":"2023-10-09T09:12:16.263624+0000","last_clean":"2023-10-09T09:12:16.263624+0000","last_became_active":"2023-10-09T09:02:02.645499+0000","last_became_peered":"2023-10-09T09:02:02.645499+0000","last_unstale":"2023-10-09T09:12:16.263624+0000","last_undegraded":"2023-10-09T09:12:16.263624+0000","last_fullsized":"2023-10-09T09:12:16.263624+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:38.768182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"252'2225","reported_seq":2237,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.143603+0000","last_change":"2023-10-09T09:12:00.136726+0000","last_active":"2023-10-09T09:12:40.143603+0000","last_peered":"2023-10-09T09:12:40.143603+0000","last_clean":"2023-10-09T09:12:40.143603+0000","last_became_active":"2023-10-09T09:12:00.135697+0000","last_became_peered":"2023-10-09T09:12:00.135697+0000","last_unstale":"2023-10-09T09:12:40.143603+0000","last_undegraded":"2023-10-09T09:12:40.143603+0000","last_fullsized":"2023-10-09T09:12:40.143603+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2225,"log_dups_size":0,"ondisk_log_size":2225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145817600,"num_objects":2225,"num_object_clones":0,"num_object_copies":6675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2225,"num_write_kb":142400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":59,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264205+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:12:16.264205+0000","last_peered":"2023-10-09T09:12:16.264205+0000","last_clean":"2023-10-09T09:12:16.264205+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:12:16.264205+0000","last_undegraded":"2023-10-09T09:12:16.264205+0000","last_fullsized":"2023-10-09T09:12:16.264205+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":114,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112605+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:12:16.112605+0000","last_peered":"2023-10-09T09:12:16.112605+0000","last_clean":"2023-10-09T09:12:16.112605+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:12:16.112605+0000","last_undegraded":"2023-10-09T09:12:16.112605+0000","last_fullsized":"2023-10-09T09:12:16.112605+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"252'2242","reported_seq":2253,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.184057+0000","last_change":"2023-10-09T09:12:00.136740+0000","last_active":"2023-10-09T09:12:40.184057+0000","last_peered":"2023-10-09T09:12:40.184057+0000","last_clean":"2023-10-09T09:12:40.184057+0000","last_became_active":"2023-10-09T09:12:00.135501+0000","last_became_peered":"2023-10-09T09:12:00.135501+0000","last_unstale":"2023-10-09T09:12:40.184057+0000","last_undegraded":"2023-10-09T09:12:40.184057+0000","last_fullsized":"2023-10-09T09:12:40.184057+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2242,"log_dups_size":0,"ondisk_log_size":2242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146931712,"num_objects":2242,"num_object_clones":0,"num_object_copies":6726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2242,"num_write_kb":143488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.837610+0000","last_change":"2023-10-09T09:02:02.623209+0000","last_active":"2023-10-09T09:12:16.837610+0000","last_peered":"2023-10-09T09:12:16.837610+0000","last_clean":"2023-10-09T09:12:16.837610+0000","last_became_active":"2023-10-09T09:02:02.622851+0000","last_became_peered":"2023-10-09T09:02:02.622851+0000","last_unstale":"2023-10-09T09:12:16.837610+0000","last_undegraded":"2023-10-09T09:12:16.837610+0000","last_fullsized":"2023-10-09T09:12:16.837610+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":114,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112597+0000","last_change":"2023-10-09T09:02:02.631213+0000","last_active":"2023-10-09T09:12:16.112597+0000","last_peered":"2023-10-09T09:12:16.112597+0000","last_clean":"2023-10-09T09:12:16.112597+0000","last_became_active":"2023-10-09T09:02:02.630779+0000","last_became_peered":"2023-10-09T09:02:02.630779+0000","last_unstale":"2023-10-09T09:12:16.112597+0000","last_undegraded":"2023-10-09T09:12:16.112597+0000","last_fullsized":"2023-10-09T09:12:16.112597+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"252'2210","reported_seq":2222,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.132834+0000","last_change":"2023-10-09T09:12:00.116717+0000","last_active":"2023-10-09T09:12:40.132834+0000","last_peered":"2023-10-09T09:12:40.132834+0000","last_clean":"2023-10-09T09:12:40.132834+0000","last_became_active":"2023-10-09T09:12:00.116306+0000","last_became_peered":"2023-10-09T09:12:00.116306+0000","last_unstale":"2023-10-09T09:12:40.132834+0000","last_undegraded":"2023-10-09T09:12:40.132834+0000","last_fullsized":"2023-10-09T09:12:40.132834+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2210,"log_dups_size":0,"ondisk_log_size":2210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144834560,"num_objects":2210,"num_object_clones":0,"num_object_copies":6630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2210,"num_write_kb":141440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":141,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.113098+0000","last_change":"2023-10-09T09:09:44.352729+0000","last_active":"2023-10-09T09:12:16.113098+0000","last_peered":"2023-10-09T09:12:16.113098+0000","last_clean":"2023-10-09T09:12:16.113098+0000","last_became_active":"2023-10-09T09:09:44.352124+0000","last_became_peered":"2023-10-09T09:09:44.352124+0000","last_unstale":"2023-10-09T09:12:16.113098+0000","last_undegraded":"2023-10-09T09:12:16.113098+0000","last_fullsized":"2023-10-09T09:12:16.113098+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112657+0000","last_change":"2023-10-09T09:02:02.627432+0000","last_active":"2023-10-09T09:12:16.112657+0000","last_peered":"2023-10-09T09:12:16.112657+0000","last_clean":"2023-10-09T09:12:16.112657+0000","last_became_active":"2023-10-09T09:02:02.627014+0000","last_became_peered":"2023-10-09T09:02:02.627014+0000","last_unstale":"2023-10-09T09:12:16.112657+0000","last_undegraded":"2023-10-09T09:12:16.112657+0000","last_fullsized":"2023-10-09T09:12:16.112657+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"252'2229","reported_seq":2241,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.163193+0000","last_change":"2023-10-09T09:12:00.116690+0000","last_active":"2023-10-09T09:12:40.163193+0000","last_peered":"2023-10-09T09:12:40.163193+0000","last_clean":"2023-10-09T09:12:40.163193+0000","last_became_active":"2023-10-09T09:12:00.116303+0000","last_became_peered":"2023-10-09T09:12:00.116303+0000","last_unstale":"2023-10-09T09:12:40.163193+0000","last_undegraded":"2023-10-09T09:12:40.163193+0000","last_fullsized":"2023-10-09T09:12:40.163193+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2229,"log_dups_size":0,"ondisk_log_size":2229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146079744,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2229,"num_write_kb":142656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.294037+0000","last_change":"2023-10-09T09:09:44.362102+0000","last_active":"2023-10-09T09:12:16.294037+0000","last_peered":"2023-10-09T09:12:16.294037+0000","last_clean":"2023-10-09T09:12:16.294037+0000","last_became_active":"2023-10-09T09:09:44.361694+0000","last_became_peered":"2023-10-09T09:09:44.361694+0000","last_unstale":"2023-10-09T09:12:16.294037+0000","last_undegraded":"2023-10-09T09:12:16.294037+0000","last_fullsized":"2023-10-09T09:12:16.294037+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":225,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112622+0000","last_change":"2023-10-09T09:09:44.353425+0000","last_active":"2023-10-09T09:12:16.112622+0000","last_peered":"2023-10-09T09:12:16.112622+0000","last_clean":"2023-10-09T09:12:16.112622+0000","last_became_active":"2023-10-09T09:09:44.352821+0000","last_became_peered":"2023-10-09T09:09:44.352821+0000","last_unstale":"2023-10-09T09:12:16.112622+0000","last_undegraded":"2023-10-09T09:12:16.112622+0000","last_fullsized":"2023-10-09T09:12:16.112622+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"252'2242","reported_seq":2254,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.108248+0000","last_change":"2023-10-09T09:12:00.116375+0000","last_active":"2023-10-09T09:12:40.108248+0000","last_peered":"2023-10-09T09:12:40.108248+0000","last_clean":"2023-10-09T09:12:40.108248+0000","last_became_active":"2023-10-09T09:12:00.115851+0000","last_became_peered":"2023-10-09T09:12:00.115851+0000","last_unstale":"2023-10-09T09:12:40.108248+0000","last_undegraded":"2023-10-09T09:12:40.108248+0000","last_fullsized":"2023-10-09T09:12:40.108248+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2242,"log_dups_size":0,"ondisk_log_size":2242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146931712,"num_objects":2242,"num_object_clones":0,"num_object_copies":6726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2242,"num_write_kb":143488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":207,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:21.149957+0000","last_change":"2023-10-09T09:09:44.376837+0000","last_active":"2023-10-09T09:12:21.149957+0000","last_peered":"2023-10-09T09:12:21.149957+0000","last_clean":"2023-10-09T09:12:21.149957+0000","last_became_active":"2023-10-09T09:09:44.376542+0000","last_became_peered":"2023-10-09T09:09:44.376542+0000","last_unstale":"2023-10-09T09:12:21.149957+0000","last_undegraded":"2023-10-09T09:12:21.149957+0000","last_fullsized":"2023-10-09T09:12:21.149957+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:53.092218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":40,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.467231+0000","last_change":"2023-10-09T09:08:24.730554+0000","last_active":"2023-10-09T09:12:16.467231+0000","last_peered":"2023-10-09T09:12:16.467231+0000","last_clean":"2023-10-09T09:12:16.467231+0000","last_became_active":"2023-10-09T09:08:24.729359+0000","last_became_peered":"2023-10-09T09:08:24.729359+0000","last_unstale":"2023-10-09T09:12:16.467231+0000","last_undegraded":"2023-10-09T09:12:16.467231+0000","last_fullsized":"2023-10-09T09:12:16.467231+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.467062+0000","last_change":"2023-10-09T09:08:24.658172+0000","last_active":"2023-10-09T09:12:16.467062+0000","last_peered":"2023-10-09T09:12:16.467062+0000","last_clean":"2023-10-09T09:12:16.467062+0000","last_became_active":"2023-10-09T09:08:24.657766+0000","last_became_peered":"2023-10-09T09:08:24.657766+0000","last_unstale":"2023-10-09T09:12:16.467062+0000","last_undegraded":"2023-10-09T09:12:16.467062+0000","last_fullsized":"2023-10-09T09:12:16.467062+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"252'2265","reported_seq":2277,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.126433+0000","last_change":"2023-10-09T09:12:00.114449+0000","last_active":"2023-10-09T09:12:40.126433+0000","last_peered":"2023-10-09T09:12:40.126433+0000","last_clean":"2023-10-09T09:12:40.126433+0000","last_became_active":"2023-10-09T09:12:00.113992+0000","last_became_peered":"2023-10-09T09:12:00.113992+0000","last_unstale":"2023-10-09T09:12:40.126433+0000","last_undegraded":"2023-10-09T09:12:40.126433+0000","last_fullsized":"2023-10-09T09:12:40.126433+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2265,"log_dups_size":0,"ondisk_log_size":2265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148439040,"num_objects":2265,"num_object_clones":0,"num_object_copies":6795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2265,"num_write_kb":144960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":40,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.467783+0000","last_change":"2023-10-09T09:08:24.658358+0000","last_active":"2023-10-09T09:12:16.467783+0000","last_peered":"2023-10-09T09:12:16.467783+0000","last_clean":"2023-10-09T09:12:16.467783+0000","last_became_active":"2023-10-09T09:08:24.657360+0000","last_became_peered":"2023-10-09T09:08:24.657360+0000","last_unstale":"2023-10-09T09:12:16.467783+0000","last_undegraded":"2023-10-09T09:12:16.467783+0000","last_fullsized":"2023-10-09T09:12:16.467783+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":114,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112260+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:12:16.112260+0000","last_peered":"2023-10-09T09:12:16.112260+0000","last_clean":"2023-10-09T09:12:16.112260+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:12:16.112260+0000","last_undegraded":"2023-10-09T09:12:16.112260+0000","last_fullsized":"2023-10-09T09:12:16.112260+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":222,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:21.147695+0000","last_change":"2023-10-09T09:03:19.556850+0000","last_active":"2023-10-09T09:12:21.147695+0000","last_peered":"2023-10-09T09:12:21.147695+0000","last_clean":"2023-10-09T09:12:21.147695+0000","last_became_active":"2023-10-09T09:03:19.556130+0000","last_became_peered":"2023-10-09T09:03:19.556130+0000","last_unstale":"2023-10-09T09:12:21.147695+0000","last_undegraded":"2023-10-09T09:12:21.147695+0000","last_fullsized":"2023-10-09T09:12:21.147695+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:48.063809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"252'2229","reported_seq":2241,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.184156+0000","last_change":"2023-10-09T09:12:00.117269+0000","last_active":"2023-10-09T09:12:40.184156+0000","last_peered":"2023-10-09T09:12:40.184156+0000","last_clean":"2023-10-09T09:12:40.184156+0000","last_became_active":"2023-10-09T09:12:00.116216+0000","last_became_peered":"2023-10-09T09:12:00.116216+0000","last_unstale":"2023-10-09T09:12:40.184156+0000","last_undegraded":"2023-10-09T09:12:40.184156+0000","last_fullsized":"2023-10-09T09:12:40.184156+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2229,"log_dups_size":0,"ondisk_log_size":2229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146079744,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2229,"num_write_kb":142656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":280,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.811994+0000","last_change":"2023-10-09T09:03:19.556935+0000","last_active":"2023-10-09T09:12:16.811994+0000","last_peered":"2023-10-09T09:12:16.811994+0000","last_clean":"2023-10-09T09:12:16.811994+0000","last_became_active":"2023-10-09T09:03:19.556639+0000","last_became_peered":"2023-10-09T09:03:19.556639+0000","last_unstale":"2023-10-09T09:12:16.811994+0000","last_undegraded":"2023-10-09T09:12:16.811994+0000","last_fullsized":"2023-10-09T09:12:16.811994+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:40.346132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":212,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.837232+0000","last_change":"2023-10-09T09:02:02.633916+0000","last_active":"2023-10-09T09:12:16.837232+0000","last_peered":"2023-10-09T09:12:16.837232+0000","last_clean":"2023-10-09T09:12:16.837232+0000","last_became_active":"2023-10-09T09:02:02.633517+0000","last_became_peered":"2023-10-09T09:02:02.633517+0000","last_unstale":"2023-10-09T09:12:16.837232+0000","last_undegraded":"2023-10-09T09:12:16.837232+0000","last_fullsized":"2023-10-09T09:12:16.837232+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":49,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.466684+0000","last_change":"2023-10-09T09:09:44.441355+0000","last_active":"2023-10-09T09:12:16.466684+0000","last_peered":"2023-10-09T09:12:16.466684+0000","last_clean":"2023-10-09T09:12:16.466684+0000","last_became_active":"2023-10-09T09:09:44.408597+0000","last_became_peered":"2023-10-09T09:09:44.408597+0000","last_unstale":"2023-10-09T09:12:16.466684+0000","last_undegraded":"2023-10-09T09:12:16.466684+0000","last_fullsized":"2023-10-09T09:12:16.466684+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"252'2216","reported_seq":2228,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.133116+0000","last_change":"2023-10-09T09:12:00.113001+0000","last_active":"2023-10-09T09:12:40.133116+0000","last_peered":"2023-10-09T09:12:40.133116+0000","last_clean":"2023-10-09T09:12:40.133116+0000","last_became_active":"2023-10-09T09:12:00.112400+0000","last_became_peered":"2023-10-09T09:12:00.112400+0000","last_unstale":"2023-10-09T09:12:40.133116+0000","last_undegraded":"2023-10-09T09:12:40.133116+0000","last_fullsized":"2023-10-09T09:12:40.133116+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2216,"log_dups_size":0,"ondisk_log_size":2216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145227776,"num_objects":2216,"num_object_clones":0,"num_object_copies":6648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2216,"num_write_kb":141824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":49,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.466486+0000","last_change":"2023-10-09T09:09:44.356239+0000","last_active":"2023-10-09T09:12:16.466486+0000","last_peered":"2023-10-09T09:12:16.466486+0000","last_clean":"2023-10-09T09:12:16.466486+0000","last_became_active":"2023-10-09T09:09:44.355854+0000","last_became_peered":"2023-10-09T09:09:44.355854+0000","last_unstale":"2023-10-09T09:12:16.466486+0000","last_undegraded":"2023-10-09T09:12:16.466486+0000","last_fullsized":"2023-10-09T09:12:16.466486+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":490,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.294425+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:12:16.294425+0000","last_peered":"2023-10-09T09:12:16.294425+0000","last_clean":"2023-10-09T09:12:16.294425+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:12:16.294425+0000","last_undegraded":"2023-10-09T09:12:16.294425+0000","last_fullsized":"2023-10-09T09:12:16.294425+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.467697+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:12:16.467697+0000","last_peered":"2023-10-09T09:12:16.467697+0000","last_clean":"2023-10-09T09:12:16.467697+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:12:16.467697+0000","last_undegraded":"2023-10-09T09:12:16.467697+0000","last_fullsized":"2023-10-09T09:12:16.467697+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.836022+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:12:16.836022+0000","last_peered":"2023-10-09T09:12:16.836022+0000","last_clean":"2023-10-09T09:12:16.836022+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:12:16.836022+0000","last_undegraded":"2023-10-09T09:12:16.836022+0000","last_fullsized":"2023-10-09T09:12:16.836022+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"252'2114","reported_seq":2127,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:38.346222+0000","last_change":"2023-10-09T09:12:00.116909+0000","last_active":"2023-10-09T09:12:38.346222+0000","last_peered":"2023-10-09T09:12:38.346222+0000","last_clean":"2023-10-09T09:12:38.346222+0000","last_became_active":"2023-10-09T09:12:00.116565+0000","last_became_peered":"2023-10-09T09:12:00.116565+0000","last_unstale":"2023-10-09T09:12:38.346222+0000","last_undegraded":"2023-10-09T09:12:38.346222+0000","last_fullsized":"2023-10-09T09:12:38.346222+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2114,"log_dups_size":0,"ondisk_log_size":2114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:01.594908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138543104,"num_objects":2114,"num_object_clones":0,"num_object_copies":6342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2114,"num_write_kb":135296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.294246+0000","last_change":"2023-10-09T09:09:44.393807+0000","last_active":"2023-10-09T09:12:16.294246+0000","last_peered":"2023-10-09T09:12:16.294246+0000","last_clean":"2023-10-09T09:12:16.294246+0000","last_became_active":"2023-10-09T09:09:44.393066+0000","last_became_peered":"2023-10-09T09:09:44.393066+0000","last_unstale":"2023-10-09T09:12:16.294246+0000","last_undegraded":"2023-10-09T09:12:16.294246+0000","last_fullsized":"2023-10-09T09:12:16.294246+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":284,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264561+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:12:16.264561+0000","last_peered":"2023-10-09T09:12:16.264561+0000","last_clean":"2023-10-09T09:12:16.264561+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:12:16.264561+0000","last_undegraded":"2023-10-09T09:12:16.264561+0000","last_fullsized":"2023-10-09T09:12:16.264561+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":263,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.263908+0000","last_change":"2023-10-09T09:09:44.378484+0000","last_active":"2023-10-09T09:12:16.263908+0000","last_peered":"2023-10-09T09:12:16.263908+0000","last_clean":"2023-10-09T09:12:16.263908+0000","last_became_active":"2023-10-09T09:09:44.378153+0000","last_became_peered":"2023-10-09T09:09:44.378153+0000","last_unstale":"2023-10-09T09:12:16.263908+0000","last_undegraded":"2023-10-09T09:12:16.263908+0000","last_fullsized":"2023-10-09T09:12:16.263908+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:18.230306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"252'2211","reported_seq":2222,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.142533+0000","last_change":"2023-10-09T09:12:00.113109+0000","last_active":"2023-10-09T09:12:40.142533+0000","last_peered":"2023-10-09T09:12:40.142533+0000","last_clean":"2023-10-09T09:12:40.142533+0000","last_became_active":"2023-10-09T09:12:00.112495+0000","last_became_peered":"2023-10-09T09:12:00.112495+0000","last_unstale":"2023-10-09T09:12:40.142533+0000","last_undegraded":"2023-10-09T09:12:40.142533+0000","last_fullsized":"2023-10-09T09:12:40.142533+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2211,"log_dups_size":0,"ondisk_log_size":2211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144900096,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2211,"num_write_kb":141504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.295749+0000","last_change":"2023-10-09T09:09:44.439090+0000","last_active":"2023-10-09T09:12:16.295749+0000","last_peered":"2023-10-09T09:12:16.295749+0000","last_clean":"2023-10-09T09:12:16.295749+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:12:16.295749+0000","last_undegraded":"2023-10-09T09:12:16.295749+0000","last_fullsized":"2023-10-09T09:12:16.295749+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":233,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:21.147457+0000","last_change":"2023-10-09T09:08:24.652222+0000","last_active":"2023-10-09T09:12:21.147457+0000","last_peered":"2023-10-09T09:12:21.147457+0000","last_clean":"2023-10-09T09:12:21.147457+0000","last_became_active":"2023-10-09T09:08:24.651702+0000","last_became_peered":"2023-10-09T09:08:24.651702+0000","last_unstale":"2023-10-09T09:12:21.147457+0000","last_undegraded":"2023-10-09T09:12:21.147457+0000","last_fullsized":"2023-10-09T09:12:21.147457+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:49.138740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":260,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264406+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:12:16.264406+0000","last_peered":"2023-10-09T09:12:16.264406+0000","last_clean":"2023-10-09T09:12:16.264406+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:12:16.264406+0000","last_undegraded":"2023-10-09T09:12:16.264406+0000","last_fullsized":"2023-10-09T09:12:16.264406+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"252'2209","reported_seq":2221,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.150333+0000","last_change":"2023-10-09T09:12:00.114006+0000","last_active":"2023-10-09T09:12:40.150333+0000","last_peered":"2023-10-09T09:12:40.150333+0000","last_clean":"2023-10-09T09:12:40.150333+0000","last_became_active":"2023-10-09T09:12:00.113612+0000","last_became_peered":"2023-10-09T09:12:00.113612+0000","last_unstale":"2023-10-09T09:12:40.150333+0000","last_undegraded":"2023-10-09T09:12:40.150333+0000","last_fullsized":"2023-10-09T09:12:40.150333+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2209,"log_dups_size":0,"ondisk_log_size":2209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144769024,"num_objects":2209,"num_object_clones":0,"num_object_copies":6627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2209,"num_write_kb":141376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":82,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.812418+0000","last_change":"2023-10-09T09:09:44.616450+0000","last_active":"2023-10-09T09:12:16.812418+0000","last_peered":"2023-10-09T09:12:16.812418+0000","last_clean":"2023-10-09T09:12:16.812418+0000","last_became_active":"2023-10-09T09:09:44.393002+0000","last_became_peered":"2023-10-09T09:09:44.393002+0000","last_unstale":"2023-10-09T09:12:16.812418+0000","last_undegraded":"2023-10-09T09:12:16.812418+0000","last_fullsized":"2023-10-09T09:12:16.812418+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:53:56.650152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":214,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.838447+0000","last_change":"2023-10-09T09:02:02.623362+0000","last_active":"2023-10-09T09:12:16.838447+0000","last_peered":"2023-10-09T09:12:16.838447+0000","last_clean":"2023-10-09T09:12:16.838447+0000","last_became_active":"2023-10-09T09:02:02.622951+0000","last_became_peered":"2023-10-09T09:02:02.622951+0000","last_unstale":"2023-10-09T09:12:16.838447+0000","last_undegraded":"2023-10-09T09:12:16.838447+0000","last_fullsized":"2023-10-09T09:12:16.838447+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":32,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.293999+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:12:16.293999+0000","last_peered":"2023-10-09T09:12:16.293999+0000","last_clean":"2023-10-09T09:12:16.293999+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:12:16.293999+0000","last_undegraded":"2023-10-09T09:12:16.293999+0000","last_fullsized":"2023-10-09T09:12:16.293999+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"252'2215","reported_seq":2227,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T09:12:40.152497+0000","last_change":"2023-10-09T09:12:00.136672+0000","last_active":"2023-10-09T09:12:40.152497+0000","last_peered":"2023-10-09T09:12:40.152497+0000","last_clean":"2023-10-09T09:12:40.152497+0000","last_became_active":"2023-10-09T09:12:00.135737+0000","last_became_peered":"2023-10-09T09:12:00.135737+0000","last_unstale":"2023-10-09T09:12:40.152497+0000","last_undegraded":"2023-10-09T09:12:40.152497+0000","last_fullsized":"2023-10-09T09:12:40.152497+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":2215,"log_dups_size":0,"ondisk_log_size":2215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145162240,"num_objects":2215,"num_object_clones":0,"num_object_copies":6645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2215,"num_write_kb":141760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":101,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.838780+0000","last_change":"2023-10-09T09:00:31.848567+0000","last_active":"2023-10-09T09:12:16.838780+0000","last_peered":"2023-10-09T09:12:16.838780+0000","last_clean":"2023-10-09T09:12:16.838780+0000","last_became_active":"2023-10-09T09:00:31.848241+0000","last_became_peered":"2023-10-09T09:00:31.848241+0000","last_unstale":"2023-10-09T09:12:16.838780+0000","last_undegraded":"2023-10-09T09:12:16.838780+0000","last_fullsized":"2023-10-09T09:12:16.838780+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":290,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264238+0000","last_change":"2023-10-09T09:09:44.610749+0000","last_active":"2023-10-09T09:12:16.264238+0000","last_peered":"2023-10-09T09:12:16.264238+0000","last_clean":"2023-10-09T09:12:16.264238+0000","last_became_active":"2023-10-09T09:09:44.391749+0000","last_became_peered":"2023-10-09T09:09:44.391749+0000","last_unstale":"2023-10-09T09:12:16.264238+0000","last_undegraded":"2023-10-09T09:12:16.264238+0000","last_fullsized":"2023-10-09T09:12:16.264238+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:23.536700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":132,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112337+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:12:16.112337+0000","last_peered":"2023-10-09T09:12:16.112337+0000","last_clean":"2023-10-09T09:12:16.112337+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:12:16.112337+0000","last_undegraded":"2023-10-09T09:12:16.112337+0000","last_fullsized":"2023-10-09T09:12:16.112337+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":271,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264300+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:12:16.264300+0000","last_peered":"2023-10-09T09:12:16.264300+0000","last_clean":"2023-10-09T09:12:16.264300+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:12:16.264300+0000","last_undegraded":"2023-10-09T09:12:16.264300+0000","last_fullsized":"2023-10-09T09:12:16.264300+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":84,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.812785+0000","last_change":"2023-10-09T09:03:19.557040+0000","last_active":"2023-10-09T09:12:16.812785+0000","last_peered":"2023-10-09T09:12:16.812785+0000","last_clean":"2023-10-09T09:12:16.812785+0000","last_became_active":"2023-10-09T09:03:19.556518+0000","last_became_peered":"2023-10-09T09:03:19.556518+0000","last_unstale":"2023-10-09T09:12:16.812785+0000","last_undegraded":"2023-10-09T09:12:16.812785+0000","last_fullsized":"2023-10-09T09:12:16.812785+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:40.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":44,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.467607+0000","last_change":"2023-10-09T09:08:24.661552+0000","last_active":"2023-10-09T09:12:16.467607+0000","last_peered":"2023-10-09T09:12:16.467607+0000","last_clean":"2023-10-09T09:12:16.467607+0000","last_became_active":"2023-10-09T09:08:24.655946+0000","last_became_peered":"2023-10-09T09:08:24.655946+0000","last_unstale":"2023-10-09T09:12:16.467607+0000","last_undegraded":"2023-10-09T09:12:16.467607+0000","last_fullsized":"2023-10-09T09:12:16.467607+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":114,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.112187+0000","last_change":"2023-10-09T09:01:50.564803+0000","last_active":"2023-10-09T09:12:16.112187+0000","last_peered":"2023-10-09T09:12:16.112187+0000","last_clean":"2023-10-09T09:12:16.112187+0000","last_became_active":"2023-10-09T09:01:50.564416+0000","last_became_peered":"2023-10-09T09:01:50.564416+0000","last_unstale":"2023-10-09T09:12:16.112187+0000","last_undegraded":"2023-10-09T09:12:16.112187+0000","last_fullsized":"2023-10-09T09:12:16.112187+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":44,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.467175+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:12:16.467175+0000","last_peered":"2023-10-09T09:12:16.467175+0000","last_clean":"2023-10-09T09:12:16.467175+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:12:16.467175+0000","last_undegraded":"2023-10-09T09:12:16.467175+0000","last_fullsized":"2023-10-09T09:12:16.467175+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":114,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.111936+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:12:16.111936+0000","last_peered":"2023-10-09T09:12:16.111936+0000","last_clean":"2023-10-09T09:12:16.111936+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:12:16.111936+0000","last_undegraded":"2023-10-09T09:12:16.111936+0000","last_fullsized":"2023-10-09T09:12:16.111936+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":188,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.838513+0000","last_change":"2023-10-09T09:09:06.657370+0000","last_active":"2023-10-09T09:12:16.838513+0000","last_peered":"2023-10-09T09:12:16.838513+0000","last_clean":"2023-10-09T09:12:16.838513+0000","last_became_active":"2023-10-09T09:09:06.656920+0000","last_became_peered":"2023-10-09T09:09:06.656920+0000","last_unstale":"2023-10-09T09:12:16.838513+0000","last_undegraded":"2023-10-09T09:12:16.838513+0000","last_fullsized":"2023-10-09T09:12:16.838513+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.811627+0000","last_change":"2023-10-09T09:09:06.665214+0000","last_active":"2023-10-09T09:12:16.811627+0000","last_peered":"2023-10-09T09:12:16.811627+0000","last_clean":"2023-10-09T09:12:16.811627+0000","last_became_active":"2023-10-09T09:09:06.664862+0000","last_became_peered":"2023-10-09T09:09:06.664862+0000","last_unstale":"2023-10-09T09:12:16.811627+0000","last_undegraded":"2023-10-09T09:12:16.811627+0000","last_fullsized":"2023-10-09T09:12:16.811627+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:29.278156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":330,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.264643+0000","last_change":"2023-10-09T09:08:24.737079+0000","last_active":"2023-10-09T09:12:16.264643+0000","last_peered":"2023-10-09T09:12:16.264643+0000","last_clean":"2023-10-09T09:12:16.264643+0000","last_became_active":"2023-10-09T09:08:24.656441+0000","last_became_peered":"2023-10-09T09:08:24.656441+0000","last_unstale":"2023-10-09T09:12:16.264643+0000","last_undegraded":"2023-10-09T09:12:16.264643+0000","last_fullsized":"2023-10-09T09:12:16.264643+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:36.720565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":132,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T09:12:16.111701+0000","last_change":"2023-10-09T09:03:19.573375+0000","last_active":"2023-10-09T09:12:16.111701+0000","last_peered":"2023-10-09T09:12:16.111701+0000","last_clean":"2023-10-09T09:12:16.111701+0000","last_became_active":"2023-10-09T09:03:19.572847+0000","last_became_peered":"2023-10-09T09:03:19.572847+0000","last_unstale":"2023-10-09T09:12:16.111701+0000","last_undegraded":"2023-10-09T09:12:16.111701+0000","last_fullsized":"2023-10-09T09:12:16.111701+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2304507904,"num_objects":35164,"num_object_clones":0,"num_object_copies":105492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":35164,"num_write_kb":2250496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6905724928,"data_stored":6905724928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35164,"ondisk_log_size":35164,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":18386,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430706,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":999796,"kb_used_data":698476,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92745356,"statfs":{"total":95995035648,"available":94971244544,"internally_reserved":0,"allocated":715239424,"data_stored":714667450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:12:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":2.04,"15min":2.5539999999999998},"min":{"1min":0.40999999999999998,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":3.2770000000000001,"5min":47.776000000000003,"15min":119.498},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.91900000000000004,"5min":2.0179999999999998,"15min":2.5699999999999998},"min":{"1min":0.54900000000000004,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":3.266,"5min":47.689,"15min":119.532},"last":0.48199999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:12:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":1.044,"15min":1.206},"min":{"1min":0.32700000000000001,"5min":0.30299999999999999,"15min":0.29699999999999999},"max":{"1min":12.682,"5min":12.682,"15min":22.93},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.70499999999999996,"15min":1.0880000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6200000000000001,"5min":3.738,"15min":22.866},"last":0.51600000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:12:24 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.5529999999999999,"15min":2.1829999999999998},"min":{"1min":0.495,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":1.246,"5min":8.2089999999999996,"15min":126.072},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.615,"15min":2.222},"min":{"1min":0.42499999999999999,"5min":0.307,"15min":0.307},"max":{"1min":1.1890000000000001,"5min":12.643000000000001,"15min":120.729},"last":0.57399999999999995}]},{"osd":3,"last update":"Mon Oct 9 09:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":2.1349999999999998,"15min":3.1779999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":3.5710000000000002,"5min":51.299999999999997,"15min":191.30799999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":2.0720000000000001,"5min":2.3860000000000001,"15min":3.2549999999999999},"min":{"1min":0.54000000000000004,"5min":0.318,"15min":0.318},"max":{"1min":12.714,"5min":50.624000000000002,"15min":191.47900000000001},"last":1.6259999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1150000000000002,"5min":1.5629999999999999,"15min":1.5680000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":13.895,"5min":13.895,"15min":138.34299999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":3.0750000000000002,"5min":1.55,"15min":1.6040000000000001},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":16.433,"5min":16.433,"15min":137.04900000000001},"last":1.6850000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.194,"15min":0.94399999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6339999999999999,"5min":9.468,"15min":9.468},"last":0.66400000000000003},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.597,"15min":1.101},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":13.82,"5min":13.82,"15min":13.82},"last":0.85099999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.647,"15min":1.1919999999999999},"min":{"1min":0.39700000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":13.867000000000001,"5min":13.867000000000001,"15min":13.867000000000001},"last":1.8500000000000001},{"interface":"front","average":{"1min":3.1680000000000001,"5min":1.742,"15min":1.28},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.34499999999999997},"max":{"1min":19.068999999999999,"5min":19.068999999999999,"15min":19.068999999999999},"last":3.9369999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626939,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1975464,"kb_used_data":1068000,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91769688,"statfs":{"total":95995035648,"available":93972160512,"internally_reserved":0,"allocated":1093632000,"data_stored":1093072118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.7709999999999999,"15min":2.3639999999999999},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.24199999999999999},"max":{"1min":12.311,"5min":19.568999999999999,"15min":153.63800000000001},"last":2.7290000000000001},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.7070000000000001,"15min":2.4340000000000002},"min":{"1min":0.46500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.268000000000001,"5min":19.498000000000001,"15min":153.72200000000001},"last":2.3679999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":1.409,"15min":1.5600000000000001},"min":{"1min":0.317,"5min":0.314,"15min":0.29699999999999999},"max":{"1min":12.369,"5min":16.879000000000001,"15min":109.453},"last":2.782},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.242,"15min":1.5800000000000001},"min":{"1min":0.39900000000000002,"5min":0.315,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":111.47499999999999},"last":2.4350000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.9179999999999999,"15min":1.4279999999999999},"min":{"1min":0.59999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.3819999999999997,"5min":17.001999999999999,"15min":18.756},"last":2.8140000000000001},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.8200000000000001,"15min":1.54},"min":{"1min":0.66600000000000004,"5min":0.309,"15min":0.28100000000000003},"max":{"1min":5.54,"5min":12.521000000000001,"15min":18.841999999999999},"last":2.4089999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.7050000000000001,"15min":1.5860000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":12.35,"5min":16.977,"15min":41.161000000000001},"last":2.8399999999999999},{"interface":"front","average":{"1min":1.6519999999999999,"5min":2.056,"15min":1.7749999999999999},"min":{"1min":0.46000000000000002,"5min":0.371,"15min":0.371},"max":{"1min":12.628,"5min":20.573,"15min":40.158000000000001},"last":4.5330000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.349,"15min":1.1930000000000001},"min":{"1min":0.35399999999999998,"5min":0.33300000000000002,"15min":0.314},"max":{"1min":12.606,"5min":12.606,"15min":32.280999999999999},"last":2.5150000000000001},{"interface":"front","average":{"1min":1.528,"5min":1.512,"15min":1.1739999999999999},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.317},"max":{"1min":12.772,"5min":16.648,"15min":34.689},"last":4.6059999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:11:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.7070000000000001,"15min":1.218},"min":{"1min":0.35699999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.359,"5min":12.359,"15min":12.359},"last":4.4740000000000002},{"interface":"front","average":{"1min":1.641,"5min":1.6359999999999999,"15min":1.1519999999999999},"min":{"1min":0.378,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":12.696,"5min":13.609,"15min":13.609},"last":5.3680000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:11:44 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.6279999999999999,"15min":1.2050000000000001},"min":{"1min":0.40000000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":12.327,"5min":12.327,"15min":12.327},"last":2.492},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.4710000000000001,"15min":1.0720000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":12.667999999999999,"5min":13.523999999999999,"15min":13.523999999999999},"last":4.5720000000000001}]}]},{"osd":1,"up_from":0,"seq":356482285887,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":78,"seq":335007449412,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1982332,"kb_used_data":1294132,"kb_used_omap":28,"kb_used_meta":688163,"kb_avail":91762820,"statfs":{"total":95995035648,"available":93965127680,"internally_reserved":0,"allocated":1325191168,"data_stored":1324205124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":704679760},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":1.177,"15min":1.4390000000000001},"min":{"1min":0.36099999999999999,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":1.28,"5min":8.6489999999999991,"15min":9.9879999999999995},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.97399999999999998,"15min":1.3819999999999999},"min":{"1min":0.34999999999999998,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":0.89500000000000002,"5min":9.2609999999999992,"15min":18.661000000000001},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":2.3719999999999999,"15min":2.8919999999999999},"min":{"1min":0.33300000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.9860000000000002,"5min":61.75,"15min":71.322999999999993},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":2.4729999999999999,"15min":3.774},"min":{"1min":0.31,"5min":0.31,"15min":0.24399999999999999},"max":{"1min":3.7130000000000001,"5min":61.718000000000004,"15min":219.00700000000001},"last":0.56299999999999994}]},{"osd":3,"last update":"Mon Oct 9 09:11:57 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.2000000000000002,"15min":3.0430000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.27700000000000002},"max":{"1min":1.4370000000000001,"5min":9.2929999999999993,"15min":219.041},"last":20.364000000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":3.1509999999999998,"15min":2.585},"min":{"1min":0.36499999999999999,"5min":0.29599999999999999,"15min":0.25},"max":{"1min":1.6379999999999999,"5min":137.94999999999999,"15min":137.94999999999999},"last":18.895}]},{"osd":4,"last update":"Mon Oct 9 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.367,"15min":1.48},"min":{"1min":0.41899999999999998,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":4.2080000000000002,"5min":9.1120000000000001,"15min":10.178000000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.88500000000000001,"5min":1.829,"15min":1.7310000000000001},"min":{"1min":0.43099999999999999,"5min":0.33000000000000002,"15min":0.29099999999999998},"max":{"1min":2.9550000000000001,"5min":10.581,"15min":11.654},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.5740000000000001,"15min":1.548},"min":{"1min":0.44600000000000001,"5min":0.41299999999999998,"15min":0.37},"max":{"1min":2.3730000000000002,"5min":8.6020000000000003,"15min":9.3689999999999998},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.171,"5min":1.7130000000000001,"15min":1.6870000000000001},"min":{"1min":0.38400000000000001,"5min":0.35299999999999998,"15min":0.33700000000000002},"max":{"1min":2.9420000000000002,"5min":16.364000000000001,"15min":16.364000000000001},"last":0.91500000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":2.75,"15min":2.2469999999999999},"min":{"1min":0.61299999999999999,"5min":0.46200000000000002,"15min":0.37},"max":{"1min":4.3140000000000001,"5min":98.864000000000004,"15min":98.864000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.022,"5min":2.754,"15min":2.452},"min":{"1min":0.44600000000000001,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":1.99,"5min":98.878,"15min":98.878},"last":0.86799999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.5609999999999999,"15min":1.6060000000000001},"min":{"1min":0.59699999999999998,"5min":0.439,"15min":0.42499999999999999},"max":{"1min":5.8159999999999998,"5min":9.2349999999999994,"15min":10.157999999999999},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.323,"5min":1.9490000000000001,"15min":1.881},"min":{"1min":0.53400000000000003,"5min":0.40799999999999997,"15min":0.40100000000000002},"max":{"1min":5.827,"5min":10.619,"15min":14.359},"last":0.748}]}]},{"osd":2,"up_from":88,"seq":377957122360,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518768,"kb_used_data":778088,"kb_used_omap":81,"kb_used_meta":740590,"kb_avail":92226384,"statfs":{"total":95995035648,"available":94439817216,"internally_reserved":0,"allocated":796762112,"data_stored":795727585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":758364378},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8499999999999996,"5min":2.8530000000000002,"15min":3.8370000000000002},"min":{"1min":0.45000000000000001,"5min":0.188,"15min":0.188},"max":{"1min":85.091999999999999,"5min":85.091999999999999,"15min":134.458},"last":0.247},{"interface":"front","average":{"1min":4.8140000000000001,"5min":2.8479999999999999,"15min":3.5710000000000002},"min":{"1min":0.26900000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":84.373999999999995,"5min":84.373999999999995,"15min":138.417},"last":0.46800000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.077,"15min":2.1640000000000001},"min":{"1min":0.34200000000000003,"5min":0.214,"15min":0.21299999999999999},"max":{"1min":3.3900000000000001,"5min":5.7130000000000001,"15min":57.939999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.97499999999999998,"15min":1.9850000000000001},"min":{"1min":0.312,"5min":0.28599999999999998,"15min":0.24199999999999999},"max":{"1min":4.0629999999999997,"5min":7.4059999999999997,"15min":57.122999999999998},"last":0.54200000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":2.5529999999999999,"15min":3.1019999999999999},"min":{"1min":0.38900000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.4059999999999999,"5min":73.221999999999994,"15min":112.47499999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.77500000000000002,"5min":2.238,"15min":3.0550000000000002},"min":{"1min":0.34799999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.7190000000000001,"5min":75.840999999999994,"15min":112.929},"last":0.32100000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.4450000000000001,"15min":1.7949999999999999},"min":{"1min":0.441,"5min":0.34399999999999997,"15min":0.316},"max":{"1min":7.8810000000000002,"5min":10.116,"15min":17.408999999999999},"last":2.613},{"interface":"front","average":{"1min":1.389,"5min":1.5189999999999999,"15min":1.673},"min":{"1min":0.53800000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.1400000000000006,"5min":8.1400000000000006,"15min":10.426},"last":0.82699999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":2.0819999999999999,"15min":1.6459999999999999},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.482,"5min":72.311000000000007,"15min":72.311000000000007},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.4179999999999999,"5min":2.3490000000000002,"15min":2.1099999999999999},"min":{"1min":0.45400000000000001,"5min":0.36899999999999999,"15min":0.35799999999999998},"max":{"1min":4.9169999999999998,"5min":73.042000000000002,"15min":73.042000000000002},"last":0.38100000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.496,"15min":1.7050000000000001},"min":{"1min":0.50900000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.5279999999999996,"5min":7.2859999999999996,"15min":10.456},"last":1.081},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.4159999999999999,"15min":1.506},"min":{"1min":0.51500000000000001,"5min":0.40999999999999998,"15min":0.39300000000000002},"max":{"1min":4.0469999999999997,"5min":10.433,"15min":12.381},"last":1.7070000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.3660000000000001,"15min":1.863},"min":{"1min":0.443,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.504,"5min":6.7309999999999999,"15min":67.491},"last":1.8160000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.48,"15min":1.502},"min":{"1min":0.48899999999999999,"5min":0.46800000000000003,"15min":0.433},"max":{"1min":2.1459999999999999,"5min":7.4329999999999998,"15min":12.438000000000001},"last":0.628}]}]},{"osd":3,"up_from":93,"seq":399431958835,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1852768,"kb_used_data":1145304,"kb_used_omap":7,"kb_used_meta":707448,"kb_avail":91892384,"statfs":{"total":95995035648,"available":94097801216,"internally_reserved":0,"allocated":1172791296,"data_stored":1171829547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":724427368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:12:00 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.4019999999999999,"15min":2.4049999999999998},"min":{"1min":0.28599999999999998,"5min":0.216,"15min":0.19400000000000001},"max":{"1min":3.1320000000000001,"5min":9.2859999999999996,"15min":233.22900000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.9199999999999999,"15min":2.7559999999999998},"min":{"1min":0.25600000000000001,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":5.327,"5min":18.030000000000001,"15min":237.447},"last":2.4140000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:12:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":1.244,"15min":2.6259999999999999},"min":{"1min":0.36599999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":0.93600000000000005,"5min":10.662000000000001,"15min":326.75700000000001},"last":1.3180000000000001},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.0960000000000001,"15min":2.508},"min":{"1min":0.33900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.0880000000000001,"5min":7.194,"15min":327.024},"last":0.56499999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:12:00 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.8759999999999999,"15min":2.4620000000000002},"min":{"1min":0.41699999999999998,"5min":0.309,"15min":0.255},"max":{"1min":0.93600000000000005,"5min":36.453000000000003,"15min":75.641999999999996},"last":1.405},{"interface":"front","average":{"1min":0.81000000000000005,"5min":2.0750000000000002,"15min":2.3639999999999999},"min":{"1min":0.36799999999999999,"5min":0.29699999999999999,"15min":0.27000000000000002},"max":{"1min":3.161,"5min":36.764000000000003,"15min":75.207999999999998},"last":0.52800000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.367,"15min":1.3069999999999999},"min":{"1min":0.51300000000000001,"5min":0.32200000000000001,"15min":0.32000000000000001},"max":{"1min":3.1899999999999999,"5min":7.2380000000000004,"15min":9.2940000000000005},"last":2.399},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.958,"15min":1.698},"min":{"1min":0.45200000000000001,"5min":0.313,"15min":0.313},"max":{"1min":5.4130000000000003,"5min":10.505000000000001,"15min":10.505000000000001},"last":2.3919999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":2.0529999999999999,"15min":1.7070000000000001},"min":{"1min":0.48599999999999999,"5min":0.38100000000000001,"15min":0.33700000000000002},"max":{"1min":5.3810000000000002,"5min":16.132999999999999,"15min":16.132999999999999},"last":2.3260000000000001},{"interface":"front","average":{"1min":0.85199999999999998,"5min":1.724,"15min":1.5940000000000001},"min":{"1min":0.54800000000000004,"5min":0.41399999999999998,"15min":0.35599999999999998},"max":{"1min":1.139,"5min":14.361000000000001,"15min":14.361000000000001},"last":3.7829999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":2.1680000000000001,"15min":1.8460000000000001},"min":{"1min":0.60299999999999998,"5min":0.375,"15min":0.308},"max":{"1min":5.9889999999999999,"5min":12.105,"15min":12.105},"last":2.3599999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.7050000000000001,"15min":1.72},"min":{"1min":0.622,"5min":0.38500000000000001,"15min":0.35099999999999998},"max":{"1min":2.0230000000000001,"5min":7.0890000000000004,"15min":9.5090000000000003},"last":1.385}]},{"osd":7,"last update":"Mon Oct 9 09:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.7869999999999999,"15min":1.653},"min":{"1min":0.64800000000000002,"5min":0.41499999999999998,"15min":0.41299999999999998},"max":{"1min":5.0339999999999998,"5min":7.1219999999999999,"15min":7.5899999999999999},"last":1.976},{"interface":"front","average":{"1min":1.641,"5min":1.651,"15min":1.4930000000000001},"min":{"1min":0.55700000000000005,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.0510000000000002,"5min":7.7999999999999998,"15min":9.2509999999999994},"last":1.948}]}]},{"osd":4,"up_from":30,"seq":128849019405,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179052,"kb_used_data":1272676,"kb_used_omap":0,"kb_used_meta":906368,"kb_avail":91566100,"statfs":{"total":95995035648,"available":93763686400,"internally_reserved":0,"allocated":1303220224,"data_stored":1302656071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928120832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.6100000000000001,"15min":2.069},"min":{"1min":0.627,"5min":0.25800000000000001,"15min":0.248},"max":{"1min":2.9100000000000001,"5min":27.677,"15min":141.07400000000001},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.8009999999999999,"15min":2.2120000000000002},"min":{"1min":0.40899999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.694,"5min":27.344000000000001,"15min":141.15600000000001},"last":3.694}]},{"osd":1,"last update":"Mon Oct 9 09:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.93899999999999995,"15min":1.0429999999999999},"min":{"1min":0.317,"5min":0.313,"15min":0.29899999999999999},"max":{"1min":3.9300000000000002,"5min":5.3140000000000001,"15min":7.5780000000000003},"last":0.373},{"interface":"front","average":{"1min":1.107,"5min":0.84499999999999997,"15min":1.155},"min":{"1min":0.5,"5min":0.30099999999999999,"15min":0.29199999999999998},"max":{"1min":5.8239999999999998,"5min":5.8239999999999998,"15min":7.6879999999999997},"last":0.92500000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.45,"15min":2.0110000000000001},"min":{"1min":0.48699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.532,"5min":27.297000000000001,"15min":142.816},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.75,"15min":1.9359999999999999},"min":{"1min":0.432,"5min":0.315,"15min":0.315},"max":{"1min":3.5830000000000002,"5min":36.844000000000001,"15min":143.81800000000001},"last":3.5830000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":7.5289999999999999,"15min":3.9449999999999998},"min":{"1min":0.56399999999999995,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8849999999999998,"5min":165.995,"15min":165.995},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.0940000000000001,"5min":7.9139999999999997,"15min":4.2160000000000002},"min":{"1min":0.46500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":2.972,"5min":165.53200000000001,"15min":165.53200000000001},"last":3.6349999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:12:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.0329999999999999,"15min":1.018},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.7729999999999997,"5min":7.9969999999999999,"15min":7.9969999999999999},"last":3.6030000000000002},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.107,"15min":0.995},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.29399999999999998},"max":{"1min":4.0609999999999999,"5min":6.5519999999999996,"15min":6.5519999999999996},"last":2.7309999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:11:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.8340000000000001,"15min":1.238},"min":{"1min":0.40200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":5.7199999999999998,"5min":74.700999999999993,"15min":74.700999999999993},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.383,"5min":1.8600000000000001,"15min":1.389},"min":{"1min":0.35999999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.8769999999999998,"5min":73.572999999999993,"15min":73.572999999999993},"last":0.877}]},{"osd":7,"last update":"Mon Oct 9 09:12:36 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.22,"15min":1.0620000000000001},"min":{"1min":0.38600000000000001,"5min":0.377,"15min":0.32800000000000001},"max":{"1min":6.1040000000000001,"5min":7.7370000000000001,"15min":7.7370000000000001},"last":3.6669999999999998},{"interface":"front","average":{"1min":1.117,"5min":1.2869999999999999,"15min":1.0900000000000001},"min":{"1min":0.46700000000000003,"5min":0.37,"15min":0.36199999999999999},"max":{"1min":3.8849999999999998,"5min":8.2539999999999996,"15min":8.2539999999999996},"last":3.8849999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823173,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1609312,"kb_used_data":707992,"kb_used_omap":0,"kb_used_meta":901312,"kb_avail":92135840,"statfs":{"total":95995035648,"available":94347100160,"internally_reserved":0,"allocated":724983808,"data_stored":724427709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922943488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:12:38 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.883,"15min":3.5179999999999998},"min":{"1min":0.59499999999999997,"5min":0.249,"15min":0.249},"max":{"1min":2.6379999999999999,"5min":56.823999999999998,"15min":192.55799999999999},"last":2.6110000000000002},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.9690000000000001,"15min":3.6720000000000002},"min":{"1min":0.55700000000000005,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":2.327,"5min":60.362000000000002,"15min":194.11000000000001},"last":0.79900000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":1.1259999999999999,"15min":1.2470000000000001},"min":{"1min":0.33500000000000002,"5min":0.32000000000000001,"15min":0.307},"max":{"1min":1.8720000000000001,"5min":5.7640000000000002,"15min":36.963999999999999},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.81299999999999994,"5min":1.1100000000000001,"15min":1.409},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":3.238,"5min":6.9359999999999999,"15min":36.158000000000001},"last":0.40400000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:12:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.3959999999999999,"15min":2.2919999999999998},"min":{"1min":0.55800000000000005,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.1760000000000002,"5min":6.4219999999999997,"15min":156.69499999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.105,"15min":2.1139999999999999},"min":{"1min":0.45400000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.4129999999999998,"5min":4.9130000000000003,"15min":156.87799999999999},"last":0.73099999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:11:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":2.1579999999999999,"15min":3.1880000000000002},"min":{"1min":0.40899999999999997,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":40.857999999999997,"5min":57.963000000000001,"15min":161.678},"last":0.77300000000000002},{"interface":"front","average":{"1min":2.7730000000000001,"5min":2.5569999999999999,"15min":3.242},"min":{"1min":0.36799999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":40.621000000000002,"5min":57.603000000000002,"15min":159.43899999999999},"last":0.54300000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.4390000000000001,"15min":1.0800000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.8540000000000001,"5min":9.1910000000000007,"15min":9.1910000000000007},"last":1.143},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.081,"15min":1.02},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.9710000000000001,"5min":5.5469999999999997,"15min":5.5469999999999997},"last":1.0629999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.651,"15min":1.2370000000000001},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.8959999999999999,"5min":16.052,"15min":16.052},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.569,"15min":1.282},"min":{"1min":0.48499999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.9960000000000004,"5min":9.0229999999999997,"15min":10.788},"last":0.69999999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.2989999999999999,"15min":1.0669999999999999},"min":{"1min":0.52800000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.9130000000000003,"5min":8.9770000000000003,"15min":8.9770000000000003},"last":0.751},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.468,"15min":1.1220000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":4.7720000000000002,"5min":7.0659999999999998,"15min":7.0659999999999998},"last":0.49399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1358364672,"data_stored":1358364672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":827981824,"data_stored":827981824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":929366016,"data_stored":929366016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1306591232,"data_stored":1306591232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":732495872,"data_stored":732495872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1018626048,"data_stored":1018626048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":732299264,"data_stored":732299264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:12:47.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-09T09:12:47.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:47 smithi016 bash[75179]: cluster 2023-10-09T09:12:46.057169+0000 mgr.y (mgr.25421) 1064 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T09:12:47.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:47 smithi016 bash[75179]: audit 2023-10-09T09:12:46.230966+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.16:0/2732657833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:12:47.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:47 smithi016 bash[77892]: cluster 2023-10-09T09:12:46.057169+0000 mgr.y (mgr.25421) 1064 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T09:12:47.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:47 smithi016 bash[77892]: audit 2023-10-09T09:12:46.230966+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.16:0/2732657833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:12:47.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:47.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:47.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:47.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:47.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:47.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:47.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:47.455 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:47 smithi094 bash[59677]: cluster 2023-10-09T09:12:46.057169+0000 mgr.y (mgr.25421) 1064 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T09:12:47.456 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:47 smithi094 bash[59677]: audit 2023-10-09T09:12:46.230966+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.16:0/2732657833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:12:47.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:47.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:47.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:47.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:47.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:47.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:47.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:47.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:47.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:47.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:47.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:47.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:47.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:47.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:47.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:47.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:47.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 46233 46217 106.963 79.4375 0.00493846 0.00933272 2023-10-09T09:12:47.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:48.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:48.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:48.139 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:48.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:48.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:48.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:48.401 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:48.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:48.506 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:48.535 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:48.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:48.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:48.772 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:48.796 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:48.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:48.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:48.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:48.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:48.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:48.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 47503 47487 105.977 79.375 0.0139739 0.00942974 2023-10-09T09:12:49.040 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:49.075 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:49.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:49.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:49.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:49 smithi016 bash[75179]: cluster 2023-10-09T09:12:48.057741+0000 mgr.y (mgr.25421) 1065 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:12:49.190 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:49 smithi016 bash[77892]: cluster 2023-10-09T09:12:48.057741+0000 mgr.y (mgr.25421) 1065 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:12:49.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:49.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:49.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:49.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:49 smithi094 bash[59677]: cluster 2023-10-09T09:12:48.057741+0000 mgr.y (mgr.25421) 1065 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:12:49.360 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:49.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:49.465 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:49.495 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:49.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:49.601 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:49.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:49.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:49.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:49.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:49.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:49.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:49.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:49.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:49.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:49.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 48812 48796 105.144 81.8125 0.0155341 0.00950478 2023-10-09T09:12:50.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:50.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:50.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:50.143 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:50 smithi094 bash[59677]: audit 2023-10-09T09:12:49.245879+0000 mon.a (mon.0) 1425 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:50.143 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:50 smithi094 bash[59677]: audit 2023-10-09T09:12:49.313855+0000 mon.a (mon.0) 1426 : audit 1 from='client.? 172.21.15.16:0/3725979985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:12:50.151 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:12:50.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:50.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:50.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:50.281 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:50 smithi016 bash[75179]: audit 2023-10-09T09:12:49.245879+0000 mon.a (mon.0) 1425 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:50.281 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:50 smithi016 bash[75179]: audit 2023-10-09T09:12:49.313855+0000 mon.a (mon.0) 1426 : audit 1 from='client.? 172.21.15.16:0/3725979985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:12:50.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:50 smithi016 bash[77892]: audit 2023-10-09T09:12:49.245879+0000 mon.a (mon.0) 1425 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:50.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:50 smithi016 bash[77892]: audit 2023-10-09T09:12:49.313855+0000 mon.a (mon.0) 1426 : audit 1 from='client.? 172.21.15.16:0/3725979985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:12:50.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:50.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:50.436 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:50.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:50.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:50.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:50.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:50.696 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:50.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:50.826 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:50.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:50.930 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:50.961 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:50.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:12:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:50.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:50.989 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 50082 50066 104.284 79.375 0.0153071 0.00958328 2023-10-09T09:12:51.069 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:51.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:51.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:51.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:51.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:51.212 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:51 smithi094 bash[59677]: cluster 2023-10-09T09:12:50.058293+0000 mgr.y (mgr.25421) 1066 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:12:51.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:51 smithi094 bash[59677]: cluster 2023-10-09T09:12:50.131634+0000 mon.a (mon.0) 1427 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:12:51.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:51 smithi094 bash[59677]: cluster 2023-10-09T09:12:50.136458+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T09:12:51.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:51 smithi094 bash[59677]: audit 2023-10-09T09:12:50.137836+0000 mon.a (mon.0) 1428 : audit 1 from='client.? 172.21.15.16:0/3725979985' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:12:51.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:51 smithi094 bash[59677]: cluster 2023-10-09T09:12:50.137913+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-10-09T09:12:51.235 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:51.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[75179]: cluster 2023-10-09T09:12:50.058293+0000 mgr.y (mgr.25421) 1066 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:12:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[75179]: cluster 2023-10-09T09:12:50.131634+0000 mon.a (mon.0) 1427 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:12:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[75179]: cluster 2023-10-09T09:12:50.136458+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T09:12:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[75179]: audit 2023-10-09T09:12:50.137836+0000 mon.a (mon.0) 1428 : audit 1 from='client.? 172.21.15.16:0/3725979985' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:12:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[75179]: cluster 2023-10-09T09:12:50.137913+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-10-09T09:12:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[77892]: cluster 2023-10-09T09:12:50.058293+0000 mgr.y (mgr.25421) 1066 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:12:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[77892]: cluster 2023-10-09T09:12:50.131634+0000 mon.a (mon.0) 1427 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:12:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[77892]: cluster 2023-10-09T09:12:50.136458+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T09:12:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[77892]: audit 2023-10-09T09:12:50.137836+0000 mon.a (mon.0) 1428 : audit 1 from='client.? 172.21.15.16:0/3725979985' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:12:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:51 smithi016 bash[77892]: cluster 2023-10-09T09:12:50.137913+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-10-09T09:12:51.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:51.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:51.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:51.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:51.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:51.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:51.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:51.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:51.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:51.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:51.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:51.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:51.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:51.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:51.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:51.896 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:51.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:51.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 51366 51350 103.509 80.25 0.00537109 0.00965385 2023-10-09T09:12:52.005 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:52.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:52.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:52.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:52.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:52.272 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:52.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:52.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:52.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:52.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:52.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:52.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:52.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:52.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:52.712 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:52.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:52.715 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:52.715 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:52.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:52.842 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:52.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:52.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:52.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:52.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 52606 52590 102.696 77.5 0.00830916 0.00973074 2023-10-09T09:12:53.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:53.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:53.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:53.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:53.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:53 smithi016 bash[75179]: cluster 2023-10-09T09:12:52.059079+0000 mgr.y (mgr.25421) 1067 : cluster 0 pgmap v1143: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T09:12:53.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:53.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:53.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:53.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:53.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:53.372 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:53 smithi094 bash[59677]: cluster 2023-10-09T09:12:52.059079+0000 mgr.y (mgr.25421) 1067 : cluster 0 pgmap v1143: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T09:12:53.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:53.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:53.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:53.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:53.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:53 smithi016 bash[77892]: cluster 2023-10-09T09:12:52.059079+0000 mgr.y (mgr.25421) 1067 : cluster 0 pgmap v1143: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T09:12:53.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:53.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:53.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:53.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:12:53.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:53.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:53.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:53.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:53.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:53.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:53.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:53.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:53.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:53.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:53.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:53.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:53.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 53840 53824 101.921 77.125 0.0120127 0.00980589 2023-10-09T09:12:54.067 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:54.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:54.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:54.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:54.238 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:54.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:54.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:54.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:54.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:54.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:54.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:54.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:54.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:54.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:54.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:54.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:54.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:54.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:54.794 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:54.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:54.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:54.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:54.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:54.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:54.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 55093 55077 101.226 78.3125 0.00658834 0.00987267 2023-10-09T09:12:55.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:55.063 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:55.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:55.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:55.167 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:55.218 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:55.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:55.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:55.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:55 smithi016 bash[75179]: cluster 2023-10-09T09:12:54.060387+0000 mgr.y (mgr.25421) 1068 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:12:55.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:55 smithi016 bash[77892]: cluster 2023-10-09T09:12:54.060387+0000 mgr.y (mgr.25421) 1068 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:12:55.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:55.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:55.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:55.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:55.476 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:55 smithi094 bash[59677]: cluster 2023-10-09T09:12:54.060387+0000 mgr.y (mgr.25421) 1068 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:12:55.499 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:55.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:55.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:55.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:55.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:55.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:55.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:55.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:55.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:55.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:55.784 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:55.811 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:12:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:12:55] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.33.4" 2023-10-09T09:12:55.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:55.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:55.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:55.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 56286 56270 100.465 74.5625 0.0112008 0.00994803 2023-10-09T09:12:55.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:56.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:56.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:56.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:56.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:56.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:56.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:56.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:56.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:56.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:56.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:56.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:56.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:56.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:56.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:56.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:56.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:56.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:56.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 57370 57354 99.5556 67.75 0.0104332 0.0100382 2023-10-09T09:12:57.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:57.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:57.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:57.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:57.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:57.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:57 smithi016 bash[75179]: cluster 2023-10-09T09:12:56.061062+0000 mgr.y (mgr.25421) 1069 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:12:57.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:57 smithi016 bash[77892]: cluster 2023-10-09T09:12:56.061062+0000 mgr.y (mgr.25421) 1069 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:12:57.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:12:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:57.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:12:57.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:57.384 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:57.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:57.488 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:57 smithi094 bash[59677]: cluster 2023-10-09T09:12:56.061062+0000 mgr.y (mgr.25421) 1069 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:12:57.517 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:57.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:57.624 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:57.652 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:57.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:57.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:57.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:57.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:12:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:57.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:12:57.885 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:57.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:57.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:57.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:57.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 58525 58509 98.8158 72.1875 0.0117435 0.0101141 2023-10-09T09:12:58.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.039 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:58.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:58.167 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:58.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:58.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.299 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:58.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:58.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:58.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:58.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:58.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:12:58.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:12:58.677 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:58.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:58.810 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.821 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:12:58.835 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:58.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:58.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:58.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 59717 59701 98.1758 74.5 0.0135407 0.0101799 2023-10-09T09:12:59.072 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:59.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:59.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:59.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:59.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:59.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:12:59.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:59.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:12:59.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:12:59 smithi094 bash[59677]: cluster 2023-10-09T09:12:58.061522+0000 mgr.y (mgr.25421) 1070 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:12:59.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:59.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:12:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:59.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:12:59.470 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:59.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:12:59.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:59.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:12:59.514 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:12:59 smithi016 bash[75179]: cluster 2023-10-09T09:12:58.061522+0000 mgr.y (mgr.25421) 1070 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:12:59.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:12:59 smithi016 bash[77892]: cluster 2023-10-09T09:12:58.061522+0000 mgr.y (mgr.25421) 1070 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:12:59.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:59.623 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:59.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:59.726 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:59.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:12:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:59.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:12:59.879 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:59.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:12:59.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:12:59.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:12:59.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 60853 60837 97.4788 71 0.0129225 0.0102522 2023-10-09T09:13:00.012 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:00.041 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:00.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:00.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:00.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:00.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:00.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:00.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:00.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:00.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:00.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:00.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:00.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:00.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:00.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:00.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:00.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:00.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:00.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:13:00.990712+0000 min lat: 0.00132275 max lat: 0.323328 avg lat: 0.0103242 2023-10-09T09:13:00.990 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:13:00.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 61980 61964 96.8027 70.4375 0.00743445 0.0103242 2023-10-09T09:13:00.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:00 smithi094 bash[59677]: cluster 2023-10-09T09:13:00.062388+0000 mgr.y (mgr.25421) 1071 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T09:13:01.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:01.096 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:01.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:01.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:01.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:01.232 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:01.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:01.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:00 smithi016 bash[75179]: cluster 2023-10-09T09:13:00.062388+0000 mgr.y (mgr.25421) 1071 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T09:13:01.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:00 smithi016 bash[77892]: cluster 2023-10-09T09:13:00.062388+0000 mgr.y (mgr.25421) 1071 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T09:13:01.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:01.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:01.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:01.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:01.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:01.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:01.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:01.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:01.613 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:01.648 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:01.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:01.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:01.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:01.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:01.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:01.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:01.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:01.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:01.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:01.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 63130 63114 96.1945 71.875 0.0109105 0.0103891 2023-10-09T09:13:02.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:02.063 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:02.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:02.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:02.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:02.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:02.297 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:02.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:02.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:02.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:02.463 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:02.565 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:02.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:02.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:02.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:02.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:02.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:02.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:02.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:02.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:02.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:02.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 64144 64128 95.4131 63.375 0.0234424 0.0104755 2023-10-09T09:13:02.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:02.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:03.099 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:03.132 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 10 pgp_num to 16 2023-10-09T09:13:03.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:03 smithi016 bash[75179]: cluster 2023-10-09T09:13:02.063179+0000 mgr.y (mgr.25421) 1072 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T09:13:03.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:03 smithi016 bash[75179]: audit 2023-10-09T09:13:02.906692+0000 mon.a (mon.0) 1430 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:03.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:03 smithi016 bash[75179]: audit 2023-10-09T09:13:02.971209+0000 mon.a (mon.0) 1431 : audit 1 from='client.? 172.21.15.16:0/180225209' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:13:03.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:03 smithi016 bash[77892]: cluster 2023-10-09T09:13:02.063179+0000 mgr.y (mgr.25421) 1072 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T09:13:03.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:03 smithi016 bash[77892]: audit 2023-10-09T09:13:02.906692+0000 mon.a (mon.0) 1430 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:03.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:03 smithi016 bash[77892]: audit 2023-10-09T09:13:02.971209+0000 mon.a (mon.0) 1431 : audit 1 from='client.? 172.21.15.16:0/180225209' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:13:03.139 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:03.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:03.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:03.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:03.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:03.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:03.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:03.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:03.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:03.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:03.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:03.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:03 smithi094 bash[59677]: cluster 2023-10-09T09:13:02.063179+0000 mgr.y (mgr.25421) 1072 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T09:13:03.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:03 smithi094 bash[59677]: audit 2023-10-09T09:13:02.906692+0000 mon.a (mon.0) 1430 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:03.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:03 smithi094 bash[59677]: audit 2023-10-09T09:13:02.971209+0000 mon.a (mon.0) 1431 : audit 1 from='client.? 172.21.15.16:0/180225209' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:13:03.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:03.553 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:03.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:03.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:03.661 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:03.687 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:03.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:03.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:03.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:03.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:03.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:03.921 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:03.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:03.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:03.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 65089 65073 94.5677 59.0625 0.00498506 0.0105658 2023-10-09T09:13:04.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:04.082 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:04.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:04.142 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[75179]: audit 2023-10-09T09:13:03.125537+0000 mon.a (mon.0) 1432 : audit 1 from='client.? 172.21.15.16:0/180225209' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:13:04.142 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[75179]: cluster 2023-10-09T09:13:03.125894+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e254: 8 total, 8 up, 7 in 2023-10-09T09:13:04.142 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[75179]: cluster 2023-10-09T09:13:03.129565+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.142 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[75179]: cluster 2023-10-09T09:13:03.129829+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.142 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[75179]: cluster 2023-10-09T09:13:03.132762+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.142 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[75179]: cluster 2023-10-09T09:13:03.137827+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:04.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:04.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:04 smithi094 bash[59677]: audit 2023-10-09T09:13:03.125537+0000 mon.a (mon.0) 1432 : audit 1 from='client.? 172.21.15.16:0/180225209' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:13:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:04 smithi094 bash[59677]: cluster 2023-10-09T09:13:03.125894+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e254: 8 total, 8 up, 7 in 2023-10-09T09:13:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:04 smithi094 bash[59677]: cluster 2023-10-09T09:13:03.129565+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:04 smithi094 bash[59677]: cluster 2023-10-09T09:13:03.129829+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:04 smithi094 bash[59677]: cluster 2023-10-09T09:13:03.132762+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:04 smithi094 bash[59677]: cluster 2023-10-09T09:13:03.137827+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:04.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:04.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:04.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[77892]: audit 2023-10-09T09:13:03.125537+0000 mon.a (mon.0) 1432 : audit 1 from='client.? 172.21.15.16:0/180225209' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:13:04.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[77892]: cluster 2023-10-09T09:13:03.125894+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e254: 8 total, 8 up, 7 in 2023-10-09T09:13:04.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[77892]: cluster 2023-10-09T09:13:03.129565+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[77892]: cluster 2023-10-09T09:13:03.129829+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[77892]: cluster 2023-10-09T09:13:03.132762+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:04 smithi016 bash[77892]: cluster 2023-10-09T09:13:03.137827+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T09:13:04.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:04.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:04.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:04.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:04.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:04.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:04.674 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:13:04.674 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:13:04.675 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T09:13:04.675 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T09:13:04.675 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:13:04.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:04.760 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:04.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:04.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:04.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:04.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:04.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:04.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 66153 66137 93.9297 66.5 0.0143097 0.0106401 2023-10-09T09:13:05.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:05.041 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:05.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:05.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:05.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:05.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:05 smithi094 bash[59677]: cluster 2023-10-09T09:13:04.064429+0000 mgr.y (mgr.25421) 1073 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:13:05.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:05 smithi094 bash[59677]: audit 2023-10-09T09:13:04.252485+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:05.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:05 smithi094 bash[59677]: audit 2023-10-09T09:13:04.254752+0000 mon.a (mon.0) 1435 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:05.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:05.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:05.416 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:05.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:05.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:05.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:05 smithi016 bash[75179]: cluster 2023-10-09T09:13:04.064429+0000 mgr.y (mgr.25421) 1073 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:13:05.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:05 smithi016 bash[75179]: audit 2023-10-09T09:13:04.252485+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:05.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:05 smithi016 bash[75179]: audit 2023-10-09T09:13:04.254752+0000 mon.a (mon.0) 1435 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:05.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:05 smithi016 bash[77892]: cluster 2023-10-09T09:13:04.064429+0000 mgr.y (mgr.25421) 1073 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:13:05.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:05 smithi016 bash[77892]: audit 2023-10-09T09:13:04.252485+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:05.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:05 smithi016 bash[77892]: audit 2023-10-09T09:13:04.254752+0000 mon.a (mon.0) 1435 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:05.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:05.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:05.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:05.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:05.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:05.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:05.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:05.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:05.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:05.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:05.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:05.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 67239 67223 93.3506 67.875 0.00742226 0.0107063 2023-10-09T09:13:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:13:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:13:05] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.33.4" 2023-10-09T09:13:06.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:06.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:06.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:06.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:06.267 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:06.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:06.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:06.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:06.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:06.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:06.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:06.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:06.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:06.670 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:06.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:06.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:06.775 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:06.803 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:06.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:06.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:06.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:06.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:06.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:06.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:06.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 68170 68154 92.5861 58.1875 0.0105159 0.0107953 2023-10-09T09:13:07.036 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:07.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:07.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:07.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:07.164 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:07.195 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:07.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:07.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:07.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:07.307 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:07 smithi094 bash[59677]: cluster 2023-10-09T09:13:06.065126+0000 mgr.y (mgr.25421) 1074 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:13:07.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:07.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:07.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:07.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:07.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:07 smithi016 bash[75179]: cluster 2023-10-09T09:13:06.065126+0000 mgr.y (mgr.25421) 1074 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:13:07.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:07 smithi016 bash[77892]: cluster 2023-10-09T09:13:06.065126+0000 mgr.y (mgr.25421) 1074 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:13:07.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:07.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:07.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:07.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:07.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:07.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:07.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:07.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:07.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:07.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:07.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:07.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:07.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:07.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 69241 69225 92.0402 66.9375 0.00585394 0.0108586 2023-10-09T09:13:07.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:08.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:08.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:08.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:08.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:08.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:08.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:08.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:08.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:08.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:08.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:08.436 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:08.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:08.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:08.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:08.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:08.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:08.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:08.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:08.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:08.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:08.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:08.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:08.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 70215 70199 91.3909 60.875 0.0115273 0.0109352 2023-10-09T09:13:09.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:09.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:09.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:09.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:09.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:09.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:09.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:13:09.260 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:09.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:09.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:09.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:09.416 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:09.452 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:09.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:09.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:09.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:09 smithi016 bash[75179]: cluster 2023-10-09T09:13:08.065498+0000 mgr.y (mgr.25421) 1075 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:13:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:09 smithi016 bash[77892]: cluster 2023-10-09T09:13:08.065498+0000 mgr.y (mgr.25421) 1075 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:13:09.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:09.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:09 smithi094 bash[59677]: cluster 2023-10-09T09:13:08.065498+0000 mgr.y (mgr.25421) 1075 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:13:09.618 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:09.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:09.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:09.763 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:09.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:09.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:09.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:09.937 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:09.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:09.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 71052 71036 90.5933 52.3125 0.0118791 0.0110323 2023-10-09T09:13:10.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:10.069 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:10.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:10.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:10.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:10.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:10.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:10.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:10.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:10.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:10.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:10.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:10.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:10.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:10.580 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:13:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:10.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:10.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:10.626 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:13:10.674 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:13:10.710 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:13:10.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:10.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:10.748 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:13:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:10.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:10.786 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:13:10.823 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:13:10.850 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:10.855 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:13:10.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:10.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:10.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 72006 71990 89.9739 59.625 0.016189 0.0111079 2023-10-09T09:13:10.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:11.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:11.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:11.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:11.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:11.137 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:11 smithi094 bash[59677]: audit 2023-10-09T09:13:10.013997+0000 mon.a (mon.0) 1436 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:11.137 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:11 smithi094 bash[59677]: audit 2023-10-09T09:13:10.020666+0000 mon.a (mon.0) 1437 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:11.137 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:11 smithi094 bash[59677]: cluster 2023-10-09T09:13:10.066110+0000 mgr.y (mgr.25421) 1076 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:13:11.137 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:11 smithi094 bash[59677]: audit 2023-10-09T09:13:10.531698+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.16:0/153007840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:11.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:11.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:11.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:11.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[75179]: audit 2023-10-09T09:13:10.013997+0000 mon.a (mon.0) 1436 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:11.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[75179]: audit 2023-10-09T09:13:10.020666+0000 mon.a (mon.0) 1437 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:11.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[75179]: cluster 2023-10-09T09:13:10.066110+0000 mgr.y (mgr.25421) 1076 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:13:11.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[75179]: audit 2023-10-09T09:13:10.531698+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.16:0/153007840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:11.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[77892]: audit 2023-10-09T09:13:10.013997+0000 mon.a (mon.0) 1436 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:11.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[77892]: audit 2023-10-09T09:13:10.020666+0000 mon.a (mon.0) 1437 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:11.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[77892]: cluster 2023-10-09T09:13:10.066110+0000 mgr.y (mgr.25421) 1076 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:13:11.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:11 smithi016 bash[77892]: audit 2023-10-09T09:13:10.531698+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.16:0/153007840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:11.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:11.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:11.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:11.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:11.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:11.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:11.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:11.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:11.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:11.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:11.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:11.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:11.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:11.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:11.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:11.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:11.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:11.991 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 73080 73064 89.5258 67.125 0.0136846 0.0111635 2023-10-09T09:13:12.016 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:12.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:12.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:12.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:12.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:12.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:12.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:12.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:12.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:12.603 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:13:12.604 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:13:12.604 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T09:13:12.604 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:13:12.604 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:13:12.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T09:13:12.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:12.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:12.797 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:12.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:12.944 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:12.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:12.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:12.992 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 74069 74053 88.9927 61.8125 0.00713411 0.0112311 2023-10-09T09:13:13.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:13.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:13.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:13.140 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:13:13.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:13.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:13 smithi094 bash[59677]: cluster 2023-10-09T09:13:12.066740+0000 mgr.y (mgr.25421) 1077 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:13:13.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:13 smithi094 bash[59677]: audit 2023-10-09T09:13:12.767154+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.16:0/3714062179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:13.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:13 smithi094 bash[59677]: audit 2023-10-09T09:13:12.767895+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:13.235 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:13.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:13.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:13.348 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:13 smithi016 bash[75179]: cluster 2023-10-09T09:13:12.066740+0000 mgr.y (mgr.25421) 1077 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:13:13.348 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:13 smithi016 bash[75179]: audit 2023-10-09T09:13:12.767154+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.16:0/3714062179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:13.348 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:13 smithi016 bash[75179]: audit 2023-10-09T09:13:12.767895+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:13.348 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:13 smithi016 bash[77892]: cluster 2023-10-09T09:13:12.066740+0000 mgr.y (mgr.25421) 1077 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:13:13.348 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:13 smithi016 bash[77892]: audit 2023-10-09T09:13:12.767154+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.16:0/3714062179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:13.348 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:13 smithi016 bash[77892]: audit 2023-10-09T09:13:12.767895+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:13.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:13.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:13.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:13.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:13.535 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:13.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:13.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:13.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:13.671 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:13.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:13.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:13.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:13.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:13.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:13.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:13.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:13.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:13.992 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 75103 75087 88.5329 64.625 0.0153792 0.0112879 2023-10-09T09:13:14.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:14.081 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:14.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:14.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.120202+0000 mon.a (mon.0) 1439 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: audit 2023-10-09T09:13:13.126024+0000 mon.a (mon.0) 1440 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.126093+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.128063+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.128716+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.129663+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.129890+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.129907+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.138 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.132215+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.139 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.132840+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.139 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[77892]: cluster 2023-10-09T09:13:13.135058+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.184 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:14.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:14.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:14.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:14.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:14.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:14.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:14.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:14.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:14.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.120202+0000 mon.a (mon.0) 1439 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:13:14.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: audit 2023-10-09T09:13:13.126024+0000 mon.a (mon.0) 1440 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:13:14.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.126093+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.128063+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.128716+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.129663+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.129890+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.129907+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.132215+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.132840+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:14 smithi016 bash[75179]: cluster 2023-10-09T09:13:13.135058+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:14.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:14.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:14.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.120202+0000 mon.a (mon.0) 1439 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:13:14.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: audit 2023-10-09T09:13:13.126024+0000 mon.a (mon.0) 1440 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.126093+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.128063+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.128716+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.129663+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.129890+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.129907+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.132215+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.132840+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:14 smithi094 bash[59677]: cluster 2023-10-09T09:13:13.135058+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T09:13:14.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:14.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:14.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:14.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:14.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:13:14.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:14.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:14.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:14.859 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:14.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:14.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:14.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:14.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 76095 76079 88.035 62 0.00832522 0.0113537 2023-10-09T09:13:15.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:15.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:15.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:15.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:15 smithi094 bash[59677]: cluster 2023-10-09T09:13:14.067237+0000 mgr.y (mgr.25421) 1078 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:13:15.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:15.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:15.274 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:15 smithi016 bash[75179]: cluster 2023-10-09T09:13:14.067237+0000 mgr.y (mgr.25421) 1078 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:13:15.274 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:15 smithi016 bash[77892]: cluster 2023-10-09T09:13:14.067237+0000 mgr.y (mgr.25421) 1078 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:13:15.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:15.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:15.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:15.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:15.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:15.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:15.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:15.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:15.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:13:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:13:15] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.33.4" 2023-10-09T09:13:15.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:15.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:15.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:15.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:15.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:15.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:15.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:15.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 77011 76995 87.4753 57.25 0.0171206 0.0114255 2023-10-09T09:13:16.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:16.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:16.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:16.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:16.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:16.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:16.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:16.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:13:16.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:16.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:16.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:16.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:16.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:16.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:16.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:16.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:16.714 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:16.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:16.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:16.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:16.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:16.866 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:16.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:16.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:16.978 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:16.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 77946 77930 86.9567 58.4375 0.0243096 0.0114934 2023-10-09T09:13:17.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:17.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:17.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:17.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:17 smithi094 bash[59677]: cluster 2023-10-09T09:13:16.067972+0000 mgr.y (mgr.25421) 1079 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:13:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:17 smithi094 bash[59677]: audit 2023-10-09T09:13:16.196248+0000 mon.a (mon.0) 1442 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:17 smithi094 bash[59677]: audit 2023-10-09T09:13:16.203286+0000 mon.a (mon.0) 1443 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:17 smithi094 bash[59677]: audit 2023-10-09T09:13:16.887435+0000 mon.a (mon.0) 1444 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:17 smithi094 bash[59677]: audit 2023-10-09T09:13:16.888860+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:17.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:17 smithi094 bash[59677]: audit 2023-10-09T09:13:16.901066+0000 mon.a (mon.0) 1446 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:17.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:17.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:17.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:17.468 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:13:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[75179]: cluster 2023-10-09T09:13:16.067972+0000 mgr.y (mgr.25421) 1079 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:13:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[75179]: audit 2023-10-09T09:13:16.196248+0000 mon.a (mon.0) 1442 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[75179]: audit 2023-10-09T09:13:16.203286+0000 mon.a (mon.0) 1443 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[75179]: audit 2023-10-09T09:13:16.887435+0000 mon.a (mon.0) 1444 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[75179]: audit 2023-10-09T09:13:16.888860+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:17.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[75179]: audit 2023-10-09T09:13:16.901066+0000 mon.a (mon.0) 1446 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[77892]: cluster 2023-10-09T09:13:16.067972+0000 mgr.y (mgr.25421) 1079 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:13:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[77892]: audit 2023-10-09T09:13:16.196248+0000 mon.a (mon.0) 1442 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[77892]: audit 2023-10-09T09:13:16.203286+0000 mon.a (mon.0) 1443 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[77892]: audit 2023-10-09T09:13:16.887435+0000 mon.a (mon.0) 1444 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[77892]: audit 2023-10-09T09:13:16.888860+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:17 smithi016 bash[77892]: audit 2023-10-09T09:13:16.901066+0000 mon.a (mon.0) 1446 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:13:17.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:17.537 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:17.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:17.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:17.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:17.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:17.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:17.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:17.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:17.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:17.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:17.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 78925 78909 86.5046 61.1875 0.00703749 0.0115539 2023-10-09T09:13:18.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:18.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:18.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:18.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:18.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:18.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:18.456 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:18.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:18.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:18.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:18.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:18.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:18.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:18.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:18.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:18.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:18.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:18.896 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:18.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:18.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:18.905 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:18.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 79941 79925 86.1079 63.5 0.0160502 0.0116073 2023-10-09T09:13:18.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:19.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:19.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:19.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:19.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:19.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:19.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:19.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:19.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:19.423 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:19 smithi094 bash[59677]: cluster 2023-10-09T09:13:18.068387+0000 mgr.y (mgr.25421) 1080 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:13:19.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:19.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:19.571 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:19.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:19.600 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:19 smithi016 bash[75179]: cluster 2023-10-09T09:13:18.068387+0000 mgr.y (mgr.25421) 1080 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:13:19.600 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:19 smithi016 bash[77892]: cluster 2023-10-09T09:13:18.068387+0000 mgr.y (mgr.25421) 1080 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:13:19.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:19.693 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:19.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:19.841 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:19.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:19.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:19.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:19.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:19.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:19.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 80858 80842 85.6198 57.3125 0.0166356 0.0116733 2023-10-09T09:13:20.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:20.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:20.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:20.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:20.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:20 smithi094 bash[59677]: audit 2023-10-09T09:13:19.246436+0000 mon.a (mon.0) 1447 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:20.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:20.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:20 smithi016 bash[75179]: audit 2023-10-09T09:13:19.246436+0000 mon.a (mon.0) 1447 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:20.238 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:20 smithi016 bash[77892]: audit 2023-10-09T09:13:19.246436+0000 mon.a (mon.0) 1447 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:20.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:20.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:20.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:20.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:20.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:20.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:20.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:20.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:20.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:20.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:20.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:20.766 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:20.797 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:20.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:20.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:20.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:20.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:20.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:13:20.996527+0000 min lat: 0.00132275 max lat: 0.323328 avg lat: 0.0117275 2023-10-09T09:13:20.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:13:20.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 81852 81836 85.2282 62.125 0.0123546 0.0117275 2023-10-09T09:13:21.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:21.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:21.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:21.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:21.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:21.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:21.329 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:21.358 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:21.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:21.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:21.462 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:21.471 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:21 smithi094 bash[59677]: cluster 2023-10-09T09:13:20.069001+0000 mgr.y (mgr.25421) 1081 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T09:13:21.497 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:21.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:21 smithi016 bash[75179]: cluster 2023-10-09T09:13:20.069001+0000 mgr.y (mgr.25421) 1081 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T09:13:21.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:21 smithi016 bash[77892]: cluster 2023-10-09T09:13:20.069001+0000 mgr.y (mgr.25421) 1081 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T09:13:21.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:21.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:21.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:21.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:21.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:21.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:21.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:21.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:21.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:21.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:21.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:21.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:21.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:21.996 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 82842 82826 84.8452 61.875 0.00754285 0.0117799 2023-10-09T09:13:22.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:22.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:22.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:22.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:22.054 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:22.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:22.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:22.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:22.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:22.322 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:22.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:22.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:22.610 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:22.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:22.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:22.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:22.745 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:22.777 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:22.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:22.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:22.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:22.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:22.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 83816 83800 84.4586 60.875 0.0167442 0.0118346 2023-10-09T09:13:23.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:23.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:23.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:23.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:23.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:23.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:23.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:23.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:23.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:23.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:23 smithi094 bash[59677]: cluster 2023-10-09T09:13:22.070006+0000 mgr.y (mgr.25421) 1082 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T09:13:23.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:23 smithi016 bash[75179]: cluster 2023-10-09T09:13:22.070006+0000 mgr.y (mgr.25421) 1082 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T09:13:23.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:23 smithi016 bash[77892]: cluster 2023-10-09T09:13:22.070006+0000 mgr.y (mgr.25421) 1082 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T09:13:23.530 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:23.558 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:23.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:23.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:23.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:23.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:23.838 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:23.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:23.944 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:23.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:23.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:23.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:23.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 84744 84728 84.0386 58 0.0356427 0.0118935 2023-10-09T09:13:24.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:24.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:24.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:24.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:24.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:24.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:24.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:24.349 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:24.452 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:24.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:24.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:24.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:24.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:24.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:24.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:24.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:24.729 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:24.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:24.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:24.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:24.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:24.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:24.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:24.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:24.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:24.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:24.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 85561 85545 83.5233 51.0625 0.0208176 0.0119666 2023-10-09T09:13:25.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:25.123 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:25.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:25.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:25.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:25.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:25.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:25.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:25.363 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:25 smithi094 bash[59677]: cluster 2023-10-09T09:13:24.070511+0000 mgr.y (mgr.25421) 1083 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T09:13:25.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:25.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:25.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:25.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:25 smithi016 bash[75179]: cluster 2023-10-09T09:13:24.070511+0000 mgr.y (mgr.25421) 1083 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T09:13:25.488 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:25 smithi016 bash[77892]: cluster 2023-10-09T09:13:24.070511+0000 mgr.y (mgr.25421) 1083 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T09:13:25.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:25.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:25.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:25.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:25.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:25.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:25.739 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:13:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:13:25] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.33.4" 2023-10-09T09:13:25.751 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:25.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:25.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:25.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:25.880 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:25.916 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:25.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:25.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 86485 86469 83.1267 57.75 0.0239889 0.0120242 2023-10-09T09:13:26.023 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:26.058 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:26.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:26.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:26.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:26.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:26.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:26.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:26.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:26.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:26.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:26.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:26.663 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:26.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:26.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:26.742 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:13:26.742 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:13:26.742 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:13:26.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:26.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:26.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:26.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:26.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:26.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:26.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 87291 87275 82.6305 50.375 0.0089535 0.0120947 2023-10-09T09:13:27.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:27.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:27.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:27.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:27.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:27.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:27.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:27 smithi016 bash[75179]: cluster 2023-10-09T09:13:26.071238+0000 mgr.y (mgr.25421) 1084 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:13:27.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:27 smithi016 bash[77892]: cluster 2023-10-09T09:13:26.071238+0000 mgr.y (mgr.25421) 1084 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:13:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:27.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:27.301 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:13:27.301 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:13:27.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:27.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:27.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:27.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:27 smithi094 bash[59677]: cluster 2023-10-09T09:13:26.071238+0000 mgr.y (mgr.25421) 1084 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:13:27.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:27.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:27.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:27.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:27.645 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:27.674 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:27.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:27.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:27.826 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:27.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:27.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:27.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:27.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:27.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 88244 88228 82.2861 59.5625 0.0140231 0.0121468 2023-10-09T09:13:28.064 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:28.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:28.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:28.215 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:28.240 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:28.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:28.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:28.380 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:28.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:28.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:28.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:28.517 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:28.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:28.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:28.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:28.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:28.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:28.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:28.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:28.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:28.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:28.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:28.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 89069 89053 81.8343 51.5625 0.014252 0.0122137 2023-10-09T09:13:29.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:29.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:29.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:29.145 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:29.171 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:29.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:29.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:29.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:29.434 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:29 smithi016 bash[75179]: cluster 2023-10-09T09:13:28.071625+0000 mgr.y (mgr.25421) 1085 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T09:13:29.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:29 smithi016 bash[77892]: cluster 2023-10-09T09:13:28.071625+0000 mgr.y (mgr.25421) 1085 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T09:13:29.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:29.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:29.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:29 smithi094 bash[59677]: cluster 2023-10-09T09:13:28.071625+0000 mgr.y (mgr.25421) 1085 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T09:13:29.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:29.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:29.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:29.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:29.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:29.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:29.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:29.993 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:29.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 89982 89966 81.4752 57.0625 0.0190872 0.0122668 2023-10-09T09:13:29.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:29.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:30.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:30.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:30.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:30.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:30.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:30.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:30.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:30.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:30.371 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:30.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:30.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:30.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:30.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:30.549 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:30.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:30.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:30.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:30.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:30.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:30.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:30.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:30.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:30.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:30.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:30.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:30.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:30 smithi094 bash[59677]: cluster 2023-10-09T09:13:30.072228+0000 mgr.y (mgr.25421) 1086 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T09:13:30.997 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 90918 90902 81.147 58.5 0.0120939 0.0123176 2023-10-09T09:13:31.050 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:31.083 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:31.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:31.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:31.227 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:31.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:31.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:31.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:31.338 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:30 smithi016 bash[75179]: cluster 2023-10-09T09:13:30.072228+0000 mgr.y (mgr.25421) 1086 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T09:13:31.338 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:30 smithi016 bash[77892]: cluster 2023-10-09T09:13:30.072228+0000 mgr.y (mgr.25421) 1086 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T09:13:31.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:31.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:31.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:31.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:31.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:31.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:31.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:31.654 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:31.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:31.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:13:31.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:31.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:31.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:31.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:31.921 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:31.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:31.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:31.999 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 91748 91732 80.7324 51.875 0.0161676 0.0123813 2023-10-09T09:13:32.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:32.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:32.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:32.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:32.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:32.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:32.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:32.264 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:13:32.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:32.298 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:13:32.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:32.334 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:13:32.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:32.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:32.371 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:13:32.414 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:13:32.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:32.455 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:13:32.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:32.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:32.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:32.483 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:13:32.529 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:13:32.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.532 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.532 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.532 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:13:32.584 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:32.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:32.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:32.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:32.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:32.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:32.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:32.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:32.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:32.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:32.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:32.999 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 92599 92583 80.3498 53.1875 0.0124006 0.0124399 2023-10-09T09:13:33.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:33.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:33.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:33.104 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:13:33.105 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:13:33.105 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T09:13:33.105 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.711948 2023-10-09T09:13:33.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0.7119484439158364 2023-10-09T09:13:33.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:33.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:33.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:33.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:33 smithi016 bash[75179]: cluster 2023-10-09T09:13:32.072823+0000 mgr.y (mgr.25421) 1087 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:13:33.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:33 smithi016 bash[75179]: audit 2023-10-09T09:13:32.217094+0000 mon.a (mon.0) 1448 : audit 0 from='client.? 172.21.15.16:0/330680430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:33.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:33 smithi016 bash[77892]: cluster 2023-10-09T09:13:32.072823+0000 mgr.y (mgr.25421) 1087 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:13:33.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:33 smithi016 bash[77892]: audit 2023-10-09T09:13:32.217094+0000 mon.a (mon.0) 1448 : audit 0 from='client.? 172.21.15.16:0/330680430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:33.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:33.295 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:33 smithi094 bash[59677]: cluster 2023-10-09T09:13:32.072823+0000 mgr.y (mgr.25421) 1087 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:13:33.295 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:33 smithi094 bash[59677]: audit 2023-10-09T09:13:32.217094+0000 mon.a (mon.0) 1448 : audit 0 from='client.? 172.21.15.16:0/330680430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:33.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:33.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:33.416 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:33.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:33.553 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:33.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:33.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:33.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:33.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:33.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:33.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:33.855 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:33.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:33.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:33.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:33.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:33.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:34.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 93378 93362 79.916 48.6875 0.020488 0.0125074 2023-10-09T09:13:34.099 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:34.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:34.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:34.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:34.236 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:34.261 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:34.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:34.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:34.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:34.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:34.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:34.531 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:34.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:34.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:34.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:34.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:34.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:34.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:34.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:34.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:34.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:34.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:34.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:34.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:35.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 94199 94183 79.5295 51.3125 0.0172347 0.0125679 2023-10-09T09:13:35.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:35.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:35.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:35.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:35.206 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:35.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:35.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:35.321 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:35 smithi016 bash[75179]: cluster 2023-10-09T09:13:34.073474+0000 mgr.y (mgr.25421) 1088 : cluster 0 pgmap v1166: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T09:13:35.321 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:35 smithi016 bash[75179]: audit 2023-10-09T09:13:34.246909+0000 mon.a (mon.0) 1449 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:35.322 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:35 smithi016 bash[77892]: cluster 2023-10-09T09:13:34.073474+0000 mgr.y (mgr.25421) 1088 : cluster 0 pgmap v1166: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T09:13:35.322 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:35 smithi016 bash[77892]: audit 2023-10-09T09:13:34.246909+0000 mon.a (mon.0) 1449 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:35.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:35.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:35.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:35.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:35.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:35 smithi094 bash[59677]: cluster 2023-10-09T09:13:34.073474+0000 mgr.y (mgr.25421) 1088 : cluster 0 pgmap v1166: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T09:13:35.459 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:35 smithi094 bash[59677]: audit 2023-10-09T09:13:34.246909+0000 mon.a (mon.0) 1449 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:35.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:35.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:35.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:35.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:35.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:35.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:35.759 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:35.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:35.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:35.867 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:35.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:35.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:35.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:35.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:13:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:13:35] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.33.4" 2023-10-09T09:13:36.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 95046 95030 79.1749 52.9375 0.0175434 0.012624 2023-10-09T09:13:36.011 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:36.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:36.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:36.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:36.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:36.165 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:13:36.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:36.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:36.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:36.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:36.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:36.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:36.407 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:36 smithi016 bash[77892]: audit 2023-10-09T09:13:35.541480+0000 mon.a (mon.0) 1450 : audit 1 from='client.? 172.21.15.16:0/2639494339' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:13:36.408 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:36 smithi016 bash[75179]: audit 2023-10-09T09:13:35.541480+0000 mon.a (mon.0) 1450 : audit 1 from='client.? 172.21.15.16:0/2639494339' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:13:36.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:36.437 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:36 smithi094 bash[59677]: audit 2023-10-09T09:13:35.541480+0000 mon.a (mon.0) 1450 : audit 1 from='client.? 172.21.15.16:0/2639494339' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:13:36.459 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:36.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:36.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:36.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:36.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:36.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:36.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:36.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:36.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:36.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:36.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:36.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:36.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:13:36.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:37.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 95920 95904 78.8518 54.625 0.0301657 0.0126766 2023-10-09T09:13:37.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:37.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:37.168 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:37.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:37.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:37.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:37.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:37.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:37.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[75179]: cluster 2023-10-09T09:13:36.074304+0000 mgr.y (mgr.25421) 1089 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T09:13:37.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[75179]: cluster 2023-10-09T09:13:36.137019+0000 mon.a (mon.0) 1451 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:13:37.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[75179]: cluster 2023-10-09T09:13:36.153276+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T09:13:37.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[75179]: cluster 2023-10-09T09:13:36.156729+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T09:13:37.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[75179]: audit 2023-10-09T09:13:36.156990+0000 mon.a (mon.0) 1452 : audit 1 from='client.? 172.21.15.16:0/2639494339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:13:37.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[75179]: cluster 2023-10-09T09:13:36.157430+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-09T09:13:37.421 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[77892]: cluster 2023-10-09T09:13:36.074304+0000 mgr.y (mgr.25421) 1089 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T09:13:37.421 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[77892]: cluster 2023-10-09T09:13:36.137019+0000 mon.a (mon.0) 1451 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:13:37.421 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[77892]: cluster 2023-10-09T09:13:36.153276+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T09:13:37.421 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[77892]: cluster 2023-10-09T09:13:36.156729+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T09:13:37.421 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[77892]: audit 2023-10-09T09:13:36.156990+0000 mon.a (mon.0) 1452 : audit 1 from='client.? 172.21.15.16:0/2639494339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:13:37.421 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:37 smithi016 bash[77892]: cluster 2023-10-09T09:13:36.157430+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-09T09:13:37.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:37 smithi094 bash[59677]: cluster 2023-10-09T09:13:36.074304+0000 mgr.y (mgr.25421) 1089 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T09:13:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:37 smithi094 bash[59677]: cluster 2023-10-09T09:13:36.137019+0000 mon.a (mon.0) 1451 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:13:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:37 smithi094 bash[59677]: cluster 2023-10-09T09:13:36.153276+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T09:13:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:37 smithi094 bash[59677]: cluster 2023-10-09T09:13:36.156729+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T09:13:37.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:37 smithi094 bash[59677]: audit 2023-10-09T09:13:36.156990+0000 mon.a (mon.0) 1452 : audit 1 from='client.? 172.21.15.16:0/2639494339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:13:37.493 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:37 smithi094 bash[59677]: cluster 2023-10-09T09:13:36.157430+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-09T09:13:37.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:37.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:37.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:37.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:37.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:37.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:37.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:37.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:37.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:37.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:37.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:38.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 96785 96769 78.5299 54.0625 0.0155944 0.0127275 2023-10-09T09:13:38.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:38.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:38.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:38.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:38.142 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:38.147 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:38.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:38.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:38.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:38.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:38.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:38.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:38.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:38.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:38.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:38.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:38.625 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:38.653 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:38.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:38.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:38.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:38.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:38.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:38.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:38.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:38.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:38.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:39.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 97647 97631 78.2137 53.875 0.0308008 0.0127796 2023-10-09T09:13:39.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:39.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:39.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:39.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:39.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:39.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:39.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:39.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:39.311 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:39 smithi016 bash[75179]: cluster 2023-10-09T09:13:38.074782+0000 mgr.y (mgr.25421) 1090 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T09:13:39.312 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:39 smithi016 bash[77892]: cluster 2023-10-09T09:13:38.074782+0000 mgr.y (mgr.25421) 1090 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T09:13:39.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:39.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:39.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:39.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:39.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:39.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:39.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:39.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:39 smithi094 bash[59677]: cluster 2023-10-09T09:13:38.074782+0000 mgr.y (mgr.25421) 1090 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T09:13:39.574 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:39.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:39.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:39.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:39.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:13:39.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:39.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:39.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:39.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:39.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:39.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:39.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:39.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:40.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 98502 98486 77.9001 53.4375 0.0279259 0.0128304 2023-10-09T09:13:40.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:40.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:40.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:40.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:40.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:40.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:40.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:40.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:40.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:40.531 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:40.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:40.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:40.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:40.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:40.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:40.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:40.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:40.944 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:40 smithi094 bash[59677]: cluster 2023-10-09T09:13:40.075257+0000 mgr.y (mgr.25421) 1091 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T09:13:40.961 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:40.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:40.990 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:40 smithi016 bash[75179]: cluster 2023-10-09T09:13:40.075257+0000 mgr.y (mgr.25421) 1091 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T09:13:40.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:40 smithi016 bash[77892]: cluster 2023-10-09T09:13:40.075257+0000 mgr.y (mgr.25421) 1091 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T09:13:41.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:13:41.000452+0000 min lat: 0.00132275 max lat: 0.323328 avg lat: 0.0128825 2023-10-09T09:13:41.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:13:41.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 99331 99315 77.574 51.8125 0.028366 0.0128825 2023-10-09T09:13:41.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:41.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:41.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:41.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:41.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:41.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:41.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:41.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:41.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:41.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:41.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:41.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:41.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:41.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:41.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:41.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:41.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:41.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:41.763 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:41.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:41.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:41.867 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:41.893 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:41.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:41.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:41.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:42.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 100168 100152 77.2621 52.3125 0.0105416 0.0129366 2023-10-09T09:13:42.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:42.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:42.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:42.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:42.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:42.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:42.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:42.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:42.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:42.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:42.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:42.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:42.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:42.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:42.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:42.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:42.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:42.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:42.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:42.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:42.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:42.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:43.000 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 101010 100994 76.9616 52.625 0.018642 0.0129868 2023-10-09T09:13:43.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:43.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:43.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:43.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:43.132 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:43 smithi016 bash[75179]: cluster 2023-10-09T09:13:42.075935+0000 mgr.y (mgr.25421) 1092 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T09:13:43.132 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:43 smithi016 bash[77892]: cluster 2023-10-09T09:13:42.075935+0000 mgr.y (mgr.25421) 1092 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T09:13:43.189 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:43.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:43.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:43 smithi094 bash[59677]: cluster 2023-10-09T09:13:42.075935+0000 mgr.y (mgr.25421) 1092 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T09:13:43.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:43.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:43.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:43.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:43.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:43.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:43.479 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:43.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:43.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:43.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:43.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:43.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:43.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:43.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:43.797 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:43.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:43.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:43.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:43.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:43.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:43.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:44.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 101670 101654 76.5313 41.25 0.00664475 0.0130601 2023-10-09T09:13:44.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:44.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:44.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:44.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:44.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:44.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:44.304 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:44.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:44.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:44.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:44.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:44.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:44.578 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:44.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:44.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:44.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:44.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:44.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:44.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:44.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:44.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:44.978 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:45.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 102513 102497 76.2475 52.6875 0.0219641 0.0131095 2023-10-09T09:13:45.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:45.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:45.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:45.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:45.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:45.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:45.394 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:45 smithi094 bash[59677]: cluster 2023-10-09T09:13:44.076438+0000 mgr.y (mgr.25421) 1093 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T09:13:45.412 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:45.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:45.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:45 smithi016 bash[75179]: cluster 2023-10-09T09:13:44.076438+0000 mgr.y (mgr.25421) 1093 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T09:13:45.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:45 smithi016 bash[77892]: cluster 2023-10-09T09:13:44.076438+0000 mgr.y (mgr.25421) 1093 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T09:13:45.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:45.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:45.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:45.685 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:45.759 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:45.791 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:13:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:13:45] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.33.4" 2023-10-09T09:13:45.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:45.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:45.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:45.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:45.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:46.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 103307 103291 75.9342 49.625 0.0313125 0.0131625 2023-10-09T09:13:46.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:46.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:46.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:46.218 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:46.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:46.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:46.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:46.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:46.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:46.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:46.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:46.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:46.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:46.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:46.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:46.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:46.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:46.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:46.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:46.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:46.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:46.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:46.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:46.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:46.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:47.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 104027 104011 75.5745 45 0.0178884 0.0132266 2023-10-09T09:13:47.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:47.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:47.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:47.159 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.7 primary-affinity to 0.711948 (8466582) 2023-10-09T09:13:47.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:47.191 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:47 smithi094 bash[59677]: cluster 2023-10-09T09:13:46.078618+0000 mgr.y (mgr.25421) 1094 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T09:13:47.191 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:47 smithi094 bash[59677]: audit 2023-10-09T09:13:46.352978+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.16:0/4098059355' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]: dispatch 2023-10-09T09:13:47.191 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:47 smithi094 bash[59677]: audit 2023-10-09T09:13:46.353590+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]: dispatch 2023-10-09T09:13:47.211 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:47.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:47.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:47.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:47.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:47.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:47.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:47.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:47.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:47 smithi016 bash[75179]: cluster 2023-10-09T09:13:46.078618+0000 mgr.y (mgr.25421) 1094 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T09:13:47.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:47 smithi016 bash[75179]: audit 2023-10-09T09:13:46.352978+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.16:0/4098059355' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]: dispatch 2023-10-09T09:13:47.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:47 smithi016 bash[75179]: audit 2023-10-09T09:13:46.353590+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]: dispatch 2023-10-09T09:13:47.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:47 smithi016 bash[77892]: cluster 2023-10-09T09:13:46.078618+0000 mgr.y (mgr.25421) 1094 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T09:13:47.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:47 smithi016 bash[77892]: audit 2023-10-09T09:13:46.352978+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.16:0/4098059355' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]: dispatch 2023-10-09T09:13:47.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:47 smithi016 bash[77892]: audit 2023-10-09T09:13:46.353590+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]: dispatch 2023-10-09T09:13:47.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:47.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:47.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:47.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:47.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:47.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:47.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:47.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:47.897 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:13:47.897 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:13:47.897 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-10-09T09:13:47.897 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:13:47.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:47.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:48.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 104812 104796 75.2698 49.0625 0.0161011 0.0132798 2023-10-09T09:13:48.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:48.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:48.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:48.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:48.146 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:48.183 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:48.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:48.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:48.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:48.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:48.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:48.436 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:48 smithi016 bash[75179]: cluster 2023-10-09T09:13:47.145790+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T09:13:48.436 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:48 smithi016 bash[75179]: audit 2023-10-09T09:13:47.148702+0000 mon.a (mon.0) 1455 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]': finished 2023-10-09T09:13:48.436 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:48 smithi016 bash[75179]: cluster 2023-10-09T09:13:47.148798+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-09T09:13:48.437 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:48 smithi016 bash[77892]: cluster 2023-10-09T09:13:47.145790+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T09:13:48.437 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:48 smithi016 bash[77892]: audit 2023-10-09T09:13:47.148702+0000 mon.a (mon.0) 1455 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]': finished 2023-10-09T09:13:48.437 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:48 smithi016 bash[77892]: cluster 2023-10-09T09:13:47.148798+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-09T09:13:48.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:48.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:48.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:48 smithi094 bash[59677]: cluster 2023-10-09T09:13:47.145790+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T09:13:48.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:48 smithi094 bash[59677]: audit 2023-10-09T09:13:47.148702+0000 mon.a (mon.0) 1455 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.7119484439158364}]': finished 2023-10-09T09:13:48.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:48 smithi094 bash[59677]: cluster 2023-10-09T09:13:47.148798+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-09T09:13:48.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:48.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:48.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:48.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:48.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:48.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:48.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:48.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:48.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:48.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:48.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:49.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 105614 105598 74.984 50.125 0.0153387 0.0133298 2023-10-09T09:13:49.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:49.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:49.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:49.246 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:49.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.079039+0000 mgr.y (mgr.25421) 1095 : cluster 0 pgmap v1175: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T09:13:49.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.153173+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.156634+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-09T09:13:49.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.160955+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.161522+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.163632+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.163678+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.165695+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.167833+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.170110+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[75179]: cluster 2023-10-09T09:13:48.172916+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.079039+0000 mgr.y (mgr.25421) 1095 : cluster 0 pgmap v1175: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.153173+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.156634+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.160955+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.161522+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.163632+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.163678+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.165695+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.167833+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.170110+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:49 smithi016 bash[77892]: cluster 2023-10-09T09:13:48.172916+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.270 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:49.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:49.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:49.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.079039+0000 mgr.y (mgr.25421) 1095 : cluster 0 pgmap v1175: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T09:13:49.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.153173+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.156634+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-09T09:13:49.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.160955+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.161522+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.163632+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.163678+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.165695+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.167833+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.170110+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:49 smithi094 bash[59677]: cluster 2023-10-09T09:13:48.172916+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T09:13:49.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:49.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:49.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:49.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:49.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:49.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:49.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:49.912 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:49.939 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:49.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:50.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 106429 106413 74.7138 50.9375 0.0318192 0.0133781 2023-10-09T09:13:50.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:50.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:50.157 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:50 smithi094 bash[59677]: audit 2023-10-09T09:13:49.247210+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:50.176 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:50.211 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:50.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:50.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:50.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.322 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:50 smithi016 bash[75179]: audit 2023-10-09T09:13:49.247210+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:50.322 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:50 smithi016 bash[77892]: audit 2023-10-09T09:13:49.247210+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:50.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:50.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:50.448 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:50.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:50.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:50.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:50.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:50.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:50.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:50.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:50.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:50.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:50.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:50.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:50.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:51.001 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0082 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 107220 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 74.4515 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 26.5714 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 168.438 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 41.25 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1191 2023-10-09T09:13:51.002 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 425.142 2023-10-09T09:13:51.003 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2695 2023-10-09T09:13:51.003 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 660 2023-10-09T09:13:51.003 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.013427 2023-10-09T09:13:51.003 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0116331 2023-10-09T09:13:51.003 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 0.323328 2023-10-09T09:13:51.003 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00132275 2023-10-09T09:13:51.009 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:13:51.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:51.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:51.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:51.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:51.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:51.254 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:51.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:51.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:51.400 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:51 smithi016 bash[75179]: cluster 2023-10-09T09:13:50.079630+0000 mgr.y (mgr.25421) 1096 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 37 MiB/s wr, 593 op/s 2023-10-09T09:13:51.400 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:51 smithi016 bash[77892]: cluster 2023-10-09T09:13:50.079630+0000 mgr.y (mgr.25421) 1096 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 37 MiB/s wr, 593 op/s 2023-10-09T09:13:51.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:51.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:51.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:51 smithi094 bash[59677]: cluster 2023-10-09T09:13:50.079630+0000 mgr.y (mgr.25421) 1096 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 37 MiB/s wr, 593 op/s 2023-10-09T09:13:51.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:51.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:51.660 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:51.693 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:51.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:51.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:51.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:51.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:51.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:13:51.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:51.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:52.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:52.115 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:52.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:52.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:52.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:52.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:52.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:52.356 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:52.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:52.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:52.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:52.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:52.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:52.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:52.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:52.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:52.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:52.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:52.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:52.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:52.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:52.941 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:52.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:52.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:53.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:53.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:53.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:53.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:53.211 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:53.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:53.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:53.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:53.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:53.464 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:53.474 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:53 smithi094 bash[59677]: cluster 2023-10-09T09:13:52.080658+0000 mgr.y (mgr.25421) 1097 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T09:13:53.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:53 smithi016 bash[75179]: cluster 2023-10-09T09:13:52.080658+0000 mgr.y (mgr.25421) 1097 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T09:13:53.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:53 smithi016 bash[77892]: cluster 2023-10-09T09:13:52.080658+0000 mgr.y (mgr.25421) 1097 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T09:13:53.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:53.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:53.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:53.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:53.633 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:53.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:53.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:53.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:53.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:53.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:53.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:53.900 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:53.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:53.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:54.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:54.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:54.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:54.175 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:54.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:54.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:54.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:54.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:54.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:54.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:54.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:54.683 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:54.713 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:54.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:54.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:54.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:54.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:54.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:54.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:54.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:55.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:55.136 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:55.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:55.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:55.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:55.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:55 smithi094 bash[59677]: cluster 2023-10-09T09:13:54.081188+0000 mgr.y (mgr.25421) 1098 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 34 MiB/s wr, 588 op/s 2023-10-09T09:13:55.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:55.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:55.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:55.371 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:55.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:55.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:55.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:55.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:55.538 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:55.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:55.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:55.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:55 smithi016 bash[75179]: cluster 2023-10-09T09:13:54.081188+0000 mgr.y (mgr.25421) 1098 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 34 MiB/s wr, 588 op/s 2023-10-09T09:13:55.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:55 smithi016 bash[77892]: cluster 2023-10-09T09:13:54.081188+0000 mgr.y (mgr.25421) 1098 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 34 MiB/s wr, 588 op/s 2023-10-09T09:13:55.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:55.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:55.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:55.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:55.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:55.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:55.912 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:13:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:13:55] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.33.4" 2023-10-09T09:13:55.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:55.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:55.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:56.037 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:56.072 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:56.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:56.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:56.176 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:56.203 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:56.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:56.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:56.294 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:13:56.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:56.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:56.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:56.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:56.346 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:13:56.398 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:13:56.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:56.448 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:13:56.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:56.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:56.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:56.494 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:13:56.534 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:13:56.569 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:13:56.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:56.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:56.608 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:13:56.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:56.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:56.616 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.616 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.616 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.616 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.616 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.616 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.617 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.617 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:13:56.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:56.734 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 5, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:13:56.734 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:13:56.734 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 2, 3, 4, 6, 5, 7] 2023-10-09T09:13:56.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T09:13:56.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:56.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:56.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:56.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:56.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:13:56.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:56.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:13:56.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:57.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:57.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:57.133 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:57.164 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:57.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:57.179 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:13:57.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:57.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:13:57.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:13:57.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:57.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:57.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:57.452 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[75179]: cluster 2023-10-09T09:13:56.081945+0000 mgr.y (mgr.25421) 1099 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 114 B/s rd, 34 MiB/s wr, 902 op/s 2023-10-09T09:13:57.452 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[75179]: audit 2023-10-09T09:13:56.231617+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.16:0/351723904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:57.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[75179]: audit 2023-10-09T09:13:56.997197+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.16:0/942653674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:57.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[75179]: audit 2023-10-09T09:13:56.997981+0000 mon.a (mon.0) 1459 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:57 smithi094 bash[59677]: cluster 2023-10-09T09:13:56.081945+0000 mgr.y (mgr.25421) 1099 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 114 B/s rd, 34 MiB/s wr, 902 op/s 2023-10-09T09:13:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:57 smithi094 bash[59677]: audit 2023-10-09T09:13:56.231617+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.16:0/351723904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:57 smithi094 bash[59677]: audit 2023-10-09T09:13:56.997197+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.16:0/942653674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:57 smithi094 bash[59677]: audit 2023-10-09T09:13:56.997981+0000 mon.a (mon.0) 1459 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:57.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:57.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:57.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:57.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[77892]: cluster 2023-10-09T09:13:56.081945+0000 mgr.y (mgr.25421) 1099 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 114 B/s rd, 34 MiB/s wr, 902 op/s 2023-10-09T09:13:57.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[77892]: audit 2023-10-09T09:13:56.231617+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.16:0/351723904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:13:57.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[77892]: audit 2023-10-09T09:13:56.997197+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.16:0/942653674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:57.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:57 smithi016 bash[77892]: audit 2023-10-09T09:13:56.997981+0000 mon.a (mon.0) 1459 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:13:57.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:57.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:57.843 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:57.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:57.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:13:57.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:57.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:13:58.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:58.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:58.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[75179]: cluster 2023-10-09T09:13:57.165651+0000 mon.a (mon.0) 1460 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[75179]: cluster 2023-10-09T09:13:57.165679+0000 mon.a (mon.0) 1461 : cluster 1 Cluster is now healthy 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[75179]: cluster 2023-10-09T09:13:57.167832+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[75179]: audit 2023-10-09T09:13:57.170911+0000 mon.a (mon.0) 1462 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[75179]: cluster 2023-10-09T09:13:57.171146+0000 mon.a (mon.0) 1463 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[75179]: cluster 2023-10-09T09:13:57.183195+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[77892]: cluster 2023-10-09T09:13:57.165651+0000 mon.a (mon.0) 1460 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[77892]: cluster 2023-10-09T09:13:57.165679+0000 mon.a (mon.0) 1461 : cluster 1 Cluster is now healthy 2023-10-09T09:13:58.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[77892]: cluster 2023-10-09T09:13:57.167832+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T09:13:58.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[77892]: audit 2023-10-09T09:13:57.170911+0000 mon.a (mon.0) 1462 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:13:58.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[77892]: cluster 2023-10-09T09:13:57.171146+0000 mon.a (mon.0) 1463 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-09T09:13:58.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:58 smithi016 bash[77892]: cluster 2023-10-09T09:13:57.183195+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T09:13:58.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:58.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:58.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.379 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:58 smithi094 bash[59677]: cluster 2023-10-09T09:13:57.165651+0000 mon.a (mon.0) 1460 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:13:58.381 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:58 smithi094 bash[59677]: cluster 2023-10-09T09:13:57.165679+0000 mon.a (mon.0) 1461 : cluster 1 Cluster is now healthy 2023-10-09T09:13:58.382 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:58 smithi094 bash[59677]: cluster 2023-10-09T09:13:57.167832+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T09:13:58.382 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:58 smithi094 bash[59677]: audit 2023-10-09T09:13:57.170911+0000 mon.a (mon.0) 1462 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:13:58.382 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:58 smithi094 bash[59677]: cluster 2023-10-09T09:13:57.171146+0000 mon.a (mon.0) 1463 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-09T09:13:58.382 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:58 smithi094 bash[59677]: cluster 2023-10-09T09:13:57.183195+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T09:13:58.402 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:58.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:58.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:58.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:58.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.665 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:58.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:58.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:58.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:58.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:58.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:58.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:58.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:58.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:59.039 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:59.070 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:59.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:59.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:59.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:13:59 smithi016 bash[75179]: cluster 2023-10-09T09:13:58.085784+0000 mgr.y (mgr.25421) 1100 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 103 B/s rd, 30 MiB/s wr, 811 op/s 2023-10-09T09:13:59.206 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:13:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:59.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:13:59.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:59.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:13:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:59.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:13:59.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:59.463 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:13:59 smithi016 bash[77892]: cluster 2023-10-09T09:13:58.085784+0000 mgr.y (mgr.25421) 1100 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 103 B/s rd, 30 MiB/s wr, 811 op/s 2023-10-09T09:13:59.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:13:59.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:59.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:13:59.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:13:59 smithi094 bash[59677]: cluster 2023-10-09T09:13:58.085784+0000 mgr.y (mgr.25421) 1100 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 103 B/s rd, 30 MiB/s wr, 811 op/s 2023-10-09T09:13:59.586 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:59.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:59.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:59.628 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:13:59.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:59.748 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:13:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:59.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:13:59.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:59.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:13:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:13:59.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:13:59.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:00.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:00.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:00.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:00.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:00.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:14:00.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:00.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:00.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:00.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:00.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:00.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:00 smithi016 bash[75179]: cluster 2023-10-09T09:14:00.086353+0000 mgr.y (mgr.25421) 1101 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 25 MiB/s wr, 806 op/s 2023-10-09T09:14:00.820 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:00 smithi016 bash[77892]: cluster 2023-10-09T09:14:00.086353+0000 mgr.y (mgr.25421) 1101 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 25 MiB/s wr, 806 op/s 2023-10-09T09:14:00.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:00.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:00.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:00.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:00.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:00 smithi094 bash[59677]: cluster 2023-10-09T09:14:00.086353+0000 mgr.y (mgr.25421) 1101 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 25 MiB/s wr, 806 op/s 2023-10-09T09:14:01.089 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:01.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:01.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:01.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:14:01.232 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:01.261 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:01.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:01.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:01.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:01.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:01.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:01.500 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:01.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:01.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:01.638 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:01.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:01.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:01.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:01.770 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:01.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:01.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:01.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:01.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:01.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:01.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:02.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:02.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:02.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:02.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:02.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:02.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:02.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:02.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:02.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:02.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:02.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:02.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:02.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:02.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:02.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:02.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:02.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:02.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:02.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:02.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:02.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:02.829 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:02.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:02.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:02.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:02.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:03.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:03.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:03.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:03.145 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:03 smithi094 bash[59677]: cluster 2023-10-09T09:14:02.086931+0000 mgr.y (mgr.25421) 1102 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 819 op/s 2023-10-09T09:14:03.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:03.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:03.277 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:03.308 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:03.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:03.419 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:03 smithi016 bash[75179]: cluster 2023-10-09T09:14:02.086931+0000 mgr.y (mgr.25421) 1102 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 819 op/s 2023-10-09T09:14:03.420 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:03 smithi016 bash[77892]: cluster 2023-10-09T09:14:02.086931+0000 mgr.y (mgr.25421) 1102 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 819 op/s 2023-10-09T09:14:03.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:03.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:03.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:03.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:03.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:03.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:03.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:03.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:03.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:03.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:03.971 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:03.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:04.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:04.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:04.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:04.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:04.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:04.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:04.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:04.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:04.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:04.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:04.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:04.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:04.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:04.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:04.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:04.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:04.765 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:04.790 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:04.805 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:04.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:04.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:04.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:04.932 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:04.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:05.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:05.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:05.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:05.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:05.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:05.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:05.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:05.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:05.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:05.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:05.578 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:05 smithi016 bash[75179]: cluster 2023-10-09T09:14:04.087547+0000 mgr.y (mgr.25421) 1103 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 3.0 MiB/s wr, 817 op/s 2023-10-09T09:14:05.578 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:05 smithi016 bash[75179]: audit 2023-10-09T09:14:04.253134+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:05.578 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:05 smithi016 bash[75179]: audit 2023-10-09T09:14:04.257223+0000 mon.a (mon.0) 1465 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:05.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:05 smithi016 bash[77892]: cluster 2023-10-09T09:14:04.087547+0000 mgr.y (mgr.25421) 1103 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 3.0 MiB/s wr, 817 op/s 2023-10-09T09:14:05.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:05 smithi016 bash[77892]: audit 2023-10-09T09:14:04.253134+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:05.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:05 smithi016 bash[77892]: audit 2023-10-09T09:14:04.257223+0000 mon.a (mon.0) 1465 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:05.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:05.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:05.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:05.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:05 smithi094 bash[59677]: cluster 2023-10-09T09:14:04.087547+0000 mgr.y (mgr.25421) 1103 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 3.0 MiB/s wr, 817 op/s 2023-10-09T09:14:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:05 smithi094 bash[59677]: audit 2023-10-09T09:14:04.253134+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:05 smithi094 bash[59677]: audit 2023-10-09T09:14:04.257223+0000 mon.a (mon.0) 1465 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:05.800 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:05.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:05.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:05.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:05.869 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:14:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:14:05] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.33.4" 2023-10-09T09:14:05.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:05.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:05.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:05.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:06.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:06.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:06.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:06.216 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:06.249 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:06.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:06.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:06.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:06.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:06.485 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:06.508 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:06.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:06.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:06.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:06.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:06.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:06.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:06.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:06.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:06.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:06.902 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:06.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:07.008 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:07.032 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:07.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:07.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:07.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:07.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:07.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:07.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:07.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:07.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:07.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:07 smithi094 bash[59677]: cluster 2023-10-09T09:14:06.088254+0000 mgr.y (mgr.25421) 1104 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T09:14:07.554 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:07.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:07.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:07.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:07.697 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:07 smithi016 bash[75179]: cluster 2023-10-09T09:14:06.088254+0000 mgr.y (mgr.25421) 1104 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T09:14:07.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:07 smithi016 bash[77892]: cluster 2023-10-09T09:14:06.088254+0000 mgr.y (mgr.25421) 1104 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T09:14:07.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:07.822 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:07.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:07.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:07.954 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:07.991 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:07.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:08.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:08.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:08.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:08.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:08.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:08.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:08.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:08.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:08.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:08.414 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:14:08.414 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:14:08.509 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:08.544 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:08.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:08.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:08.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:08.681 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:08.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:08.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:08.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:08.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:08.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:08.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:08.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:09.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:09.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:09.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:09.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:09.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:09.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:09.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:09.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:09.358 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:09 smithi016 bash[77892]: cluster 2023-10-09T09:14:08.088682+0000 mgr.y (mgr.25421) 1105 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T09:14:09.358 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:09 smithi016 bash[75179]: cluster 2023-10-09T09:14:08.088682+0000 mgr.y (mgr.25421) 1105 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T09:14:09.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:09.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:09.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:09.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:09.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:09.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:09.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:09.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:09.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:09.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:09 smithi094 bash[59677]: cluster 2023-10-09T09:14:08.088682+0000 mgr.y (mgr.25421) 1105 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T09:14:09.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:09.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:09.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:09.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:09.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:09.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:10.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:10.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:10.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:10.153 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:10.197 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:10.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:10.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:10.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:10.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:10.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:10.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:10.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:10.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:10.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:10.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:10.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:10.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:10.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:10.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:10.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:10.847 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:10 smithi094 bash[59677]: cluster 2023-10-09T09:14:10.089125+0000 mgr.y (mgr.25421) 1106 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T09:14:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:10.880 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:10.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:10.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:10.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:10 smithi016 bash[75179]: cluster 2023-10-09T09:14:10.089125+0000 mgr.y (mgr.25421) 1106 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T09:14:10.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:10 smithi016 bash[77892]: cluster 2023-10-09T09:14:10.089125+0000 mgr.y (mgr.25421) 1106 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T09:14:11.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:11.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:11.125 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:11.150 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:11.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:14:11.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:11.281 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:11.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:11.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:11.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:11.519 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:11.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:11.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:11.668 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:11.695 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:11.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:11.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:11.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:11.968 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:11.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:12.071 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:12.110 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:12.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:12.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:12.215 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:12.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:12.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:12.363 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:12.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:12.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:12.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:12.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:12.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:12.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:12.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:12.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:12.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:12.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:12.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:12.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:12.813 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:12.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:12.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:12.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:12.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:12.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:13.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:13.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:13.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:13.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:13.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:13.200 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:13 smithi016 bash[75179]: cluster 2023-10-09T09:14:12.089806+0000 mgr.y (mgr.25421) 1107 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T09:14:13.201 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:13 smithi016 bash[77892]: cluster 2023-10-09T09:14:12.089806+0000 mgr.y (mgr.25421) 1107 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T09:14:13.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:13.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:13.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:13.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:13.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:13.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:13.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:13.476 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:13 smithi094 bash[59677]: cluster 2023-10-09T09:14:12.089806+0000 mgr.y (mgr.25421) 1107 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T09:14:13.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:13.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:13.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:13.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:13.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:13.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:13.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:13.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:13.768 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:13.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:13.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:13.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:13.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:14.005 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:14.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:14.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:14.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:14.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:14.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:14.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:14.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:14.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:14.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:14.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:14.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:14.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:14.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:14.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:14.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:14.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:14.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:14.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:14.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:14.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:14.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:15.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:15.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:15.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:15.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:15.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:15.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:15.340 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:15.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:15.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:15.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:15.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:15 smithi094 bash[59677]: cluster 2023-10-09T09:14:14.090309+0000 mgr.y (mgr.25421) 1108 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T09:14:15.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:15 smithi016 bash[75179]: cluster 2023-10-09T09:14:14.090309+0000 mgr.y (mgr.25421) 1108 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T09:14:15.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:15 smithi016 bash[77892]: cluster 2023-10-09T09:14:14.090309+0000 mgr.y (mgr.25421) 1108 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T09:14:15.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:15.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:15.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:15.651 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:15.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:15.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:14:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:14:15] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.33.4" 2023-10-09T09:14:15.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:15.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:15.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:15.883 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:15.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:15.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:15.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:16.015 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:16.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:16.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:16.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:16.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:16.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:16.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:16.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:16.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:16.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:16.445 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:16.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:16.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:16.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:16.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:16.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:16.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:16.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:16.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:16.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:16.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:16.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:16.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:16.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:17.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:17.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:17.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:17.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:17 smithi094 bash[59677]: cluster 2023-10-09T09:14:16.093076+0000 mgr.y (mgr.25421) 1109 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T09:14:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:17 smithi094 bash[59677]: audit 2023-10-09T09:14:16.979841+0000 mon.a (mon.0) 1466 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:17.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:17.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:17.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:17.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:17.384 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:17.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:17.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:17.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:17 smithi016 bash[75179]: cluster 2023-10-09T09:14:16.093076+0000 mgr.y (mgr.25421) 1109 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T09:14:17.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:17 smithi016 bash[75179]: audit 2023-10-09T09:14:16.979841+0000 mon.a (mon.0) 1466 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:17.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:17 smithi016 bash[77892]: cluster 2023-10-09T09:14:16.093076+0000 mgr.y (mgr.25421) 1109 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T09:14:17.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:17 smithi016 bash[77892]: audit 2023-10-09T09:14:16.979841+0000 mon.a (mon.0) 1466 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:17.492 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:17.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:17.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:17.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:17.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:17.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:17.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:17.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:17.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:17.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:17.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:17.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:17.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:17.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:18.024 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:18.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:18.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:18.170 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:18.214 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:18.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:18.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:18.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:18.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:18.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:18.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:18.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:18.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:18.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:18.577 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:18.607 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:18.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:18.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:18.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:18.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:18 smithi016 bash[75179]: debug 2023-10-09T09:14:18.514+0000 7fa046a4d700 -1 mon.a@0(leader).osd e259 definitely_dead 0 2023-10-09T09:14:18.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:18.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:18.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:18.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:18.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:18.928 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:18.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:19.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:19.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:19 smithi094 bash[59677]: cluster 2023-10-09T09:14:18.093546+0000 mgr.y (mgr.25421) 1110 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T09:14:19.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:19 smithi094 bash[59677]: audit 2023-10-09T09:14:18.516010+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.16:0/1588408218' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:14:19.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:19 smithi094 bash[59677]: audit 2023-10-09T09:14:18.516592+0000 mon.a (mon.0) 1467 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:14:19.165 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:19 smithi094 bash[59677]: cluster 2023-10-09T09:14:18.516745+0000 mon.a (mon.0) 1468 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:14:19.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:19.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:19.183 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.2. 2023-10-09T09:14:19.236 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[75179]: cluster 2023-10-09T09:14:18.093546+0000 mgr.y (mgr.25421) 1110 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T09:14:19.236 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[75179]: audit 2023-10-09T09:14:18.516010+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.16:0/1588408218' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:14:19.236 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[75179]: audit 2023-10-09T09:14:18.516592+0000 mon.a (mon.0) 1467 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:14:19.236 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[75179]: cluster 2023-10-09T09:14:18.516745+0000 mon.a (mon.0) 1468 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:14:19.237 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[77892]: cluster 2023-10-09T09:14:18.093546+0000 mgr.y (mgr.25421) 1110 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T09:14:19.237 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[77892]: audit 2023-10-09T09:14:18.516010+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.16:0/1588408218' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:14:19.237 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[77892]: audit 2023-10-09T09:14:18.516592+0000 mon.a (mon.0) 1467 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:14:19.237 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:19 smithi016 bash[77892]: cluster 2023-10-09T09:14:18.516745+0000 mon.a (mon.0) 1468 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:14:19.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:19.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:19.400 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.428 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:19.532 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:19.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:19.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:19.693 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:19.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.825 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:19.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.956 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:19.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:20.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:20.076 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:20.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:20.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:20.179 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:20.211 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.168804+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: audit 2023-10-09T09:14:19.172009+0000 mon.a (mon.0) 1469 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.172127+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e260: 8 total, 8 up, 6 in 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.179396+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.179939+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: audit 2023-10-09T09:14:19.248280+0000 mon.a (mon.0) 1471 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.725576+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.726110+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.727423+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.727801+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e261: 8 total, 8 up, 6 in 2023-10-09T09:14:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.729455+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.730362+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.730746+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:20 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.733026+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:20.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:20.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:20.323 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.168804+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.323 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: audit 2023-10-09T09:14:19.172009+0000 mon.a (mon.0) 1469 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:14:20.323 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.172127+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e260: 8 total, 8 up, 6 in 2023-10-09T09:14:20.323 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.179396+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.179939+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: audit 2023-10-09T09:14:19.248280+0000 mon.a (mon.0) 1471 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.725576+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.726110+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.727423+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.727801+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e261: 8 total, 8 up, 6 in 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.729455+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.730362+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.730746+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.733026+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.168804+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: audit 2023-10-09T09:14:19.172009+0000 mon.a (mon.0) 1469 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.172127+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e260: 8 total, 8 up, 6 in 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.179396+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.179939+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: audit 2023-10-09T09:14:19.248280+0000 mon.a (mon.0) 1471 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:20.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.725576+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.726110+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.727423+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.727801+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e261: 8 total, 8 up, 6 in 2023-10-09T09:14:20.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.729455+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.730362+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.730746+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.326 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:20 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.733026+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:20.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:20.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:20.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:20.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:20.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:20.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:20.598 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:20.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:20.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:20.702 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:20.742 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:20.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:20.846 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:20.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:20.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:20.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:21.011 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:21.045 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:21.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:21.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:21.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:21.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:21.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:19.723741+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.094295+0000 mgr.y (mgr.25421) 1111 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.727986+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.727987+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e262: 8 total, 8 up, 6 in 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.728846+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.732267+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.732873+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.427 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.735561+0000 osd.4 (osd.4) 226 : cluster 0 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 259'8010 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.750390+0000 osd.6 (osd.6) 199 : cluster 0 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 259'8029 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:19.723741+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.094295+0000 mgr.y (mgr.25421) 1111 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.727986+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.727987+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e262: 8 total, 8 up, 6 in 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.728846+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.732267+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.732873+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.735561+0000 osd.4 (osd.4) 226 : cluster 0 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 259'8010 2023-10-09T09:14:21.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:21 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.750390+0000 osd.6 (osd.6) 199 : cluster 0 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 259'8029 2023-10-09T09:14:21.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:21.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:19.723741+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T09:14:21.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.094295+0000 mgr.y (mgr.25421) 1111 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T09:14:21.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.727986+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.727987+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e262: 8 total, 8 up, 6 in 2023-10-09T09:14:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.728846+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.732267+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.732873+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.735561+0000 osd.4 (osd.4) 226 : cluster 0 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 259'8010 2023-10-09T09:14:21.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:21 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.750390+0000 osd.6 (osd.6) 199 : cluster 0 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 259'8029 2023-10-09T09:14:21.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:21.573 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:21.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:21.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:21.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:21.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:21.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:21.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:21.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:21.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:22.064 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:22.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:22.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:22.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:22.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:22.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:22.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:22.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:22 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.731314+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:22.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:22 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.735135+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:22.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:22 smithi016 bash[75179]: cluster 2023-10-09T09:14:20.737993+0000 osd.0 (osd.0) 27 : cluster 0 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 259'8241 2023-10-09T09:14:22.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:22 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.731314+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:22.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:22.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:22.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:22.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:22.466 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:22 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.731314+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:22.466 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:22 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.735135+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:22.467 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:22 smithi094 bash[59677]: cluster 2023-10-09T09:14:20.737993+0000 osd.0 (osd.0) 27 : cluster 0 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 259'8241 2023-10-09T09:14:22.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:22.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:22.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:22 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.735135+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T09:14:22.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:22 smithi016 bash[77892]: cluster 2023-10-09T09:14:20.737993+0000 osd.0 (osd.0) 27 : cluster 0 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 259'8241 2023-10-09T09:14:22.600 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:22.629 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:22.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:22.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:22.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:22.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:22.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:22.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:22.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:22.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:23.035 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:23.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:23.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:23.131 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:14:23.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:23.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:23.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:23.172 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:14:23.204 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:14:23.252 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:14:23.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:23.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:23.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:23.312 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:14:23.354 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:14:23.382 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:14:23.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:23.422 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:23 smithi094 bash[59677]: cluster 2023-10-09T09:14:22.094906+0000 mgr.y (mgr.25421) 1112 : cluster 0 pgmap v1197: 89 pgs: 4 remapped+peering, 85 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 9 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:14:23.422 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:23 smithi094 bash[59677]: audit 2023-10-09T09:14:23.086886+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.16:0/922788266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:14:23.422 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:14:23.424 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.425 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.425 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.425 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.425 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.425 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.425 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.425 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:14:23.447 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:23.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:23.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:23.453 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:23 smithi016 bash[77892]: cluster 2023-10-09T09:14:22.094906+0000 mgr.y (mgr.25421) 1112 : cluster 0 pgmap v1197: 89 pgs: 4 remapped+peering, 85 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 9 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:14:23.453 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:23 smithi016 bash[77892]: audit 2023-10-09T09:14:23.086886+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.16:0/922788266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:14:23.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:23 smithi016 bash[75179]: cluster 2023-10-09T09:14:22.094906+0000 mgr.y (mgr.25421) 1112 : cluster 0 pgmap v1197: 89 pgs: 4 remapped+peering, 85 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 9 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:14:23.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:23 smithi016 bash[75179]: audit 2023-10-09T09:14:23.086886+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.16:0/922788266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:14:23.552 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:23.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:23.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:23.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:23.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:23.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:23.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:23.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:23.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:23.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:23.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:23.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:24.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:24.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:24.149 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:24.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:24.195 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:14:24.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:24 smithi016 bash[75179]: audit 2023-10-09T09:14:23.423703+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.16:0/2925568723' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:14:24.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:24 smithi016 bash[75179]: audit 2023-10-09T09:14:23.424411+0000 mon.a (mon.0) 1474 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:14:24.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:24 smithi016 bash[77892]: audit 2023-10-09T09:14:23.423703+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.16:0/2925568723' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:14:24.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:24 smithi016 bash[77892]: audit 2023-10-09T09:14:23.424411+0000 mon.a (mon.0) 1474 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:14:24.252 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:24.275 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:24.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:24.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:24.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:24.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:24 smithi094 bash[59677]: audit 2023-10-09T09:14:23.423703+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.16:0/2925568723' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:14:24.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:24 smithi094 bash[59677]: audit 2023-10-09T09:14:23.424411+0000 mon.a (mon.0) 1474 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:14:24.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:24.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:24.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:24.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:24.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:24.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:24.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:24.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:24.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:24.928 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:24.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:24.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:25.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:25.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:25.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:25.156 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:25.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:25.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:25.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:25.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.095525+0000 mgr.y (mgr.25421) 1113 : cluster 0 pgmap v1198: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 83 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 6703/255579 objects degraded (2.623%); 5274/255579 objects misplaced (2.064%); 885 KiB/s, 1 keys/s, 14 objects/s recovering 2023-10-09T09:14:25.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.179019+0000 mon.a (mon.0) 1475 : cluster 3 Health check failed: Degraded data redundancy: 6703/255579 objects degraded (2.623%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:14:25.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.179628+0000 mon.a (mon.0) 1476 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:14:25.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: audit 2023-10-09T09:14:24.190841+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:14:25.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.190888+0000 mon.a (mon.0) 1478 : cluster 0 osdmap e263: 8 total, 8 up, 6 in 2023-10-09T09:14:25.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.328188+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.330003+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.095525+0000 mgr.y (mgr.25421) 1113 : cluster 0 pgmap v1198: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 83 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 6703/255579 objects degraded (2.623%); 5274/255579 objects misplaced (2.064%); 885 KiB/s, 1 keys/s, 14 objects/s recovering 2023-10-09T09:14:25.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.179019+0000 mon.a (mon.0) 1475 : cluster 3 Health check failed: Degraded data redundancy: 6703/255579 objects degraded (2.623%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:14:25.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.179628+0000 mon.a (mon.0) 1476 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:14:25.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: audit 2023-10-09T09:14:24.190841+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:14:25.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.190888+0000 mon.a (mon.0) 1478 : cluster 0 osdmap e263: 8 total, 8 up, 6 in 2023-10-09T09:14:25.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.328188+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.330003+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.333884+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.340803+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.343572+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:25.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:25.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:25.448 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:25.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.095525+0000 mgr.y (mgr.25421) 1113 : cluster 0 pgmap v1198: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 83 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 6703/255579 objects degraded (2.623%); 5274/255579 objects misplaced (2.064%); 885 KiB/s, 1 keys/s, 14 objects/s recovering 2023-10-09T09:14:25.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.179019+0000 mon.a (mon.0) 1475 : cluster 3 Health check failed: Degraded data redundancy: 6703/255579 objects degraded (2.623%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.179628+0000 mon.a (mon.0) 1476 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: audit 2023-10-09T09:14:24.190841+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.190888+0000 mon.a (mon.0) 1478 : cluster 0 osdmap e263: 8 total, 8 up, 6 in 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.328188+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.330003+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.333884+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.340803+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.458 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:25 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.343572+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.488 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:25.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:25.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:25.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:25.631 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:25.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:25.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:25.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:25.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.333884+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.340803+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.343572+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:25.798 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:25.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:25.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:25.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:25.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:25.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:25.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:26.004 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:14:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:14:25] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.33.4" 2023-10-09T09:14:26.026 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:26.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:26.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:26.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:26.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:26.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:26.304 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:26.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:26 smithi094 bash[59677]: cluster 2023-10-09T09:14:24.329741+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:26.313 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:26 smithi094 bash[59677]: audit 2023-10-09T09:14:25.282736+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:26.313 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:26 smithi094 bash[59677]: audit 2023-10-09T09:14:25.287923+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:26.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:26.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:26.448 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:26.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:26.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:26.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:26 smithi016 bash[77892]: cluster 2023-10-09T09:14:24.329741+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:26.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:26 smithi016 bash[77892]: audit 2023-10-09T09:14:25.282736+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:26.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:26 smithi016 bash[77892]: audit 2023-10-09T09:14:25.287923+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:26.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:26 smithi016 bash[75179]: cluster 2023-10-09T09:14:24.329741+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T09:14:26.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:26 smithi016 bash[75179]: audit 2023-10-09T09:14:25.282736+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:26.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:26 smithi016 bash[75179]: audit 2023-10-09T09:14:25.287923+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:26.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:26.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:26.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:26.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:26.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:26.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:26.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:26.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:26.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:26.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:26.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:27.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:27.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:27.126 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:27.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:27.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:27.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:27.398 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:27.407 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:27 smithi094 bash[59677]: cluster 2023-10-09T09:14:26.096686+0000 mgr.y (mgr.25421) 1114 : cluster 0 pgmap v1200: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 13421/250299 objects degraded (5.362%); 20288/250299 objects misplaced (8.106%); 5.8 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T09:14:27.407 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:27 smithi094 bash[59677]: audit 2023-10-09T09:14:26.329318+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:27.408 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:27 smithi094 bash[59677]: audit 2023-10-09T09:14:26.337889+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:27.436 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:27.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:27.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:27.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:27.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:27.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:27 smithi016 bash[75179]: cluster 2023-10-09T09:14:26.096686+0000 mgr.y (mgr.25421) 1114 : cluster 0 pgmap v1200: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 13421/250299 objects degraded (5.362%); 20288/250299 objects misplaced (8.106%); 5.8 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T09:14:27.702 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:27 smithi016 bash[75179]: audit 2023-10-09T09:14:26.329318+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:27.702 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:27 smithi016 bash[75179]: audit 2023-10-09T09:14:26.337889+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:27.702 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:27 smithi016 bash[77892]: cluster 2023-10-09T09:14:26.096686+0000 mgr.y (mgr.25421) 1114 : cluster 0 pgmap v1200: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 13421/250299 objects degraded (5.362%); 20288/250299 objects misplaced (8.106%); 5.8 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T09:14:27.702 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:27 smithi016 bash[77892]: audit 2023-10-09T09:14:26.329318+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:27.702 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:27 smithi016 bash[77892]: audit 2023-10-09T09:14:26.337889+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:27.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:27.707 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7] out_osds: [1, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:14:27.707 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:14:27.708 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T09:14:27.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T09:14:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:27.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:27.844 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:27.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:27.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:27.978 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:27.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:27.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:28.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:28.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:28.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:28.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:28.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:28.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:28.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:28.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:28.414 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:28.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:28.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:28.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:28.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:28.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:28.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:28.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:28.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:28.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:28.808 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:28.835 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:28.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:28.860 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:14:28.860 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T09:14:28.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T09:14:28.860 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [] 2023-10-09T09:14:28.860 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:14:28.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:28.962 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:28.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:29.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:29.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:14:29.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:29.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:29.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:29.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:29.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:29.339 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:29 smithi016 bash[75179]: cluster 2023-10-09T09:14:28.097202+0000 mgr.y (mgr.25421) 1115 : cluster 0 pgmap v1201: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 13421/250299 objects degraded (5.362%); 20288/250299 objects misplaced (8.106%); 4.6 MiB/s, 1 keys/s, 74 objects/s recovering 2023-10-09T09:14:29.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:29.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:29.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:29.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:29.511 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:29.540 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:29.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:29.606 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:29 smithi016 bash[77892]: cluster 2023-10-09T09:14:28.097202+0000 mgr.y (mgr.25421) 1115 : cluster 0 pgmap v1201: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 13421/250299 objects degraded (5.362%); 20288/250299 objects misplaced (8.106%); 4.6 MiB/s, 1 keys/s, 74 objects/s recovering 2023-10-09T09:14:29.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:29.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:29 smithi094 bash[59677]: cluster 2023-10-09T09:14:28.097202+0000 mgr.y (mgr.25421) 1115 : cluster 0 pgmap v1201: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 13421/250299 objects degraded (5.362%); 20288/250299 objects misplaced (8.106%); 4.6 MiB/s, 1 keys/s, 74 objects/s recovering 2023-10-09T09:14:29.690 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:29.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:29.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:14:29.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:29.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:29.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:29.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:29.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:29.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:29.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:29.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:30.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:30.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:30.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:30.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:30.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:14:30.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:30.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:30.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:30.332 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:30.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:30.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:30.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:30.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:30.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:30.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:30.619 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:30.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:30.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:30.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:30.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:30 smithi094 bash[59677]: audit 2023-10-09T09:14:29.506364+0000 mon.a (mon.0) 1483 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:30.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:30 smithi094 bash[59677]: audit 2023-10-09T09:14:29.512339+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:30.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:30 smithi094 bash[59677]: cluster 2023-10-09T09:14:29.724687+0000 mon.a (mon.0) 1485 : cluster 3 Health check update: Degraded data redundancy: 13421/250299 objects degraded (5.362%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:30.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:30.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:30.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:30.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:30.861 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:30 smithi016 bash[75179]: audit 2023-10-09T09:14:29.506364+0000 mon.a (mon.0) 1483 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:30.861 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:30 smithi016 bash[75179]: audit 2023-10-09T09:14:29.512339+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:30.861 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:30 smithi016 bash[75179]: cluster 2023-10-09T09:14:29.724687+0000 mon.a (mon.0) 1485 : cluster 3 Health check update: Degraded data redundancy: 13421/250299 objects degraded (5.362%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:30.862 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:30 smithi016 bash[77892]: audit 2023-10-09T09:14:29.506364+0000 mon.a (mon.0) 1483 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:30.862 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:30 smithi016 bash[77892]: audit 2023-10-09T09:14:29.512339+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:30.862 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:30 smithi016 bash[77892]: cluster 2023-10-09T09:14:29.724687+0000 mon.a (mon.0) 1485 : cluster 3 Health check update: Degraded data redundancy: 13421/250299 objects degraded (5.362%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:30.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:30.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:30.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:30.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:31.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:31.131 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:31.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:31.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:31.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:31.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:31.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:31.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:31.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:31.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:31.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:31.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:31.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:31.676 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:31 smithi094 bash[59677]: cluster 2023-10-09T09:14:30.097870+0000 mgr.y (mgr.25421) 1116 : cluster 0 pgmap v1202: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 276 op/s; 13421/248808 objects degraded (5.394%); 20132/248808 objects misplaced (8.091%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T09:14:31.697 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:31.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:31.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:31 smithi016 bash[75179]: cluster 2023-10-09T09:14:30.097870+0000 mgr.y (mgr.25421) 1116 : cluster 0 pgmap v1202: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 276 op/s; 13421/248808 objects degraded (5.394%); 20132/248808 objects misplaced (8.091%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T09:14:31.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:31 smithi016 bash[77892]: cluster 2023-10-09T09:14:30.097870+0000 mgr.y (mgr.25421) 1116 : cluster 0 pgmap v1202: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 276 op/s; 13421/248808 objects degraded (5.394%); 20132/248808 objects misplaced (8.091%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T09:14:31.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:31.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:31.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:31.968 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:31.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:32.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:32.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:32.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:32.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:32.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:32.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:32.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:32.358 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:32.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:32.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:32.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:32.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:32.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:32.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:32.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:32.635 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:32.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:32.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:32.683 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:32.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:32.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:32.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:32.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:32.937 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:32.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:33.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:33.091 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:33.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:33.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:33.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:33.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:33.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:33.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:33.370 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:33.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:33.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:33.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:33.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:33.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:33.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:33.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:33.631 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:33 smithi094 bash[59677]: cluster 2023-10-09T09:14:32.098755+0000 mgr.y (mgr.25421) 1117 : cluster 0 pgmap v1203: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 13549/241782 objects degraded (5.604%); 19281/241782 objects misplaced (7.975%); 6.0 MiB/s, 97 objects/s recovering 2023-10-09T09:14:33.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:33.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:33.700 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:33.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:33.808 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:33.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:33.888 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:33 smithi016 bash[77892]: cluster 2023-10-09T09:14:32.098755+0000 mgr.y (mgr.25421) 1117 : cluster 0 pgmap v1203: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 13549/241782 objects degraded (5.604%); 19281/241782 objects misplaced (7.975%); 6.0 MiB/s, 97 objects/s recovering 2023-10-09T09:14:33.888 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:33 smithi016 bash[75179]: cluster 2023-10-09T09:14:32.098755+0000 mgr.y (mgr.25421) 1117 : cluster 0 pgmap v1203: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 13549/241782 objects degraded (5.604%); 19281/241782 objects misplaced (7.975%); 6.0 MiB/s, 97 objects/s recovering 2023-10-09T09:14:33.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:33.953 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:33.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:34.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:34.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:34.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:34.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:34.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:34.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:34.350 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:34.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:34.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:34.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:34.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:34.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:34.709 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:34.719 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:34.719 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:14:34.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:34.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:34.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:34.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:34.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:34.992 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:34.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:35.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:35.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:35.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:35.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:35.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:35.276 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:35.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:35.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:35.415 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:35.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:35.521 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:35.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:35 smithi094 bash[59677]: cluster 2023-10-09T09:14:34.099478+0000 mgr.y (mgr.25421) 1118 : cluster 0 pgmap v1204: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 13549/240159 objects degraded (5.642%); 19102/240159 objects misplaced (7.954%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T09:14:35.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:35 smithi094 bash[59677]: audit 2023-10-09T09:14:34.252936+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:35.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:35 smithi094 bash[59677]: audit 2023-10-09T09:14:34.254186+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:35.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:35 smithi094 bash[59677]: cluster 2023-10-09T09:14:34.729904+0000 mon.a (mon.0) 1488 : cluster 3 Health check update: Degraded data redundancy: 13549/240159 objects degraded (5.642%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:35.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:35.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:35.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:35.664 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[75179]: cluster 2023-10-09T09:14:34.099478+0000 mgr.y (mgr.25421) 1118 : cluster 0 pgmap v1204: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 13549/240159 objects degraded (5.642%); 19102/240159 objects misplaced (7.954%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T09:14:35.664 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[75179]: audit 2023-10-09T09:14:34.252936+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:35.664 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[75179]: audit 2023-10-09T09:14:34.254186+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:35.664 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[75179]: cluster 2023-10-09T09:14:34.729904+0000 mon.a (mon.0) 1488 : cluster 3 Health check update: Degraded data redundancy: 13549/240159 objects degraded (5.642%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:35.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[77892]: cluster 2023-10-09T09:14:34.099478+0000 mgr.y (mgr.25421) 1118 : cluster 0 pgmap v1204: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 13549/240159 objects degraded (5.642%); 19102/240159 objects misplaced (7.954%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T09:14:35.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[77892]: audit 2023-10-09T09:14:34.252936+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:35.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[77892]: audit 2023-10-09T09:14:34.254186+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:35.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[77892]: cluster 2023-10-09T09:14:34.729904+0000 mon.a (mon.0) 1488 : cluster 3 Health check update: Degraded data redundancy: 13549/240159 objects degraded (5.642%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:35.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:35.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:35.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:35.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:35.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:35.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:35.955 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:14:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:14:35] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T09:14:35.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:35.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:36.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:36.118 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:36.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:36.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:36.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:36.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:36.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:36.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:36.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:36.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:36.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:36.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:36.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:36.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:36.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:36.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:36.656 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:36.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:36.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:36.793 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:36.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:36.906 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:36.944 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:36.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:37.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:37.087 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:37.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:37.194 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:37.224 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:37.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:37.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:37.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:37.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:37.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:37.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:37.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:37.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:37.586 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:37.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:37.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:37.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:37.734 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:37 smithi016 bash[75179]: cluster 2023-10-09T09:14:36.100260+0000 mgr.y (mgr.25421) 1119 : cluster 0 pgmap v1205: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 13700/233172 objects degraded (5.875%); 18240/233172 objects misplaced (7.823%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T09:14:37.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:37 smithi016 bash[77892]: cluster 2023-10-09T09:14:36.100260+0000 mgr.y (mgr.25421) 1119 : cluster 0 pgmap v1205: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 13700/233172 objects degraded (5.875%); 18240/233172 objects misplaced (7.823%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T09:14:37.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:37 smithi094 bash[59677]: cluster 2023-10-09T09:14:36.100260+0000 mgr.y (mgr.25421) 1119 : cluster 0 pgmap v1205: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 13700/233172 objects degraded (5.875%); 18240/233172 objects misplaced (7.823%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T09:14:37.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:37.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:37.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:37.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:37.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:37.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:37.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:37.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:38.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:38.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:38.105 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.131 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:38.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:38.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:38.379 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:38.530 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.560 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:38.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:38.710 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:38.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:38.814 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.844 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:38.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:38.971 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:38.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:38.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:39.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:39.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:39.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:39.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:39.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:39.238 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:39.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:39.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:39.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:39.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:39.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:39.480 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:39.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:39 smithi094 bash[59677]: cluster 2023-10-09T09:14:38.100787+0000 mgr.y (mgr.25421) 1120 : cluster 0 pgmap v1206: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 13700/233172 objects degraded (5.875%); 18240/233172 objects misplaced (7.823%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:14:39.507 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:39.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:39.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:39.622 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:39 smithi016 bash[75179]: cluster 2023-10-09T09:14:38.100787+0000 mgr.y (mgr.25421) 1120 : cluster 0 pgmap v1206: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 13700/233172 objects degraded (5.875%); 18240/233172 objects misplaced (7.823%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:14:39.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:39 smithi016 bash[77892]: cluster 2023-10-09T09:14:38.100787+0000 mgr.y (mgr.25421) 1120 : cluster 0 pgmap v1206: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 13700/233172 objects degraded (5.875%); 18240/233172 objects misplaced (7.823%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:14:39.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:39.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:39.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:39.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:39.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:39.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:39.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:39.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:40.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:40.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:40.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:40.178 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:40.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:40.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:40.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:40.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:40 smithi094 bash[59677]: cluster 2023-10-09T09:14:39.732255+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 13700/233172 objects degraded (5.875%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:40.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:40.543 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:40 smithi016 bash[75179]: cluster 2023-10-09T09:14:39.732255+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 13700/233172 objects degraded (5.875%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:40.543 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:40 smithi016 bash[77892]: cluster 2023-10-09T09:14:39.732255+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 13700/233172 objects degraded (5.875%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:40.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:40.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:40.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:40.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:40.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:40.793 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:40.821 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:40.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:40.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:40.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:41.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:41.101 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:41.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:41.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:41.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:41.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:41.334 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:41.343 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:41 smithi094 bash[59677]: cluster 2023-10-09T09:14:40.101447+0000 mgr.y (mgr.25421) 1121 : cluster 0 pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 13700/231660 objects degraded (5.914%); 18070/231660 objects misplaced (7.800%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:14:41.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:41.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:41.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:41.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:41 smithi016 bash[75179]: cluster 2023-10-09T09:14:40.101447+0000 mgr.y (mgr.25421) 1121 : cluster 0 pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 13700/231660 objects degraded (5.914%); 18070/231660 objects misplaced (7.800%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:14:41.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:41 smithi016 bash[77892]: cluster 2023-10-09T09:14:40.101447+0000 mgr.y (mgr.25421) 1121 : cluster 0 pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 13700/231660 objects degraded (5.914%); 18070/231660 objects misplaced (7.800%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:14:41.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:41.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:41.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:41.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:41.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:41.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:41.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:41.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:41.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:41.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:41.880 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:41.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:41.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:42.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:42.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:42.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:42.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:42.183 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:42.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:42.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:42.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:42.334 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:42.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:42.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:42.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:42.486 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:42.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:42.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:42.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:42.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:42.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:42.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:42.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:42.760 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:42.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:42.864 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:42.912 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:42.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:43.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:43.037 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:43.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:43.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:43.170 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:43.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:43.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:43.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:43.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:43.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:43.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:43.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:43 smithi094 bash[59677]: cluster 2023-10-09T09:14:42.102131+0000 mgr.y (mgr.25421) 1122 : cluster 0 pgmap v1208: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 656 op/s; 13776/225165 objects degraded (6.118%); 17143/225165 objects misplaced (7.614%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T09:14:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:43 smithi094 bash[59677]: audit 2023-10-09T09:14:42.809643+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:43 smithi094 bash[59677]: audit 2023-10-09T09:14:42.814859+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:43 smithi094 bash[59677]: audit 2023-10-09T09:14:42.815852+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:43 smithi094 bash[59677]: audit 2023-10-09T09:14:42.816694+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:43 smithi094 bash[59677]: audit 2023-10-09T09:14:42.821405+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.533 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:43.570 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:43.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:43.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:43.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[75179]: cluster 2023-10-09T09:14:42.102131+0000 mgr.y (mgr.25421) 1122 : cluster 0 pgmap v1208: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 656 op/s; 13776/225165 objects degraded (6.118%); 17143/225165 objects misplaced (7.614%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T09:14:43.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[75179]: audit 2023-10-09T09:14:42.809643+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[75179]: audit 2023-10-09T09:14:42.814859+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[75179]: audit 2023-10-09T09:14:42.815852+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[75179]: audit 2023-10-09T09:14:42.816694+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[75179]: audit 2023-10-09T09:14:42.821405+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[77892]: cluster 2023-10-09T09:14:42.102131+0000 mgr.y (mgr.25421) 1122 : cluster 0 pgmap v1208: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 656 op/s; 13776/225165 objects degraded (6.118%); 17143/225165 objects misplaced (7.614%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[77892]: audit 2023-10-09T09:14:42.809643+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[77892]: audit 2023-10-09T09:14:42.814859+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[77892]: audit 2023-10-09T09:14:42.815852+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[77892]: audit 2023-10-09T09:14:42.816694+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:43.713 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:43 smithi016 bash[77892]: audit 2023-10-09T09:14:42.821405+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:43.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:43.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:43.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:43.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:43.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:44.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:44.087 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:44.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:44.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:44.193 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:44.218 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:44.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:44.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:44.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:44.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:44.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:44.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:44.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:44.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:44.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:44.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:44.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:44.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:44.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:44.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:44.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:44.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:44.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:44.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:44.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:44.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:45.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:45.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:45.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:45.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:45.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:45.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:45.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:45.401 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:45 smithi016 bash[75179]: cluster 2023-10-09T09:14:44.102681+0000 mgr.y (mgr.25421) 1123 : cluster 0 pgmap v1209: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 13776/223398 objects degraded (6.167%); 16965/223398 objects misplaced (7.594%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:14:45.401 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:45 smithi016 bash[75179]: cluster 2023-10-09T09:14:44.733623+0000 mon.a (mon.0) 1495 : cluster 3 Health check update: Degraded data redundancy: 13776/223398 objects degraded (6.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:45.402 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:45 smithi016 bash[77892]: cluster 2023-10-09T09:14:44.102681+0000 mgr.y (mgr.25421) 1123 : cluster 0 pgmap v1209: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 13776/223398 objects degraded (6.167%); 16965/223398 objects misplaced (7.594%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:14:45.402 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:45 smithi016 bash[77892]: cluster 2023-10-09T09:14:44.733623+0000 mon.a (mon.0) 1495 : cluster 3 Health check update: Degraded data redundancy: 13776/223398 objects degraded (6.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:45.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:45.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:45.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:45.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:45.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:45.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:45 smithi094 bash[59677]: cluster 2023-10-09T09:14:44.102681+0000 mgr.y (mgr.25421) 1123 : cluster 0 pgmap v1209: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 13776/223398 objects degraded (6.167%); 16965/223398 objects misplaced (7.594%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:14:45.655 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:45 smithi094 bash[59677]: cluster 2023-10-09T09:14:44.733623+0000 mon.a (mon.0) 1495 : cluster 3 Health check update: Degraded data redundancy: 13776/223398 objects degraded (6.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:45.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:45.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:45.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:45.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:45.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:45.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:45.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:14:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:14:45] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T09:14:46.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:46.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:46.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:46.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:46.209 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:46.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:46.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:46.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:46.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:46.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:46.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:46.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:46.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:46.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:46.595 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:46 smithi016 bash[75179]: debug 2023-10-09T09:14:46.501+0000 7fa046a4d700 -1 mon.a@0(leader).osd e263 definitely_dead 0 2023-10-09T09:14:46.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:46.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:46.716 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:46.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:46.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:46.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:46.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:46.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:46.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:46.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:47.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.004 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:47.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:47.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:47.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:47.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:47.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:47.297 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.1. 2023-10-09T09:14:47.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:47.368 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:47 smithi016 bash[75179]: cluster 2023-10-09T09:14:46.103410+0000 mgr.y (mgr.25421) 1124 : cluster 0 pgmap v1210: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 13887/216261 objects degraded (6.421%); 16102/216261 objects misplaced (7.446%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T09:14:47.368 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:47 smithi016 bash[75179]: audit 2023-10-09T09:14:46.504132+0000 mon.a (mon.0) 1496 : audit 0 from='client.? 172.21.15.16:0/1440605364' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T09:14:47.368 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:47 smithi016 bash[77892]: cluster 2023-10-09T09:14:46.103410+0000 mgr.y (mgr.25421) 1124 : cluster 0 pgmap v1210: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 13887/216261 objects degraded (6.421%); 16102/216261 objects misplaced (7.446%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T09:14:47.368 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:47 smithi016 bash[77892]: audit 2023-10-09T09:14:46.504132+0000 mon.a (mon.0) 1496 : audit 0 from='client.? 172.21.15.16:0/1440605364' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T09:14:47.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:47.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:47.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:47.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:47.621 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:47 smithi094 bash[59677]: cluster 2023-10-09T09:14:46.103410+0000 mgr.y (mgr.25421) 1124 : cluster 0 pgmap v1210: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 13887/216261 objects degraded (6.421%); 16102/216261 objects misplaced (7.446%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T09:14:47.621 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:47 smithi094 bash[59677]: audit 2023-10-09T09:14:46.504132+0000 mon.a (mon.0) 1496 : audit 0 from='client.? 172.21.15.16:0/1440605364' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T09:14:47.639 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:47.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:47.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:47.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:47.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:47.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:47.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:48.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:48.030 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:48.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:48.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:48.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:48.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:48.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:48.285 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.276695+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.285 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: audit 2023-10-09T09:14:47.279082+0000 mon.a (mon.0) 1497 : audit 1 from='client.? 172.21.15.16:0/1440605364' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T09:14:48.285 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.279206+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T09:14:48.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.279422+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.281301+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.286 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.281935+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:48.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.276695+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: audit 2023-10-09T09:14:47.279082+0000 mon.a (mon.0) 1497 : audit 1 from='client.? 172.21.15.16:0/1440605364' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T09:14:48.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.279206+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T09:14:48.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.279422+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.281301+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.281935+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.283128+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.284104+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.285403+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.294006+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.295079+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[75179]: cluster 2023-10-09T09:14:47.296086+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.353 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.276695+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.353 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: audit 2023-10-09T09:14:47.279082+0000 mon.a (mon.0) 1497 : audit 1 from='client.? 172.21.15.16:0/1440605364' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T09:14:48.353 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.279206+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T09:14:48.353 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.279422+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.353 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.281301+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.353 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.281935+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.353 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.283128+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.284104+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.285403+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.294006+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.295079+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:48 smithi016 bash[77892]: cluster 2023-10-09T09:14:47.296086+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.406 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:48.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:48.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:48.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:48.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:48.662 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:48.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.283128+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.284104+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.285403+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.294006+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.295079+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:48 smithi094 bash[59677]: cluster 2023-10-09T09:14:47.296086+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T09:14:48.690 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:48.793 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:48.819 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:48.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:48.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:48.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:48.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:49.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:49.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:49.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:49.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:49.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:49.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:49.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:49.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.104040+0000 mgr.y (mgr.25421) 1125 : cluster 0 pgmap v1212: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 563 op/s; 13887/216261 objects degraded (6.421%); 16102/216261 objects misplaced (7.446%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T09:14:49.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.283713+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.288812+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.289111+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.289260+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.290227+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.290454+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.293200+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.294430+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: cluster 2023-10-09T09:14:48.295266+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.460 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[75179]: audit 2023-10-09T09:14:49.248598+0000 mon.a (mon.0) 1500 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.104040+0000 mgr.y (mgr.25421) 1125 : cluster 0 pgmap v1212: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 563 op/s; 13887/216261 objects degraded (6.421%); 16102/216261 objects misplaced (7.446%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.283713+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.288812+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.289111+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.289260+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.290227+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.290454+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.461 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.293200+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.294430+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: cluster 2023-10-09T09:14:48.295266+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:49 smithi016 bash[77892]: audit 2023-10-09T09:14:49.248598+0000 mon.a (mon.0) 1500 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:49.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:49.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:49.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:49.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:49.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:49.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:49.713 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.104040+0000 mgr.y (mgr.25421) 1125 : cluster 0 pgmap v1212: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 563 op/s; 13887/216261 objects degraded (6.421%); 16102/216261 objects misplaced (7.446%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T09:14:49.713 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.283713+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.713 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.288812+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T09:14:49.713 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.289111+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.713 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.289260+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.290227+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.290454+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.293200+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.294430+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: cluster 2023-10-09T09:14:48.295266+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T09:14:49.714 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:49 smithi094 bash[59677]: audit 2023-10-09T09:14:49.248598+0000 mon.a (mon.0) 1500 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:49.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:49.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:49.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:49.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:49.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:49.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:50.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:50.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:50.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:50.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:50.270 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:14:50.297 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:14:50.300 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:50.319 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:14:50.349 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:14:50.376 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:14:50.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:50.417 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:14:50.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:50.457 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:14:50.486 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:14:50.490 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.490 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.491 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.491 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.491 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.491 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.491 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.492 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:14:50.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:50.548 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.291157+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.292737+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.293485+0000 mon.a (mon.0) 1501 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.294197+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.295129+0000 osd.0 (osd.0) 28 : cluster 0 10.2 starting backfill to osd.1 from (0'0,0'0] MAX to 263'8938 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.296388+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.299751+0000 osd.6 (osd.6) 209 : cluster 0 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 263'9042 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.308735+0000 osd.6 (osd.6) 210 : cluster 0 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 263'8974 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: cluster 2023-10-09T09:14:49.734837+0000 mon.a (mon.0) 1502 : cluster 3 Health check update: Degraded data redundancy: 13887/216261 objects degraded (6.421%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:50.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[75179]: audit 2023-10-09T09:14:50.235341+0000 mon.a (mon.0) 1503 : audit 0 from='client.? 172.21.15.16:0/603484557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.291157+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.292737+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.293485+0000 mon.a (mon.0) 1501 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.294197+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.295129+0000 osd.0 (osd.0) 28 : cluster 0 10.2 starting backfill to osd.1 from (0'0,0'0] MAX to 263'8938 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.296388+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.299751+0000 osd.6 (osd.6) 209 : cluster 0 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 263'9042 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.308735+0000 osd.6 (osd.6) 210 : cluster 0 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 263'8974 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: cluster 2023-10-09T09:14:49.734837+0000 mon.a (mon.0) 1502 : cluster 3 Health check update: Degraded data redundancy: 13887/216261 objects degraded (6.421%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:50.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:50 smithi016 bash[77892]: audit 2023-10-09T09:14:50.235341+0000 mon.a (mon.0) 1503 : audit 0 from='client.? 172.21.15.16:0/603484557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:14:50.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:50.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:50.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.291157+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.292737+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.293485+0000 mon.a (mon.0) 1501 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T09:14:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.294197+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.295129+0000 osd.0 (osd.0) 28 : cluster 0 10.2 starting backfill to osd.1 from (0'0,0'0] MAX to 263'8938 2023-10-09T09:14:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.296388+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T09:14:50.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.299751+0000 osd.6 (osd.6) 209 : cluster 0 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 263'9042 2023-10-09T09:14:50.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.308735+0000 osd.6 (osd.6) 210 : cluster 0 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 263'8974 2023-10-09T09:14:50.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: cluster 2023-10-09T09:14:49.734837+0000 mon.a (mon.0) 1502 : cluster 3 Health check update: Degraded data redundancy: 13887/216261 objects degraded (6.421%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:50.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:50 smithi094 bash[59677]: audit 2023-10-09T09:14:50.235341+0000 mon.a (mon.0) 1503 : audit 0 from='client.? 172.21.15.16:0/603484557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:14:50.801 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:50.830 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:50.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:50.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:50.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:50.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:51.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:51.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:51.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:51.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:51.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:51.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:51.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:51.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:51.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:51.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:51.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:51.492 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:51.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:51.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:51 smithi016 bash[77892]: cluster 2023-10-09T09:14:50.104530+0000 mgr.y (mgr.25421) 1126 : cluster 0 pgmap v1215: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 482 op/s; 8112/214704 objects degraded (3.778%); 20295/214704 objects misplaced (9.453%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:14:51.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:51 smithi016 bash[75179]: cluster 2023-10-09T09:14:50.104530+0000 mgr.y (mgr.25421) 1126 : cluster 0 pgmap v1215: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 482 op/s; 8112/214704 objects degraded (3.778%); 20295/214704 objects misplaced (9.453%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:14:51.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:51.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:51.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:51.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:51.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:51.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:51 smithi094 bash[59677]: cluster 2023-10-09T09:14:50.104530+0000 mgr.y (mgr.25421) 1126 : cluster 0 pgmap v1215: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 482 op/s; 8112/214704 objects degraded (3.778%); 20295/214704 objects misplaced (9.453%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:14:51.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:51.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:51.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:51.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:51.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:51.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:51.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:51.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:52.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:52.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:52.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:52.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:52.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:52.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:52.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:52.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:52.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:52.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:52.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:52.677 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:52.782 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:52.791 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T09:14:52.825 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:52.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:52.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:52.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:52.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:53.057 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:53.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:53.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:53.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:53.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:53.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:53.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:53.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:53.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:53.345 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:53 smithi094 bash[59677]: cluster 2023-10-09T09:14:52.105221+0000 mgr.y (mgr.25421) 1127 : cluster 0 pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 13915/211047 objects degraded (6.593%); 23263/211047 objects misplaced (11.023%); 14 MiB/s, 223 objects/s recovering 2023-10-09T09:14:53.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:53.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:53.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:53.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:53.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:53.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:53.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:53.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:53.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:53 smithi016 bash[75179]: cluster 2023-10-09T09:14:52.105221+0000 mgr.y (mgr.25421) 1127 : cluster 0 pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 13915/211047 objects degraded (6.593%); 23263/211047 objects misplaced (11.023%); 14 MiB/s, 223 objects/s recovering 2023-10-09T09:14:53.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:53 smithi016 bash[77892]: cluster 2023-10-09T09:14:52.105221+0000 mgr.y (mgr.25421) 1127 : cluster 0 pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 13915/211047 objects degraded (6.593%); 23263/211047 objects misplaced (11.023%); 14 MiB/s, 223 objects/s recovering 2023-10-09T09:14:53.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:53.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:53.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:53.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:53.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:53.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:53.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:53.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:53.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:53.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:54.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:54.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:54.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:54.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:54.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:54.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:54.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:54.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:54.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:54.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:54.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:54.403 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:54.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:54.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:54.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:54.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:54.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:54.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:54.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:54.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:54.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:54.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:54.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:54.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:54.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:54.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:55.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:55.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:55.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:55.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:55.294 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:55.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:55.400 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:55 smithi016 bash[75179]: cluster 2023-10-09T09:14:54.105741+0000 mgr.y (mgr.25421) 1128 : cluster 0 pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 332 op/s; 13915/209442 objects degraded (6.644%); 27210/209442 objects misplaced (12.992%); 13 MiB/s, 215 objects/s recovering 2023-10-09T09:14:55.400 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:55 smithi016 bash[75179]: cluster 2023-10-09T09:14:54.736024+0000 mon.a (mon.0) 1504 : cluster 3 Health check update: Degraded data redundancy: 13915/209442 objects degraded (6.644%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:55.400 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:55 smithi016 bash[75179]: audit 2023-10-09T09:14:55.152335+0000 mon.a (mon.0) 1505 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:55.401 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:55 smithi016 bash[77892]: cluster 2023-10-09T09:14:54.105741+0000 mgr.y (mgr.25421) 1128 : cluster 0 pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 332 op/s; 13915/209442 objects degraded (6.644%); 27210/209442 objects misplaced (12.992%); 13 MiB/s, 215 objects/s recovering 2023-10-09T09:14:55.401 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:55 smithi016 bash[77892]: cluster 2023-10-09T09:14:54.736024+0000 mon.a (mon.0) 1504 : cluster 3 Health check update: Degraded data redundancy: 13915/209442 objects degraded (6.644%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:55.401 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:55 smithi016 bash[77892]: audit 2023-10-09T09:14:55.152335+0000 mon.a (mon.0) 1505 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:55.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:55.530 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:55.538 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:55 smithi094 bash[59677]: cluster 2023-10-09T09:14:54.105741+0000 mgr.y (mgr.25421) 1128 : cluster 0 pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 332 op/s; 13915/209442 objects degraded (6.644%); 27210/209442 objects misplaced (12.992%); 13 MiB/s, 215 objects/s recovering 2023-10-09T09:14:55.538 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:55 smithi094 bash[59677]: cluster 2023-10-09T09:14:54.736024+0000 mon.a (mon.0) 1504 : cluster 3 Health check update: Degraded data redundancy: 13915/209442 objects degraded (6.644%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:55.538 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:55 smithi094 bash[59677]: audit 2023-10-09T09:14:55.152335+0000 mon.a (mon.0) 1505 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:14:55.563 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:55.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:55.675 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:14:55.675 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:14:55.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:55.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:55.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:55.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:55.946 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:55.982 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:55.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:55.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:14:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:14:55] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T09:14:56.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:56.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:56.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:56.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:56.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:56.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:56.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:56.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:56.316 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:14:56.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:56.360 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:56 smithi016 bash[75179]: audit 2023-10-09T09:14:56.038477+0000 mon.a (mon.0) 1506 : audit 1 from='client.? 172.21.15.16:0/1686326120' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:14:56.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:56 smithi016 bash[77892]: audit 2023-10-09T09:14:56.038477+0000 mon.a (mon.0) 1506 : audit 1 from='client.? 172.21.15.16:0/1686326120' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:14:56.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:56.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:56.494 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:56.541 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:56.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:56.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:56.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:56.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:56 smithi094 bash[59677]: audit 2023-10-09T09:14:56.038477+0000 mon.a (mon.0) 1506 : audit 1 from='client.? 172.21.15.16:0/1686326120' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:14:56.697 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:56.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:56.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:56.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:56.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:56.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:56.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:14:56.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:56.968 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:56.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:57.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:57.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:57.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:57.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:57.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:57.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:57.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:57.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:57.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:57.360 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:57.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:57.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:57.473 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[75179]: cluster 2023-10-09T09:14:56.108304+0000 mgr.y (mgr.25421) 1129 : cluster 0 pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 13915/202821 objects degraded (6.861%); 26275/202821 objects misplaced (12.955%); 14 MiB/s, 231 objects/s recovering 2023-10-09T09:14:57.473 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[75179]: cluster 2023-10-09T09:14:56.299527+0000 mon.a (mon.0) 1507 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:14:57.473 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[75179]: cluster 2023-10-09T09:14:56.309247+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T09:14:57.473 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[75179]: cluster 2023-10-09T09:14:56.311116+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[75179]: audit 2023-10-09T09:14:56.311121+0000 mon.a (mon.0) 1508 : audit 1 from='client.? 172.21.15.16:0/1686326120' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[75179]: cluster 2023-10-09T09:14:56.311217+0000 mon.a (mon.0) 1509 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[77892]: cluster 2023-10-09T09:14:56.108304+0000 mgr.y (mgr.25421) 1129 : cluster 0 pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 13915/202821 objects degraded (6.861%); 26275/202821 objects misplaced (12.955%); 14 MiB/s, 231 objects/s recovering 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[77892]: cluster 2023-10-09T09:14:56.299527+0000 mon.a (mon.0) 1507 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[77892]: cluster 2023-10-09T09:14:56.309247+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[77892]: cluster 2023-10-09T09:14:56.311116+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[77892]: audit 2023-10-09T09:14:56.311121+0000 mon.a (mon.0) 1508 : audit 1 from='client.? 172.21.15.16:0/1686326120' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:14:57.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:57 smithi016 bash[77892]: cluster 2023-10-09T09:14:56.311217+0000 mon.a (mon.0) 1509 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T09:14:57.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:57.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:57.606 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:57.615 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:57 smithi094 bash[59677]: cluster 2023-10-09T09:14:56.108304+0000 mgr.y (mgr.25421) 1129 : cluster 0 pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 13915/202821 objects degraded (6.861%); 26275/202821 objects misplaced (12.955%); 14 MiB/s, 231 objects/s recovering 2023-10-09T09:14:57.615 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:57 smithi094 bash[59677]: cluster 2023-10-09T09:14:56.299527+0000 mon.a (mon.0) 1507 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:14:57.615 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:57 smithi094 bash[59677]: cluster 2023-10-09T09:14:56.309247+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T09:14:57.615 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:57 smithi094 bash[59677]: cluster 2023-10-09T09:14:56.311116+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T09:14:57.616 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:57 smithi094 bash[59677]: audit 2023-10-09T09:14:56.311121+0000 mon.a (mon.0) 1508 : audit 1 from='client.? 172.21.15.16:0/1686326120' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:14:57.616 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:57 smithi094 bash[59677]: cluster 2023-10-09T09:14:56.311217+0000 mon.a (mon.0) 1509 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T09:14:57.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:57.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:57.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:57.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:57.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:57.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:57.792 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:14:57.793 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:14:57.793 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.7858663985604675 2023-10-09T09:14:57.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 0.7858663985604675 2023-10-09T09:14:57.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:57.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:14:57.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:57.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:14:58.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:58.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:14:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:58.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:14:58.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:58.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:58.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:58.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:14:58.310 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:58.349 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:58.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:58.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:58.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:14:58.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:58.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:14:58.596 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:58.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:14:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:58.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:14:58.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:58.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:58.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:58.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:58.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:58.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:58.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:59.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:59.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:59.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:14:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:59.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:14:59.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:59.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:59.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:59.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:59.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:59.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:14:59.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:14:59 smithi094 bash[59677]: cluster 2023-10-09T09:14:58.108862+0000 mgr.y (mgr.25421) 1130 : cluster 0 pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 13915/202821 objects degraded (6.861%); 26275/202821 objects misplaced (12.955%); 13 MiB/s, 208 objects/s recovering 2023-10-09T09:14:59.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:59.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:14:59.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:59.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:14:59 smithi016 bash[75179]: cluster 2023-10-09T09:14:58.108862+0000 mgr.y (mgr.25421) 1130 : cluster 0 pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 13915/202821 objects degraded (6.861%); 26275/202821 objects misplaced (12.955%); 13 MiB/s, 208 objects/s recovering 2023-10-09T09:14:59.712 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:14:59 smithi016 bash[77892]: cluster 2023-10-09T09:14:58.108862+0000 mgr.y (mgr.25421) 1130 : cluster 0 pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 13915/202821 objects degraded (6.861%); 26275/202821 objects misplaced (12.955%); 13 MiB/s, 208 objects/s recovering 2023-10-09T09:14:59.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:14:59.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:59.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:14:59.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:14:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:14:59.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:14:59.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:00.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:00.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:00.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:00.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:00.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:00.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:00.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:00.378 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:00.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:00.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:00.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:00.528 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:00 smithi016 bash[75179]: cluster 2023-10-09T09:14:59.737255+0000 mon.a (mon.0) 1510 : cluster 3 Health check update: Degraded data redundancy: 13915/202821 objects degraded (6.861%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:00.528 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:00 smithi016 bash[77892]: cluster 2023-10-09T09:14:59.737255+0000 mon.a (mon.0) 1510 : cluster 3 Health check update: Degraded data redundancy: 13915/202821 objects degraded (6.861%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:00.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:00.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:00.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:00.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:00 smithi094 bash[59677]: cluster 2023-10-09T09:14:59.737255+0000 mon.a (mon.0) 1510 : cluster 3 Health check update: Degraded data redundancy: 13915/202821 objects degraded (6.861%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:00.696 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:00.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:00.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:00.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:00.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:00.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:00.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:01.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:15:01.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:01.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:01.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:01.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:01.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:01.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:01.372 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:01.380 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:01 smithi094 bash[59677]: cluster 2023-10-09T09:15:00.109603+0000 mgr.y (mgr.25421) 1131 : cluster 0 pgmap v1221: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 13957/200637 objects degraded (6.956%); 25745/200637 objects misplaced (12.832%); 12 MiB/s, 194 objects/s recovering 2023-10-09T09:15:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:01.404 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:01.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:01.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:01 smithi016 bash[75179]: cluster 2023-10-09T09:15:00.109603+0000 mgr.y (mgr.25421) 1131 : cluster 0 pgmap v1221: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 13957/200637 objects degraded (6.956%); 25745/200637 objects misplaced (12.832%); 12 MiB/s, 194 objects/s recovering 2023-10-09T09:15:01.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:01 smithi016 bash[77892]: cluster 2023-10-09T09:15:00.109603+0000 mgr.y (mgr.25421) 1131 : cluster 0 pgmap v1221: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 13957/200637 objects degraded (6.956%); 25745/200637 objects misplaced (12.832%); 12 MiB/s, 194 objects/s recovering 2023-10-09T09:15:01.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:01.532 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:01.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:01.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:01.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:01.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:01.701 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:01.708 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:01.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:01.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:01.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:01.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:01.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:01.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:01.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:01.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:02.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:02.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:02.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:02.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:02.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:02.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:02.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:02.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:02.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:02.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:02.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:02.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:02.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:02.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:02.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:02.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:02.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:02.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:02.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:02.722 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:02.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:02.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:02.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:02.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:02.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:02.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:03.045 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:03.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:03.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:03.175 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:03.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:03.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:03.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:03.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:03.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:03.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:03.583 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:03 smithi016 bash[75179]: cluster 2023-10-09T09:15:02.110333+0000 mgr.y (mgr.25421) 1132 : cluster 0 pgmap v1222: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 13957/194208 objects degraded (7.187%); 24757/194208 objects misplaced (12.748%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T09:15:03.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:03.584 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:03 smithi016 bash[77892]: cluster 2023-10-09T09:15:02.110333+0000 mgr.y (mgr.25421) 1132 : cluster 0 pgmap v1222: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 13957/194208 objects degraded (7.187%); 24757/194208 objects misplaced (12.748%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T09:15:03.685 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:03.695 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:03 smithi094 bash[59677]: cluster 2023-10-09T09:15:02.110333+0000 mgr.y (mgr.25421) 1132 : cluster 0 pgmap v1222: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 13957/194208 objects degraded (7.187%); 24757/194208 objects misplaced (12.748%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T09:15:03.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:03.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:03.822 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:03.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:03.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:03.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:03.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:03.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:04.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:04.117 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:04.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:04.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:04.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:04.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:04.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:04.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:04.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:04.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:04.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:04.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:04.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:04 smithi094 bash[59677]: audit 2023-10-09T09:15:04.248954+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:04.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:04.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:04.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:04.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:04.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:04.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:04.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:04.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:04 smithi016 bash[75179]: audit 2023-10-09T09:15:04.248954+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:04.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:04 smithi016 bash[77892]: audit 2023-10-09T09:15:04.248954+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:04.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:04.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:04.890 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:04.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:04.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:04.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:05.022 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:05.059 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:05.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:05.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:05.203 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:05.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:05.311 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:05.335 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:05.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:05.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:05.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:05.469 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:05.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:05.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:05.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:05.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:05.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:05.613 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:05 smithi016 bash[75179]: cluster 2023-10-09T09:15:04.111089+0000 mgr.y (mgr.25421) 1133 : cluster 0 pgmap v1223: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 14006/192105 objects degraded (7.291%); 24252/192105 objects misplaced (12.624%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T09:15:05.613 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:05 smithi016 bash[75179]: cluster 2023-10-09T09:15:04.738471+0000 mon.a (mon.0) 1512 : cluster 3 Health check update: Degraded data redundancy: 14006/192105 objects degraded (7.291%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:05.614 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:05 smithi016 bash[77892]: cluster 2023-10-09T09:15:04.111089+0000 mgr.y (mgr.25421) 1133 : cluster 0 pgmap v1223: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 14006/192105 objects degraded (7.291%); 24252/192105 objects misplaced (12.624%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T09:15:05.614 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:05 smithi016 bash[77892]: cluster 2023-10-09T09:15:04.738471+0000 mon.a (mon.0) 1512 : cluster 3 Health check update: Degraded data redundancy: 14006/192105 objects degraded (7.291%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:05.714 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:05.724 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:05 smithi094 bash[59677]: cluster 2023-10-09T09:15:04.111089+0000 mgr.y (mgr.25421) 1133 : cluster 0 pgmap v1223: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 14006/192105 objects degraded (7.291%); 24252/192105 objects misplaced (12.624%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T09:15:05.725 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:05 smithi094 bash[59677]: cluster 2023-10-09T09:15:04.738471+0000 mon.a (mon.0) 1512 : cluster 3 Health check update: Degraded data redundancy: 14006/192105 objects degraded (7.291%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:05.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:05.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:05.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:05.764 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:05.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:05.866 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:15:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:15:05] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.33.4" 2023-10-09T09:15:05.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:05.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:05.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:06.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:06.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:06.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:06.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:06.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:06.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:06.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:06.395 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:06.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:06.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:06.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:06.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:06.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:06.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:06.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:06.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:06.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:06.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:06.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:06.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:06.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:07.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:07.124 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:07.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:07.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:07.267 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:07.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:07.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:07.371 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:07.380 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:07 smithi094 bash[59677]: cluster 2023-10-09T09:15:06.112028+0000 mgr.y (mgr.25421) 1134 : cluster 0 pgmap v1224: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 14006/185763 objects degraded (7.540%); 23404/185763 objects misplaced (12.599%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:15:07.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:07.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:07.510 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:07.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:07.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:07.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:07.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:07.660 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:07.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:07 smithi016 bash[75179]: cluster 2023-10-09T09:15:06.112028+0000 mgr.y (mgr.25421) 1134 : cluster 0 pgmap v1224: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 14006/185763 objects degraded (7.540%); 23404/185763 objects misplaced (12.599%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:15:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:07 smithi016 bash[77892]: cluster 2023-10-09T09:15:06.112028+0000 mgr.y (mgr.25421) 1134 : cluster 0 pgmap v1224: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 14006/185763 objects degraded (7.540%); 23404/185763 objects misplaced (12.599%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:15:07.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:07.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:07.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:07.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:07.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:07.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:08.025 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:08.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:08.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:08.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:08.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:08.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:08.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:08.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:08.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:08.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:08.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:08.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:08.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:08.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:08.690 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[75179]: cluster 2023-10-09T09:15:07.350189+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[75179]: cluster 2023-10-09T09:15:07.350612+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[75179]: cluster 2023-10-09T09:15:07.352489+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[75179]: cluster 2023-10-09T09:15:07.353648+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[75179]: cluster 2023-10-09T09:15:07.355008+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[75179]: cluster 2023-10-09T09:15:07.358206+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[75179]: cluster 2023-10-09T09:15:07.359017+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[77892]: cluster 2023-10-09T09:15:07.350189+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T09:15:08.691 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[77892]: cluster 2023-10-09T09:15:07.350612+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[77892]: cluster 2023-10-09T09:15:07.352489+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[77892]: cluster 2023-10-09T09:15:07.353648+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[77892]: cluster 2023-10-09T09:15:07.355008+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[77892]: cluster 2023-10-09T09:15:07.358206+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:08 smithi016 bash[77892]: cluster 2023-10-09T09:15:07.359017+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:08.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:08 smithi094 bash[59677]: cluster 2023-10-09T09:15:07.350189+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T09:15:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:08 smithi094 bash[59677]: cluster 2023-10-09T09:15:07.350612+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:08 smithi094 bash[59677]: cluster 2023-10-09T09:15:07.352489+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:08 smithi094 bash[59677]: cluster 2023-10-09T09:15:07.353648+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:08 smithi094 bash[59677]: cluster 2023-10-09T09:15:07.355008+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:08 smithi094 bash[59677]: cluster 2023-10-09T09:15:07.358206+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:08 smithi094 bash[59677]: cluster 2023-10-09T09:15:07.359017+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T09:15:08.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:08.857 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:08.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:08.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:08.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:08.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:08.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:09.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:09.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:09.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:09.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:09.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:09.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:09.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:09.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:09.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:09.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:09.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:09.489 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:09.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:09 smithi094 bash[59677]: cluster 2023-10-09T09:15:08.112589+0000 mgr.y (mgr.25421) 1135 : cluster 0 pgmap v1226: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 14006/185763 objects degraded (7.540%); 23404/185763 objects misplaced (12.599%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:15:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:09 smithi094 bash[59677]: cluster 2023-10-09T09:15:08.355758+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T09:15:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:09 smithi094 bash[59677]: cluster 2023-10-09T09:15:08.358648+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T09:15:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:09 smithi094 bash[59677]: cluster 2023-10-09T09:15:08.359031+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T09:15:09.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:09.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:09.625 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:09.685 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:09.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:09.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[75179]: cluster 2023-10-09T09:15:08.112589+0000 mgr.y (mgr.25421) 1135 : cluster 0 pgmap v1226: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 14006/185763 objects degraded (7.540%); 23404/185763 objects misplaced (12.599%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:15:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[75179]: cluster 2023-10-09T09:15:08.355758+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T09:15:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[75179]: cluster 2023-10-09T09:15:08.358648+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T09:15:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[75179]: cluster 2023-10-09T09:15:08.359031+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T09:15:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[77892]: cluster 2023-10-09T09:15:08.112589+0000 mgr.y (mgr.25421) 1135 : cluster 0 pgmap v1226: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 14006/185763 objects degraded (7.540%); 23404/185763 objects misplaced (12.599%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T09:15:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[77892]: cluster 2023-10-09T09:15:08.355758+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T09:15:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[77892]: cluster 2023-10-09T09:15:08.358648+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T09:15:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:09 smithi016 bash[77892]: cluster 2023-10-09T09:15:08.359031+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T09:15:09.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:09.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:09.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:09.918 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:09.954 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:09.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:09.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:10.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:10.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:10.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:10.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:10.230 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:10.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:10.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:10.377 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:10.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:10.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:10.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:10.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:10 smithi094 bash[59677]: cluster 2023-10-09T09:15:09.739342+0000 mon.a (mon.0) 1515 : cluster 3 Health check update: Degraded data redundancy: 14006/185763 objects degraded (7.540%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:10.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:10.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:10.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:10.606 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:10.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:10.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:10.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:10.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:10 smithi016 bash[75179]: cluster 2023-10-09T09:15:09.739342+0000 mon.a (mon.0) 1515 : cluster 3 Health check update: Degraded data redundancy: 14006/185763 objects degraded (7.540%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:10.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:10 smithi016 bash[77892]: cluster 2023-10-09T09:15:09.739342+0000 mon.a (mon.0) 1515 : cluster 3 Health check update: Degraded data redundancy: 14006/185763 objects degraded (7.540%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:10.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:10.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:10.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:10.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:10.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:10.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:11.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:11.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:11.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:11.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:11.171 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:11.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:11.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:11.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:11.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:11.358 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:11 smithi094 bash[59677]: cluster 2023-10-09T09:15:10.113277+0000 mgr.y (mgr.25421) 1136 : cluster 0 pgmap v1228: 89 pgs: 2 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 14089/183648 objects degraded (7.672%); 22465/183648 objects misplaced (12.233%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:15:11.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:11.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:11.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:11.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:11 smithi016 bash[75179]: cluster 2023-10-09T09:15:10.113277+0000 mgr.y (mgr.25421) 1136 : cluster 0 pgmap v1228: 89 pgs: 2 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 14089/183648 objects degraded (7.672%); 22465/183648 objects misplaced (12.233%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:15:11.557 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:11 smithi016 bash[77892]: cluster 2023-10-09T09:15:10.113277+0000 mgr.y (mgr.25421) 1136 : cluster 0 pgmap v1228: 89 pgs: 2 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 14089/183648 objects degraded (7.672%); 22465/183648 objects misplaced (12.233%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:15:11.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:11.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:11.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:11.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:11.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:11.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:11.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:11.987 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:11.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:12.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:12.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:12.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:12.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:12.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:12.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:12.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:12.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:12.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:12.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:12.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:12.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:12.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:12.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:12.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:12.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:12.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:12.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:12.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:12.832 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:12.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:12.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:12.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:12.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:13.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:13.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:13.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:13.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:13.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:13.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:13.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:13 smithi016 bash[75179]: cluster 2023-10-09T09:15:12.113956+0000 mgr.y (mgr.25421) 1137 : cluster 0 pgmap v1229: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 14089/177831 objects degraded (7.923%); 21913/177831 objects misplaced (12.322%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T09:15:13.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:13.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:13.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:13.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:13.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:13.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:13.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:13.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:13 smithi016 bash[77892]: cluster 2023-10-09T09:15:12.113956+0000 mgr.y (mgr.25421) 1137 : cluster 0 pgmap v1229: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 14089/177831 objects degraded (7.923%); 21913/177831 objects misplaced (12.322%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T09:15:13.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:13.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:13 smithi094 bash[59677]: cluster 2023-10-09T09:15:12.113956+0000 mgr.y (mgr.25421) 1137 : cluster 0 pgmap v1229: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 14089/177831 objects degraded (7.923%); 21913/177831 objects misplaced (12.322%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T09:15:13.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:13.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:13.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:13.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:13.940 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:13.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:14.046 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:14.082 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:14.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:14.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:14.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:14.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:14.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:14.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:14.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:14.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:14.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:14.497 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:14.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:14.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:14.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:14.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:14.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:14.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:14.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:14.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:14.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:14.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:14.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:14.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:14.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:15.234 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:15.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:15.340 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:15.377 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:15.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:15.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:15.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:15.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:15 smithi094 bash[59677]: cluster 2023-10-09T09:15:14.114590+0000 mgr.y (mgr.25421) 1138 : cluster 0 pgmap v1230: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 14153/175818 objects degraded (8.050%); 20867/175818 objects misplaced (11.869%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T09:15:15.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:15 smithi094 bash[59677]: cluster 2023-10-09T09:15:14.740349+0000 mon.a (mon.0) 1516 : cluster 3 Health check update: Degraded data redundancy: 14153/175818 objects degraded (8.050%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:15.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:15.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:15.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:15.655 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:15.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:15 smithi016 bash[75179]: cluster 2023-10-09T09:15:14.114590+0000 mgr.y (mgr.25421) 1138 : cluster 0 pgmap v1230: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 14153/175818 objects degraded (8.050%); 20867/175818 objects misplaced (11.869%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T09:15:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:15 smithi016 bash[75179]: cluster 2023-10-09T09:15:14.740349+0000 mon.a (mon.0) 1516 : cluster 3 Health check update: Degraded data redundancy: 14153/175818 objects degraded (8.050%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:15 smithi016 bash[77892]: cluster 2023-10-09T09:15:14.114590+0000 mgr.y (mgr.25421) 1138 : cluster 0 pgmap v1230: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 14153/175818 objects degraded (8.050%); 20867/175818 objects misplaced (11.869%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T09:15:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:15 smithi016 bash[77892]: cluster 2023-10-09T09:15:14.740349+0000 mon.a (mon.0) 1516 : cluster 3 Health check update: Degraded data redundancy: 14153/175818 objects degraded (8.050%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:15.759 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:15.788 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:15.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:15.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:15.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:15.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:16.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:16.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:16.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:16.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:16.063 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:15:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:15:15] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.33.4" 2023-10-09T09:15:16.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:16.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:16.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:16.291 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:16.323 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:16.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:16.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:16.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:16.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:16.565 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:16.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:16.600 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:16.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:16.701 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:16.739 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:16.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:16.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:16.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:16.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:16.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:17.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:17.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:17.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:17.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:17.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:17.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:17.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:17.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:17 smithi094 bash[59677]: cluster 2023-10-09T09:15:16.115242+0000 mgr.y (mgr.25421) 1139 : cluster 0 pgmap v1231: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 14153/169449 objects degraded (8.352%); 20331/169449 objects misplaced (11.998%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T09:15:17.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:17.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:17.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:17.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:17.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:17.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:17 smithi016 bash[75179]: cluster 2023-10-09T09:15:16.115242+0000 mgr.y (mgr.25421) 1139 : cluster 0 pgmap v1231: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 14153/169449 objects degraded (8.352%); 20331/169449 objects misplaced (11.998%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T09:15:17.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:17 smithi016 bash[77892]: cluster 2023-10-09T09:15:16.115242+0000 mgr.y (mgr.25421) 1139 : cluster 0 pgmap v1231: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 14153/169449 objects degraded (8.352%); 20331/169449 objects misplaced (11.998%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T09:15:17.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:17.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:17.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:17.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:17.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:17.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:17.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:18.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:18.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:18.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:18.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:18.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:18.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:18.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:18.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:18.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:18.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:18.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:18.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:18.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:18.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:18.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:18.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:18.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:18.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:18.729 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:18.759 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:18.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:18.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:18.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:18.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:18.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:18.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:18.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:19.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:19.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:19.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:19.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:19.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:19.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:19.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:19.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:19.300 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:19.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:19.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:19.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:19.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:19 smithi016 bash[77892]: cluster 2023-10-09T09:15:18.115941+0000 mgr.y (mgr.25421) 1140 : cluster 0 pgmap v1232: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 14153/169449 objects degraded (8.352%); 20331/169449 objects misplaced (11.998%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T09:15:19.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:19 smithi016 bash[77892]: audit 2023-10-09T09:15:19.249539+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:19.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:19 smithi016 bash[75179]: cluster 2023-10-09T09:15:18.115941+0000 mgr.y (mgr.25421) 1140 : cluster 0 pgmap v1232: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 14153/169449 objects degraded (8.352%); 20331/169449 objects misplaced (11.998%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T09:15:19.417 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:19 smithi016 bash[75179]: audit 2023-10-09T09:15:19.249539+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:19.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:19.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:19.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:19.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:19.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:19.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:19.676 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:19.685 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:19 smithi094 bash[59677]: cluster 2023-10-09T09:15:18.115941+0000 mgr.y (mgr.25421) 1140 : cluster 0 pgmap v1232: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 14153/169449 objects degraded (8.352%); 20331/169449 objects misplaced (11.998%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T09:15:19.685 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:19 smithi094 bash[59677]: audit 2023-10-09T09:15:19.249539+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:19.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:19.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:19.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:19.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:19.856 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:19.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:19.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:19.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:20.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:20.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:20.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:20.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:20.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:20.246 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:20.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:20.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:20.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:20.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:20.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:20.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:20.662 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:20 smithi016 bash[75179]: cluster 2023-10-09T09:15:19.741396+0000 mon.a (mon.0) 1518 : cluster 3 Health check update: Degraded data redundancy: 14153/169449 objects degraded (8.352%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:20.662 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:20 smithi016 bash[77892]: cluster 2023-10-09T09:15:19.741396+0000 mon.a (mon.0) 1518 : cluster 3 Health check update: Degraded data redundancy: 14153/169449 objects degraded (8.352%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:20.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:20 smithi094 bash[59677]: cluster 2023-10-09T09:15:19.741396+0000 mon.a (mon.0) 1518 : cluster 3 Health check update: Degraded data redundancy: 14153/169449 objects degraded (8.352%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:20.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:20.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:20.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:20.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:20.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:21.070 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:21.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:21.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:21.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:21.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:21.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:21.287 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:15:21.287 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:15:21.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:21.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:21.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:21.403 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:15:21.452 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:15:21.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:21.487 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:21 smithi016 bash[77892]: cluster 2023-10-09T09:15:20.116630+0000 mgr.y (mgr.25421) 1141 : cluster 0 pgmap v1233: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 14201/167358 objects degraded (8.485%); 19247/167358 objects misplaced (11.500%); 11 MiB/s, 169 objects/s recovering 2023-10-09T09:15:21.487 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:21 smithi016 bash[77892]: audit 2023-10-09T09:15:21.324301+0000 mon.a (mon.0) 1519 : audit 0 from='client.? 172.21.15.16:0/1970740661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:15:21.488 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:21 smithi016 bash[75179]: cluster 2023-10-09T09:15:20.116630+0000 mgr.y (mgr.25421) 1141 : cluster 0 pgmap v1233: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 14201/167358 objects degraded (8.485%); 19247/167358 objects misplaced (11.500%); 11 MiB/s, 169 objects/s recovering 2023-10-09T09:15:21.488 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:21 smithi016 bash[75179]: audit 2023-10-09T09:15:21.324301+0000 mon.a (mon.0) 1519 : audit 0 from='client.? 172.21.15.16:0/1970740661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:15:21.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:15:21.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:21.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:21.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:21.553 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:15:21.593 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:15:21.639 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:15:21.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:21.674 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:15:21.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:21.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:21.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:21.723 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:15:21.731 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.731 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.731 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:15:21.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:21 smithi094 bash[59677]: cluster 2023-10-09T09:15:20.116630+0000 mgr.y (mgr.25421) 1141 : cluster 0 pgmap v1233: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 14201/167358 objects degraded (8.485%); 19247/167358 objects misplaced (11.500%); 11 MiB/s, 169 objects/s recovering 2023-10-09T09:15:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:21 smithi094 bash[59677]: audit 2023-10-09T09:15:21.324301+0000 mon.a (mon.0) 1519 : audit 0 from='client.? 172.21.15.16:0/1970740661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:15:21.797 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:21.844 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:21.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:21.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:21.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:21.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:21.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:21.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:22.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:22.116 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:22.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:22.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:22.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:22.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:22.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:22.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:22.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:22.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:22.403 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:15:22.405 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 0.785866 (c92e) 2023-10-09T09:15:22.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:22 smithi016 bash[75179]: audit 2023-10-09T09:15:21.736668+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.16:0/1394978016' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]: dispatch 2023-10-09T09:15:22.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:22 smithi016 bash[75179]: audit 2023-10-09T09:15:21.737399+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]: dispatch 2023-10-09T09:15:22.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:22 smithi016 bash[75179]: audit 2023-10-09T09:15:21.749266+0000 mon.a (mon.0) 1521 : audit 1 from='client.? 172.21.15.16:0/818309077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:15:22.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:22 smithi016 bash[77892]: audit 2023-10-09T09:15:21.736668+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.16:0/1394978016' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]: dispatch 2023-10-09T09:15:22.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:22 smithi016 bash[77892]: audit 2023-10-09T09:15:21.737399+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]: dispatch 2023-10-09T09:15:22.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:22 smithi016 bash[77892]: audit 2023-10-09T09:15:21.749266+0000 mon.a (mon.0) 1521 : audit 1 from='client.? 172.21.15.16:0/818309077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:15:22.500 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:22.537 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:22.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:22.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:22.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:22.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:22.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:22.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:22.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:22 smithi094 bash[59677]: audit 2023-10-09T09:15:21.736668+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.16:0/1394978016' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]: dispatch 2023-10-09T09:15:22.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:22 smithi094 bash[59677]: audit 2023-10-09T09:15:21.737399+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]: dispatch 2023-10-09T09:15:22.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:22 smithi094 bash[59677]: audit 2023-10-09T09:15:21.749266+0000 mon.a (mon.0) 1521 : audit 1 from='client.? 172.21.15.16:0/818309077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:15:22.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:22.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:22.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:22.899 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:22.927 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:22.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:23.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:23.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:23.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:23.188 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:23.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:23.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:23.321 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:23.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:23.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:23.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:23 smithi094 bash[59677]: cluster 2023-10-09T09:15:22.117269+0000 mgr.y (mgr.25421) 1142 : cluster 0 pgmap v1234: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 14201/161112 objects degraded (8.814%); 18715/161112 objects misplaced (11.616%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:15:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:23 smithi094 bash[59677]: cluster 2023-10-09T09:15:22.381414+0000 mon.a (mon.0) 1522 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:15:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:23 smithi094 bash[59677]: cluster 2023-10-09T09:15:22.393289+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T09:15:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:23 smithi094 bash[59677]: cluster 2023-10-09T09:15:22.395629+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T09:15:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:23 smithi094 bash[59677]: audit 2023-10-09T09:15:22.395970+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]': finished 2023-10-09T09:15:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:23 smithi094 bash[59677]: audit 2023-10-09T09:15:22.396123+0000 mon.a (mon.0) 1524 : audit 1 from='client.? 172.21.15.16:0/818309077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:15:23.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:23 smithi094 bash[59677]: cluster 2023-10-09T09:15:22.399748+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-09T09:15:23.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:23.580 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:23.683 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:23.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:23.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[75179]: cluster 2023-10-09T09:15:22.117269+0000 mgr.y (mgr.25421) 1142 : cluster 0 pgmap v1234: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 14201/161112 objects degraded (8.814%); 18715/161112 objects misplaced (11.616%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[75179]: cluster 2023-10-09T09:15:22.381414+0000 mon.a (mon.0) 1522 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[75179]: cluster 2023-10-09T09:15:22.393289+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[75179]: cluster 2023-10-09T09:15:22.395629+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[75179]: audit 2023-10-09T09:15:22.395970+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]': finished 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[75179]: audit 2023-10-09T09:15:22.396123+0000 mon.a (mon.0) 1524 : audit 1 from='client.? 172.21.15.16:0/818309077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[75179]: cluster 2023-10-09T09:15:22.399748+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[77892]: cluster 2023-10-09T09:15:22.117269+0000 mgr.y (mgr.25421) 1142 : cluster 0 pgmap v1234: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 14201/161112 objects degraded (8.814%); 18715/161112 objects misplaced (11.616%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[77892]: cluster 2023-10-09T09:15:22.381414+0000 mon.a (mon.0) 1522 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:15:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[77892]: cluster 2023-10-09T09:15:22.393289+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T09:15:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[77892]: cluster 2023-10-09T09:15:22.395629+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T09:15:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[77892]: audit 2023-10-09T09:15:22.395970+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.7858663985604675}]': finished 2023-10-09T09:15:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[77892]: audit 2023-10-09T09:15:22.396123+0000 mon.a (mon.0) 1524 : audit 1 from='client.? 172.21.15.16:0/818309077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:15:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:23 smithi016 bash[77892]: cluster 2023-10-09T09:15:22.399748+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-09T09:15:23.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:23.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:23.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:23.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:23.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:23.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:23.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:24.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:24.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:24.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:24.206 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:24.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:24.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:24.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:24.336 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:24.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:24.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:24.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:24.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:24.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:24.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:24.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:24.611 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:24 smithi094 bash[59677]: cluster 2023-10-09T09:15:23.394817+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.611 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:24 smithi094 bash[59677]: cluster 2023-10-09T09:15:23.395308+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.611 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:24 smithi094 bash[59677]: cluster 2023-10-09T09:15:23.398610+0000 mon.a (mon.0) 1526 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-09T09:15:24.611 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:24 smithi094 bash[59677]: cluster 2023-10-09T09:15:23.399518+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.611 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:24 smithi094 bash[59677]: cluster 2023-10-09T09:15:23.399724+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.612 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:24 smithi094 bash[59677]: cluster 2023-10-09T09:15:23.399816+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.634 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:24.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:24.737 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[75179]: cluster 2023-10-09T09:15:23.394817+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[75179]: cluster 2023-10-09T09:15:23.395308+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[75179]: cluster 2023-10-09T09:15:23.398610+0000 mon.a (mon.0) 1526 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-09T09:15:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[75179]: cluster 2023-10-09T09:15:23.399518+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[75179]: cluster 2023-10-09T09:15:23.399724+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[75179]: cluster 2023-10-09T09:15:23.399816+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[77892]: cluster 2023-10-09T09:15:23.394817+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[77892]: cluster 2023-10-09T09:15:23.395308+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[77892]: cluster 2023-10-09T09:15:23.398610+0000 mon.a (mon.0) 1526 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-09T09:15:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[77892]: cluster 2023-10-09T09:15:23.399518+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[77892]: cluster 2023-10-09T09:15:23.399724+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:24 smithi016 bash[77892]: cluster 2023-10-09T09:15:23.399816+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T09:15:24.764 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:24.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:24.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:24.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:24.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:24.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:24.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:25.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:25.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:25.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:25.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:15:25.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:25.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:25.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:25.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:25.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:25.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:25.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:25.411 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.117845+0000 mgr.y (mgr.25421) 1143 : cluster 0 pgmap v1237: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 14255/159513 objects degraded (8.937%); 17369/159513 objects misplaced (10.889%); 14 MiB/s, 230 objects/s recovering 2023-10-09T09:15:25.411 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.407358+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.412 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.411536+0000 mon.a (mon.0) 1527 : cluster 0 osdmap e272: 8 total, 8 up, 7 in 2023-10-09T09:15:25.412 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.412011+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.412 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.414262+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.412 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.415319+0000 osd.7 (osd.7) 172 : cluster 0 10.9 starting backfill to osd.3 from (0'0,0'0] MAX to 269'10052 2023-10-09T09:15:25.412 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.419078+0000 osd.6 (osd.6) 216 : cluster 0 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 269'10169 2023-10-09T09:15:25.412 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[75179]: cluster 2023-10-09T09:15:24.742628+0000 mon.a (mon.0) 1528 : cluster 3 Health check update: Degraded data redundancy: 14255/159513 objects degraded (8.937%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:25.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:25.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:25.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:25.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:25.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:25.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:25.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:25.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:25.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.117845+0000 mgr.y (mgr.25421) 1143 : cluster 0 pgmap v1237: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 14255/159513 objects degraded (8.937%); 17369/159513 objects misplaced (10.889%); 14 MiB/s, 230 objects/s recovering 2023-10-09T09:15:25.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.407358+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.411536+0000 mon.a (mon.0) 1527 : cluster 0 osdmap e272: 8 total, 8 up, 7 in 2023-10-09T09:15:25.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.412011+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.414262+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.415319+0000 osd.7 (osd.7) 172 : cluster 0 10.9 starting backfill to osd.3 from (0'0,0'0] MAX to 269'10052 2023-10-09T09:15:25.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.419078+0000 osd.6 (osd.6) 216 : cluster 0 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 269'10169 2023-10-09T09:15:25.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[77892]: cluster 2023-10-09T09:15:24.742628+0000 mon.a (mon.0) 1528 : cluster 3 Health check update: Degraded data redundancy: 14255/159513 objects degraded (8.937%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:25.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.117845+0000 mgr.y (mgr.25421) 1143 : cluster 0 pgmap v1237: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 14255/159513 objects degraded (8.937%); 17369/159513 objects misplaced (10.889%); 14 MiB/s, 230 objects/s recovering 2023-10-09T09:15:25.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.407358+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.411536+0000 mon.a (mon.0) 1527 : cluster 0 osdmap e272: 8 total, 8 up, 7 in 2023-10-09T09:15:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.412011+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.414262+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T09:15:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.415319+0000 osd.7 (osd.7) 172 : cluster 0 10.9 starting backfill to osd.3 from (0'0,0'0] MAX to 269'10052 2023-10-09T09:15:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.419078+0000 osd.6 (osd.6) 216 : cluster 0 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 269'10169 2023-10-09T09:15:25.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:25 smithi094 bash[59677]: cluster 2023-10-09T09:15:24.742628+0000 mon.a (mon.0) 1528 : cluster 3 Health check update: Degraded data redundancy: 14255/159513 objects degraded (8.937%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:25.760 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:25.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:25.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:25.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:25.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:25.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:26.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:26.032 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:15:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:15:25] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-09T09:15:26.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:26.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:26.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:26.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:26.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:26.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:26.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:26.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:26.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:26.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:26.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:26.700 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:26.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:26.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:26.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:26.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:26.970 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:26.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:27.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:27.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:27.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:27.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:27.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:27.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:27.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:27.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:27.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:27.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:27.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:27.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:27 smithi016 bash[75179]: cluster 2023-10-09T09:15:26.118550+0000 mgr.y (mgr.25421) 1144 : cluster 0 pgmap v1239: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 10811/156192 objects degraded (6.922%); 25804/156192 objects misplaced (16.521%); 25 MiB/s, 397 objects/s recovering 2023-10-09T09:15:27.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:27.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:27.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:27.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:27.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:27.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:27.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:27.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:27.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:27 smithi094 bash[59677]: cluster 2023-10-09T09:15:26.118550+0000 mgr.y (mgr.25421) 1144 : cluster 0 pgmap v1239: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 10811/156192 objects degraded (6.922%); 25804/156192 objects misplaced (16.521%); 25 MiB/s, 397 objects/s recovering 2023-10-09T09:15:27.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:27 smithi016 bash[77892]: cluster 2023-10-09T09:15:26.118550+0000 mgr.y (mgr.25421) 1144 : cluster 0 pgmap v1239: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 10811/156192 objects degraded (6.922%); 25804/156192 objects misplaced (16.521%); 25 MiB/s, 397 objects/s recovering 2023-10-09T09:15:27.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:27.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:27.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:27.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:27.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:27.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:28.007 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:28.040 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:28.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:28.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:28.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:28.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:28.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:28.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:28.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:28.838 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:28.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:28.978 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:28.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:28.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:29.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:29.111 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:29.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:29.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:29.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:29.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:29.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:29.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:29.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:29.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:29.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:29.428 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:29 smithi094 bash[59677]: cluster 2023-10-09T09:15:28.119164+0000 mgr.y (mgr.25421) 1145 : cluster 0 pgmap v1240: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 273 op/s; 10811/156192 objects degraded (6.922%); 25804/156192 objects misplaced (16.521%); 25 MiB/s, 397 objects/s recovering 2023-10-09T09:15:29.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:29 smithi094 bash[59677]: cluster 2023-10-09T09:15:28.428076+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e273: 8 total, 8 up, 7 in 2023-10-09T09:15:29.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:29 smithi094 bash[59677]: cluster 2023-10-09T09:15:28.430701+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:29 smithi094 bash[59677]: cluster 2023-10-09T09:15:28.436485+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:29 smithi094 bash[59677]: cluster 2023-10-09T09:15:28.436916+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[75179]: cluster 2023-10-09T09:15:28.119164+0000 mgr.y (mgr.25421) 1145 : cluster 0 pgmap v1240: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 273 op/s; 10811/156192 objects degraded (6.922%); 25804/156192 objects misplaced (16.521%); 25 MiB/s, 397 objects/s recovering 2023-10-09T09:15:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[75179]: cluster 2023-10-09T09:15:28.428076+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e273: 8 total, 8 up, 7 in 2023-10-09T09:15:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[75179]: cluster 2023-10-09T09:15:28.430701+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[75179]: cluster 2023-10-09T09:15:28.436485+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[75179]: cluster 2023-10-09T09:15:28.436916+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[77892]: cluster 2023-10-09T09:15:28.119164+0000 mgr.y (mgr.25421) 1145 : cluster 0 pgmap v1240: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 273 op/s; 10811/156192 objects degraded (6.922%); 25804/156192 objects misplaced (16.521%); 25 MiB/s, 397 objects/s recovering 2023-10-09T09:15:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[77892]: cluster 2023-10-09T09:15:28.428076+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e273: 8 total, 8 up, 7 in 2023-10-09T09:15:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[77892]: cluster 2023-10-09T09:15:28.430701+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[77892]: cluster 2023-10-09T09:15:28.436485+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:29 smithi016 bash[77892]: cluster 2023-10-09T09:15:28.436916+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T09:15:29.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:29.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:29.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:29.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:29.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:29.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:29.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:29.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:29.883 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:29.922 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:29.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:29.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:30.029 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:30.081 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:30.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:30.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:30.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:30.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:30.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:30.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:15:30.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:15:30.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:30.367 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:30.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:30.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:30.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:30.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[77892]: cluster 2023-10-09T09:15:29.436408+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e274: 8 total, 8 up, 7 in 2023-10-09T09:15:30.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[77892]: cluster 2023-10-09T09:15:29.437552+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[77892]: cluster 2023-10-09T09:15:29.437577+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[77892]: cluster 2023-10-09T09:15:29.440097+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[77892]: cluster 2023-10-09T09:15:29.745970+0000 mon.a (mon.0) 1531 : cluster 3 Health check update: Degraded data redundancy: 10811/156192 objects degraded (6.922%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[75179]: cluster 2023-10-09T09:15:29.436408+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e274: 8 total, 8 up, 7 in 2023-10-09T09:15:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[75179]: cluster 2023-10-09T09:15:29.437552+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[75179]: cluster 2023-10-09T09:15:29.437577+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[75179]: cluster 2023-10-09T09:15:29.440097+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:30 smithi016 bash[75179]: cluster 2023-10-09T09:15:29.745970+0000 mon.a (mon.0) 1531 : cluster 3 Health check update: Degraded data redundancy: 10811/156192 objects degraded (6.922%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:30.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:30.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:30.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:30.612 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:30 smithi094 bash[59677]: cluster 2023-10-09T09:15:29.436408+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e274: 8 total, 8 up, 7 in 2023-10-09T09:15:30.612 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:30 smithi094 bash[59677]: cluster 2023-10-09T09:15:29.437552+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.612 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:30 smithi094 bash[59677]: cluster 2023-10-09T09:15:29.437577+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.612 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:30 smithi094 bash[59677]: cluster 2023-10-09T09:15:29.440097+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T09:15:30.612 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:30 smithi094 bash[59677]: cluster 2023-10-09T09:15:29.745970+0000 mon.a (mon.0) 1531 : cluster 3 Health check update: Degraded data redundancy: 10811/156192 objects degraded (6.922%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:30.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:30.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:30.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:30.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:30.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:30.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:30.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:30.910 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:30.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:31.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:31.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:31.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:31.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:31.150 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:31.182 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:31.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:31.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:31.286 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:31.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:31.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:31.353 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:31.426 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:31.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:31.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:31.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:31.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:31.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:31.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:31.693 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:31.702 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:31 smithi094 bash[59677]: cluster 2023-10-09T09:15:30.119719+0000 mgr.y (mgr.25421) 1146 : cluster 0 pgmap v1243: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 10811/154890 objects degraded (6.980%); 24629/154890 objects misplaced (15.901%); 24 MiB/s, 391 objects/s recovering 2023-10-09T09:15:31.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:31.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:31 smithi016 bash[75179]: cluster 2023-10-09T09:15:30.119719+0000 mgr.y (mgr.25421) 1146 : cluster 0 pgmap v1243: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 10811/154890 objects degraded (6.980%); 24629/154890 objects misplaced (15.901%); 24 MiB/s, 391 objects/s recovering 2023-10-09T09:15:31.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:31 smithi016 bash[77892]: cluster 2023-10-09T09:15:30.119719+0000 mgr.y (mgr.25421) 1146 : cluster 0 pgmap v1243: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 10811/154890 objects degraded (6.980%); 24629/154890 objects misplaced (15.901%); 24 MiB/s, 391 objects/s recovering 2023-10-09T09:15:31.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:31.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:31.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:31.893 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:31.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:31.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:31.997 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:32.032 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:32.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:32.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:32.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:32.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:32.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:32.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:32.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:32.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:32.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:32.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:32.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:32.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:32.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:32.696 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:32.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:32.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:32.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:32.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:32.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:32.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:32.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:33.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:33.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:33.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:33.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:33.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:33.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:33.308 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:15:33.308 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:15:33.309 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T09:15:33.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:15:33.347 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:33.371 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:33.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:33.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:33.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:33 smithi016 bash[75179]: cluster 2023-10-09T09:15:32.120585+0000 mgr.y (mgr.25421) 1147 : cluster 0 pgmap v1244: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 10673/149970 objects degraded (7.117%); 22479/149970 objects misplaced (14.989%); 33 MiB/s, 522 objects/s recovering 2023-10-09T09:15:33.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:33 smithi094 bash[59677]: cluster 2023-10-09T09:15:32.120585+0000 mgr.y (mgr.25421) 1147 : cluster 0 pgmap v1244: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 10673/149970 objects degraded (7.117%); 22479/149970 objects misplaced (14.989%); 33 MiB/s, 522 objects/s recovering 2023-10-09T09:15:33.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:33.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:33.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:33.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:33.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:33.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:33.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:33.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:33 smithi016 bash[77892]: cluster 2023-10-09T09:15:32.120585+0000 mgr.y (mgr.25421) 1147 : cluster 0 pgmap v1244: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 10673/149970 objects degraded (7.117%); 22479/149970 objects misplaced (14.989%); 33 MiB/s, 522 objects/s recovering 2023-10-09T09:15:33.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:33.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:33.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:33.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:33.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:34.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:34.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:34.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:34.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:34.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:34.189 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:34.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:34.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:34.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:34.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:34.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:34.409 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:34.418 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:34.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:34.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:34.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:34.562 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:34.593 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:34.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:34.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:34.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:34 smithi016 bash[75179]: audit 2023-10-09T09:15:34.249620+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:34.705 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:34 smithi016 bash[77892]: audit 2023-10-09T09:15:34.249620+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:34.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:34.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:34.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:34.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:34 smithi094 bash[59677]: audit 2023-10-09T09:15:34.249620+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:34.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:34.855 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:34.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:34.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:34.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:34.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:35.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:35.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:35.233 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:35.276 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:35.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:35.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:35.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:35.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:35.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:35 smithi016 bash[75179]: cluster 2023-10-09T09:15:34.121272+0000 mgr.y (mgr.25421) 1148 : cluster 0 pgmap v1245: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 10673/148248 objects degraded (7.199%); 22196/148248 objects misplaced (14.972%); 23 MiB/s, 360 objects/s recovering 2023-10-09T09:15:35.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:35 smithi016 bash[75179]: cluster 2023-10-09T09:15:34.747912+0000 mon.a (mon.0) 1533 : cluster 3 Health check update: Degraded data redundancy: 10673/148248 objects degraded (7.199%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:35.555 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:35 smithi016 bash[77892]: cluster 2023-10-09T09:15:34.121272+0000 mgr.y (mgr.25421) 1148 : cluster 0 pgmap v1245: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 10673/148248 objects degraded (7.199%); 22196/148248 objects misplaced (14.972%); 23 MiB/s, 360 objects/s recovering 2023-10-09T09:15:35.555 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:35 smithi016 bash[77892]: cluster 2023-10-09T09:15:34.747912+0000 mon.a (mon.0) 1533 : cluster 3 Health check update: Degraded data redundancy: 10673/148248 objects degraded (7.199%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:35.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:35.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:35 smithi094 bash[59677]: cluster 2023-10-09T09:15:34.121272+0000 mgr.y (mgr.25421) 1148 : cluster 0 pgmap v1245: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 10673/148248 objects degraded (7.199%); 22196/148248 objects misplaced (14.972%); 23 MiB/s, 360 objects/s recovering 2023-10-09T09:15:35.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:35 smithi094 bash[59677]: cluster 2023-10-09T09:15:34.747912+0000 mon.a (mon.0) 1533 : cluster 3 Health check update: Degraded data redundancy: 10673/148248 objects degraded (7.199%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:35.690 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:35.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:35.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:35.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:35.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:35.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:35.961 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:15:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:15:35] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-09T09:15:35.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:35.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:36.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:36.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:36.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:36.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:36.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:36.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:36.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:36.358 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:36.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:36.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:36.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:36.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:36.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:36.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:36.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:36.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:36.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:36.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:36.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:36.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:36.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:36.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:36.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:36.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:36.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:37.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:37.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:37.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:37.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:37.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:37.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:37.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:37.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:37.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:37.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:37.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:37 smithi016 bash[75179]: cluster 2023-10-09T09:15:36.121981+0000 mgr.y (mgr.25421) 1149 : cluster 0 pgmap v1246: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 10514/142746 objects degraded (7.366%); 20348/142746 objects misplaced (14.255%); 33 MiB/s, 533 objects/s recovering 2023-10-09T09:15:37.555 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:37.564 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:37 smithi094 bash[59677]: cluster 2023-10-09T09:15:36.121981+0000 mgr.y (mgr.25421) 1149 : cluster 0 pgmap v1246: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 10514/142746 objects degraded (7.366%); 20348/142746 objects misplaced (14.255%); 33 MiB/s, 533 objects/s recovering 2023-10-09T09:15:37.579 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:37.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:37.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:37.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:37.713 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:37 smithi016 bash[77892]: cluster 2023-10-09T09:15:36.121981+0000 mgr.y (mgr.25421) 1149 : cluster 0 pgmap v1246: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 10514/142746 objects degraded (7.366%); 20348/142746 objects misplaced (14.255%); 33 MiB/s, 533 objects/s recovering 2023-10-09T09:15:37.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:37.815 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:37.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:37.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:37.962 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:37.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:37.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:38.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:38.128 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:38.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:38.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:38.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:38.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:38.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:38.412 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:38.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:38.519 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:38.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:38.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:38.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:38.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:38.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:38.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:38.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:38.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:38.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:38.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:38.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:38.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:39.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:39.101 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:39.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:39.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:39.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:39.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:39.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:39.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:39.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:39.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:39.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:39.532 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:39.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:39.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:39.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:39.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:39.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:39 smithi094 bash[59677]: cluster 2023-10-09T09:15:38.122661+0000 mgr.y (mgr.25421) 1150 : cluster 0 pgmap v1247: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 10514/142746 objects degraded (7.366%); 20348/142746 objects misplaced (14.255%); 28 MiB/s, 440 objects/s recovering 2023-10-09T09:15:39.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:39.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:39 smithi016 bash[75179]: cluster 2023-10-09T09:15:38.122661+0000 mgr.y (mgr.25421) 1150 : cluster 0 pgmap v1247: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 10514/142746 objects degraded (7.366%); 20348/142746 objects misplaced (14.255%); 28 MiB/s, 440 objects/s recovering 2023-10-09T09:15:39.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:39 smithi016 bash[77892]: cluster 2023-10-09T09:15:38.122661+0000 mgr.y (mgr.25421) 1150 : cluster 0 pgmap v1247: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 10514/142746 objects degraded (7.366%); 20348/142746 objects misplaced (14.255%); 28 MiB/s, 440 objects/s recovering 2023-10-09T09:15:39.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:39.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:39.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:39.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:39.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:39.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:39.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:40.025 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:40.050 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:40.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:40.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:40.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:40.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:40.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:40.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:40.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:40.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:40.571 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:40.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:40.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:40.683 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[75179]: cluster 2023-10-09T09:15:39.749163+0000 mon.a (mon.0) 1534 : cluster 3 Health check update: Degraded data redundancy: 10514/142746 objects degraded (7.366%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:40.683 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[75179]: cluster 2023-10-09T09:15:39.755525+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e275: 8 total, 8 up, 7 in 2023-10-09T09:15:40.683 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[75179]: cluster 2023-10-09T09:15:39.756782+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.683 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[75179]: cluster 2023-10-09T09:15:39.758593+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.683 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[75179]: cluster 2023-10-09T09:15:39.760814+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[77892]: cluster 2023-10-09T09:15:39.749163+0000 mon.a (mon.0) 1534 : cluster 3 Health check update: Degraded data redundancy: 10514/142746 objects degraded (7.366%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:40.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[77892]: cluster 2023-10-09T09:15:39.755525+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e275: 8 total, 8 up, 7 in 2023-10-09T09:15:40.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[77892]: cluster 2023-10-09T09:15:39.756782+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[77892]: cluster 2023-10-09T09:15:39.758593+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:40 smithi016 bash[77892]: cluster 2023-10-09T09:15:39.760814+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:40.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:40 smithi094 bash[59677]: cluster 2023-10-09T09:15:39.749163+0000 mon.a (mon.0) 1534 : cluster 3 Health check update: Degraded data redundancy: 10514/142746 objects degraded (7.366%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:40 smithi094 bash[59677]: cluster 2023-10-09T09:15:39.755525+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e275: 8 total, 8 up, 7 in 2023-10-09T09:15:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:40 smithi094 bash[59677]: cluster 2023-10-09T09:15:39.756782+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:40 smithi094 bash[59677]: cluster 2023-10-09T09:15:39.758593+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:40 smithi094 bash[59677]: cluster 2023-10-09T09:15:39.760814+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T09:15:40.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:40.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:40.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:40.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:40.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:40.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:41.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:41.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:41.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:41.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:41.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:41.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:41.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:41.356 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:41.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:41.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:41.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:41.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:41.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:41.510 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:15:41.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:41.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:41.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:41.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:41.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[75179]: cluster 2023-10-09T09:15:40.123225+0000 mgr.y (mgr.25421) 1151 : cluster 0 pgmap v1249: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 10514/140916 objects degraded (7.461%); 20055/140916 objects misplaced (14.232%); 21 MiB/s, 333 objects/s recovering 2023-10-09T09:15:41.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[75179]: cluster 2023-10-09T09:15:40.767600+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T09:15:41.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[75179]: cluster 2023-10-09T09:15:40.767665+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T09:15:41.729 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[75179]: cluster 2023-10-09T09:15:40.770812+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e276: 8 total, 8 up, 7 in 2023-10-09T09:15:41.730 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[77892]: cluster 2023-10-09T09:15:40.123225+0000 mgr.y (mgr.25421) 1151 : cluster 0 pgmap v1249: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 10514/140916 objects degraded (7.461%); 20055/140916 objects misplaced (14.232%); 21 MiB/s, 333 objects/s recovering 2023-10-09T09:15:41.730 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[77892]: cluster 2023-10-09T09:15:40.767600+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T09:15:41.730 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[77892]: cluster 2023-10-09T09:15:40.767665+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T09:15:41.730 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:41 smithi016 bash[77892]: cluster 2023-10-09T09:15:40.770812+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e276: 8 total, 8 up, 7 in 2023-10-09T09:15:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:41 smithi094 bash[59677]: cluster 2023-10-09T09:15:40.123225+0000 mgr.y (mgr.25421) 1151 : cluster 0 pgmap v1249: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 10514/140916 objects degraded (7.461%); 20055/140916 objects misplaced (14.232%); 21 MiB/s, 333 objects/s recovering 2023-10-09T09:15:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:41 smithi094 bash[59677]: cluster 2023-10-09T09:15:40.767600+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T09:15:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:41 smithi094 bash[59677]: cluster 2023-10-09T09:15:40.767665+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T09:15:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:41 smithi094 bash[59677]: cluster 2023-10-09T09:15:40.770812+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e276: 8 total, 8 up, 7 in 2023-10-09T09:15:41.753 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:41.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:41.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:41.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:41.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:41.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:41.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:42.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:42.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:42.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:42.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:42.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:42.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:42.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:42.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:42.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:42.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:42.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:42.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:42.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:43.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:43.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:43.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:43.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:43.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:43.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:43.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:43.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:43.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:43.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:43.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:43.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:43 smithi094 bash[59677]: cluster 2023-10-09T09:15:42.123756+0000 mgr.y (mgr.25421) 1152 : cluster 0 pgmap v1251: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 10345/135480 objects degraded (7.636%); 18099/135480 objects misplaced (13.359%); 24 MiB/s, 384 objects/s recovering 2023-10-09T09:15:43.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:43 smithi094 bash[59677]: audit 2023-10-09T09:15:42.883273+0000 mon.a (mon.0) 1537 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:43.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:43.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:43.595 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:43.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:43.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:43.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:43.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:43 smithi016 bash[75179]: cluster 2023-10-09T09:15:42.123756+0000 mgr.y (mgr.25421) 1152 : cluster 0 pgmap v1251: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 10345/135480 objects degraded (7.636%); 18099/135480 objects misplaced (13.359%); 24 MiB/s, 384 objects/s recovering 2023-10-09T09:15:43.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:43 smithi016 bash[75179]: audit 2023-10-09T09:15:42.883273+0000 mon.a (mon.0) 1537 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:43.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:43 smithi016 bash[77892]: cluster 2023-10-09T09:15:42.123756+0000 mgr.y (mgr.25421) 1152 : cluster 0 pgmap v1251: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 10345/135480 objects degraded (7.636%); 18099/135480 objects misplaced (13.359%); 24 MiB/s, 384 objects/s recovering 2023-10-09T09:15:43.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:43 smithi016 bash[77892]: audit 2023-10-09T09:15:42.883273+0000 mon.a (mon.0) 1537 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:43.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:43.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:43.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:43.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:43.941 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:43.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:44.046 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:44.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:44.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:44.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:44.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:44.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:44.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:44.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:44.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:44.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:44.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:44.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:44.500 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:15:44.500 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:143522 9.2 9:43e1e0b6:::benchmark_data_smithi016_172771_object46936:head [delete] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:09:43.334921+0000", 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 361.16092603499999, 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2314118660000002, 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.501 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143522 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.334921+0000", 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.502 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.334921+0000", 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.334919+0000", 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:15:44.503 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.334927+0000", 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3599999999999996e-06 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.334941+0000", 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3969e-05 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.504 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.334950+0000", 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7409999999999998e-06 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.343217+0000", 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0082671410000000004 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.505 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.506 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.506 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.343225+0000", 2023-10-09T09:15:44.506 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0679999999999997e-06 2023-10-09T09:15:44.506 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.506 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.506 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.507 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.337812+0000", 2023-10-09T09:15:44.507 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.99458663999999997 2023-10-09T09:15:44.507 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.507 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.507 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.507 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.337821+0000", 2023-10-09T09:15:44.507 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.431e-06 2023-10-09T09:15:44.508 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.508 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.508 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.508 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.556550+0000", 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.218729044 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.556705+0000", 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000154788 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.556846+0000", 2023-10-09T09:15:44.509 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014121700000000001 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.561548+0000", 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004702276 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.561558+0000", 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0880000000000001e-06 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.510 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.566117+0000", 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0045595109999999996 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.566156+0000", 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8970000000000001e-05 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.511 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.566214+0000", 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8087000000000001e-05 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.566332+0000", 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011832699999999999 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.512 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:143528 9.7 9:e65e7f9c:::benchmark_data_smithi016_172771_object46942:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:09:43.367985+0000", 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 361.12786131500002, 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.303170626, 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143528 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.513 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.367985+0000", 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.367985+0000", 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.514 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.367983+0000", 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.367996+0000", 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2585999999999999e-05 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.368020+0000", 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4403999999999999e-05 2023-10-09T09:15:44.515 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.368032+0000", 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.221e-05 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.368092+0000", 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9357000000000001e-05 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.516 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.368103+0000", 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1728e-05 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.596781+0000", 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2286772530000001 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.596941+0000", 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016039299999999999 2023-10-09T09:15:44.517 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,5,6,7", 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.597103+0000", 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000162382 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.602333+0000", 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0052297799999999998 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.518 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.602344+0000", 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1317e-05 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.612024+0000", 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0096793279999999992 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.519 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.612082+0000", 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7958999999999999e-05 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.670912+0000", 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.058830367000000001 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.520 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.670975+0000", 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2942999999999997e-05 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671156+0000", 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018084899999999999 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.521 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:143531 9.2 9:4c0e13b3:::benchmark_data_smithi016_172771_object46945:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:09:43.386824+0000", 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 361.10902230200003, 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1896933789999999, 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143531 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.522 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.386824+0000", 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.386824+0000", 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.523 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.386822+0000", 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.386833+0000", 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1136e-05 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.524 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.386858+0000", 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4871e-05 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.386870+0000", 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1620999999999999e-05 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.387023+0000", 2023-10-09T09:15:44.525 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015303800000000001 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:43.387045+0000", 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2163000000000001e-05 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.337836+0000", 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.95079080400000004 2023-10-09T09:15:44.526 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.337840+0000", 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3510000000000004e-06 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.561582+0000", 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2237423869999999 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.527 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.561734+0000", 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015171100000000001 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.561824+0000", 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9371999999999997e-05 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.528 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576083+0000", 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014259061 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576097+0000", 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4677e-05 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576223+0000", 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012582899999999999 2023-10-09T09:15:44.529 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576357+0000", 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013386 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576416+0000", 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9307e-05 2023-10-09T09:15:44.530 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576518+0000", 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010141699999999999 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:143546 9.2 9:43dd0215:::benchmark_data_smithi016_172771_object46960:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T09:15:44.531 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:09:44.627760+0000", 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 359.86808613599999, 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.94891599599999998, 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143546 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.532 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627760+0000", 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627760+0000", 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.533 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627759+0000", 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627766+0000", 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0040000000000002e-06 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627781+0000", 2023-10-09T09:15:44.534 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5085e-05 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627789+0000", 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1820000000000006e-06 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627844+0000", 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5157000000000001e-05 2023-10-09T09:15:44.535 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.627853+0000", 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7669999999999992e-06 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.566366+0000", 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.93851272299999999 2023-10-09T09:15:44.536 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.566518+0000", 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000152137 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.566660+0000", 2023-10-09T09:15:44.538 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000141898 2023-10-09T09:15:44.539 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.539 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.539 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.539 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576103+0000", 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0094438360000000006 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576287+0000", 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018396600000000001 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576325+0000", 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7697000000000002e-05 2023-10-09T09:15:44.540 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576551+0000", 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00022611899999999999 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576591+0000", 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9954999999999999e-05 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.541 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576676+0000", 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5321000000000005e-05 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:143549 9.2 9:46536a13:::benchmark_data_smithi016_172771_object46963:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:09:44.671589+0000", 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 359.82425765699998, 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.91642599000000002, 2023-10-09T09:15:44.542 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143549 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671589+0000", 2023-10-09T09:15:44.543 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671589+0000", 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671588+0000", 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.544 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671593+0000", 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9579999999999998e-06 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671603+0000", 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9820000000000003e-06 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671611+0000", 2023-10-09T09:15:44.545 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4200000000000001e-06 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671672+0000", 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1017000000000002e-05 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.671681+0000", 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0899999999999994e-06 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.546 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.571388+0000", 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.89970728200000005 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.571580+0000", 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019251199999999999 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.547 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.571678+0000", 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7306999999999995e-05 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576332+0000", 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0046538229999999996 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576708+0000", 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00037616400000000001 2023-10-09T09:15:44.548 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576733+0000", 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.495e-05 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.587781+0000", 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011048583000000001 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.549 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.587849+0000", 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8022999999999996e-05 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.588015+0000", 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016559800000000001 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.550 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:143554 9.2 9:49accc88:::benchmark_data_smithi016_172771_object46968:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:09:44.701208+0000", 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 359.79463824700002, 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.89927408600000003, 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143554 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.551 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701208+0000", 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701208+0000", 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.552 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701207+0000", 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701214+0000", 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9770000000000003e-06 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701224+0000", 2023-10-09T09:15:44.553 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0467e-05 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701233+0000", 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9509999999999995e-06 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701317+0000", 2023-10-09T09:15:44.554 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3207999999999993e-05 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:44.701333+0000", 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6039e-05 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576750+0000", 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.87541762499999998 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.555 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576873+0000", 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000122841 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.576992+0000", 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011924799999999999 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.556 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.581447+0000", 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0044544490000000001 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.581619+0000", 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017216200000000001 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.591460+0000", 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0098408669999999997 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.557 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.600188+0000", 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0087287470000000002 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.600283+0000", 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4576999999999999e-05 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:09:45.600482+0000", 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019935499999999999 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:167501 9.f 9:ff959119:::benchmark_data_smithi016_172771_object70915:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:10:34.120786+0000", 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 310.37506062599999, 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.50122034400000004, 2023-10-09T09:15:44.559 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167501 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.120786+0000", 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.560 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.120786+0000", 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.120784+0000", 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.561 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.120795+0000", 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1477999999999999e-05 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.120820+0000", 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4777e-05 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.120833+0000", 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2378000000000001e-05 2023-10-09T09:15:44.562 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.121903+0000", 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001070501 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.122126+0000", 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000222776 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.563 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.122343+0000", 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000217598 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.164500+0000", 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.042156138000000003 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.165338+0000", 2023-10-09T09:15:44.564 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00083875099999999999 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.513695+0000", 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34835709500000001 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.621787+0000", 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.108091987 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.621854+0000", 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6778000000000003e-05 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.622006+0000", 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000152075 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.566 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:167502 9.7 9.c6815ac7 (undecoded) ondisk+write+known_if_redirected e237)", 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:10:34.137829+0000", 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 310.35801749799998, 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.37544488199999998, 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167502 2023-10-09T09:15:44.567 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.568 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.568 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.568 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.569 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.137829+0000", 2023-10-09T09:15:44.569 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.569 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.569 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.569 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.569 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.137829+0000", 2023-10-09T09:15:44.569 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.137827+0000", 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.137841+0000", 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3709999999999999e-05 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.570 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.137858+0000", 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7884999999999999e-05 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.137870+0000", 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1555e-05 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.571 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.513260+0000", 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.37539012500000002 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.513274+0000", 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3835999999999999e-05 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.572 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:167502 9.7 9:e35a8163:::benchmark_data_smithi016_172771_object70916:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e238)", 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:10:34.158225+0000", 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 310.33762115899998, 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.59769279399999997, 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167502 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.573 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.158225+0000", 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.158225+0000", 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.574 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.158223+0000", 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999971 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.158239+0000", 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6682999999999999e-05 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.158264+0000", 2023-10-09T09:15:44.575 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4930999999999999e-05 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.158278+0000", 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3341e-05 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.513885+0000", 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35560718299999999 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.576 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.513895+0000", 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9240000000000002e-06 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.738944+0000", 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22504927 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.577 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.739059+0000", 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000114398 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,5,6,7", 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.739511+0000", 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00045277099999999998 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.742217+0000", 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002705889 2023-10-09T09:15:44.578 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.742569+0000", 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00035188299999999999 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.742956+0000", 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00038673400000000001 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.579 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.743689+0000", 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00073324700000000002 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.755773+0000", 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012083730000000001 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.755809+0000", 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6019999999999997e-05 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.580 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.755918+0000", 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109443 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:167516 9.12 9:4c5932d3:::benchmark_data_smithi016_172771_object70930:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:10:34.631682+0000", 2023-10-09T09:15:44.581 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 309.86416451299999, 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1141464989999998, 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167516 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.582 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631682+0000", 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631682+0000", 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631681+0000", 2023-10-09T09:15:44.583 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631690+0000", 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9199999999999993e-06 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631705+0000", 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5214e-05 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.584 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631713+0000", 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3780000000000001e-06 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631748+0000", 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5361999999999998e-05 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.585 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.631754+0000", 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4720000000000003e-06 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.682456+0000", 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.050701748 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.682470+0000", 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4242e-05 2023-10-09T09:15:44.586 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.732868+0000", 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.050397872 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.733001+0000", 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013343 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.733344+0000", 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00034310900000000002 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.738521+0000", 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0051765630000000003 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.738574+0000", 2023-10-09T09:15:44.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3443000000000003e-05 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.740042+0000", 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001468085 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.745697+0000", 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0056545570000000002 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.589 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.745740+0000", 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2809000000000003e-05 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.745829+0000", 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8743000000000001e-05 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.590 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:167526 9.12 9:4933e39c:::benchmark_data_smithi016_172771_object70940:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:10:34.765614+0000", 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 309.730232935, 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.980434684, 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167526 2023-10-09T09:15:44.591 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765614+0000", 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765614+0000", 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765613+0000", 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765619+0000", 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2879999999999998e-06 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.593 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765632+0000", 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3427999999999999e-05 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765640+0000", 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8210000000000002e-06 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765687+0000", 2023-10-09T09:15:44.594 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6385999999999999e-05 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.765697+0000", 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0378e-05 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.682502+0000", 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.91680468000000004 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.595 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.682512+0000", 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.065e-05 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.735164+0000", 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0526519050000001 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.596 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.735247+0000", 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2787000000000005e-05 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.735531+0000", 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00028364299999999998 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.738554+0000", 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0030237850000000002 2023-10-09T09:15:44.597 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.740049+0000", 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00149468 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.740089+0000", 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0059999999999999e-05 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.598 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.745890+0000", 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0058004340000000001 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.745942+0000", 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2064999999999997e-05 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.746048+0000", 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000106698 2023-10-09T09:15:44.599 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:167561 9.12 9:4c60fc5b:::benchmark_data_smithi016_172771_object70975:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:10:34.876340+0000", 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 309.619506489, 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.869966765, 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.600 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167561 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876340+0000", 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.601 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876340+0000", 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876338+0000", 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876347+0000", 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1130000000000001e-06 2023-10-09T09:15:44.602 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876367+0000", 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0136000000000002e-05 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876381+0000", 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4025999999999999e-05 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.603 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876446+0000", 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5809000000000006e-05 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:34.876457+0000", 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0614e-05 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.682545+0000", 2023-10-09T09:15:44.604 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.80608778199999997 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.682555+0000", 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0488e-05 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.736918+0000", 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.054362502 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.736998+0000", 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0525000000000006e-05 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6,7", 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.737185+0000", 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018697599999999999 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.740033+0000", 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028476909999999999 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.740055+0000", 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2334999999999999e-05 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.740122+0000", 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6478999999999998e-05 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.607 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.746111+0000", 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0059892110000000004 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.746143+0000", 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2128999999999997e-05 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.608 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.746307+0000", 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016365300000000001 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45290.0:167551 9.17 9:efd0b90c:::benchmark_data_smithi016_172771_object70965:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e240)", 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:10:35.679744+0000", 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 308.81610252799999, 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.057995558, 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.609 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45290", 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167551 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679744+0000", 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.610 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679744+0000", 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679741+0000", 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.611 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679752+0000", 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0091999999999999e-05 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679798+0000", 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6786000000000002e-05 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679812+0000", 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3689999999999999e-05 2023-10-09T09:15:44.612 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679923+0000", 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011078600000000001 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op must wait for map", 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.679933+0000", 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.041e-05 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.684362+0000", 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0044288779999999998 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:35.684376+0000", 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3535e-05 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.730627+0000", 2023-10-09T09:15:44.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0462510810000001 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.730762+0000", 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000135315 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,5,6,7", 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.731068+0000", 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000306457 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.615 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.733325+0000", 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022561149999999999 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.733579+0000", 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025470000000000001 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.735878+0000", 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022982609999999998 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.735929+0000", 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1851999999999998e-05 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.737586+0000", 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0016571019999999999 2023-10-09T09:15:44.617 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.737659+0000", 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2337999999999998e-05 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:10:36.737740+0000", 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0778999999999993e-05 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.618 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:101159 10.8 e256/248 10:11555de3:::benchmark_data_smithi016_193276_object101158:head v 256'6452, mlcod=256'6450)", 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:13:43.167337+0000", 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 121.328509318, 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27235795200000001, 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.619 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167337+0000", 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167337+0000", 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167336+0000", 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:15:44.620 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167359+0000", 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2203000000000001e-05 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167370+0000", 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1116e-05 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.621 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167376+0000", 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1979999999999998e-06 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167419+0000", 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3061e-05 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167442+0000", 2023-10-09T09:15:44.622 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2929e-05 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.439479+0000", 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.272036736 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.439695+0000", 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021662499999999999 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.623 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:137544 10.8 e263/261 10:1a62b621:::benchmark_data_smithi016_193276_object30321:head v 263'8769, mlcod=263'8765)", 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:36.495101+0000", 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 68.000745084000002, 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014940716999999999, 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.624 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495101+0000", 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495101+0000", 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495100+0000", 2023-10-09T09:15:44.625 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495109+0000", 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8140000000000007e-06 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495123+0000", 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3505e-05 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.626 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495128+0000", 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6629999999999998e-06 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495173+0000", 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5036e-05 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.627 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.495206+0000", 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2602999999999999e-05 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.509980+0000", 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014773514 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:36.510042+0000", 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2674000000000003e-05 2023-10-09T09:15:44.628 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:139344 10.f e263/261 10:f4dea1fb:::benchmark_data_smithi016_193276_object32121:head v 263'8656, mlcod=263'8653)", 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:39.900187+0000", 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 64.595659474000001, 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014776869999999999, 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.629 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900187+0000", 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900187+0000", 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900180+0000", 2023-10-09T09:15:44.630 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999928 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900203+0000", 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2687000000000001e-05 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900232+0000", 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9488e-05 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.631 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900248+0000", 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5945999999999999e-05 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900323+0000", 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4590000000000002e-05 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900363+0000", 2023-10-09T09:15:44.632 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9832000000000001e-05 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.914888+0000", 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014525323 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.914964+0000", 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5995999999999995e-05 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.633 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:139345 10.f e263/261 10:f9b65025:::benchmark_data_smithi016_193276_object32122:head v 263'8657, mlcod=263'8653)", 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:39.900277+0000", 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 64.595569394999998, 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014904742, 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.634 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900277+0000", 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900277+0000", 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900277+0000", 2023-10-09T09:15:44.635 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900284+0000", 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8480000000000001e-06 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900292+0000", 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8359999999999992e-06 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.636 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.900298+0000", 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4500000000000003e-06 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.903627+0000", 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0033296100000000002 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.903650+0000", 2023-10-09T09:15:44.637 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2328e-05 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.915129+0000", 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.01147953 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:39.915182+0000", 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2707999999999997e-05 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.638 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:144043 10.e e266/261 10:7fc0ff88:::benchmark_data_smithi016_193276_object36820:head v 266'8898, mlcod=266'8896)", 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:50.000863+0000", 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 54.494983173999998, 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014452075, 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.639 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.000863+0000", 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.000863+0000", 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.000862+0000", 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:15:44.640 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.000875+0000", 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3361e-05 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.000907+0000", 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1857999999999999e-05 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.641 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.000918+0000", 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1073000000000001e-05 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.001007+0000", 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8953e-05 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.001041+0000", 2023-10-09T09:15:44.642 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4094000000000003e-05 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.015136+0000", 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014094456999999999 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.015315+0000", 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000179853 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.643 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:144094 10.8 e266/261 10:1d685325:::benchmark_data_smithi016_193276_object36871:head v 266'9210, mlcod=266'9208)", 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:50.113976+0000", 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 54.381871029000003, 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018915276000000002, 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.644 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.113976+0000", 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.113976+0000", 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.113975+0000", 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.645 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.114063+0000", 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8321999999999999e-05 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.114134+0000", 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1051999999999997e-05 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.646 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.114143+0000", 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3160000000000003e-06 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.114217+0000", 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3368999999999995e-05 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.114244+0000", 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6996999999999999e-05 2023-10-09T09:15:44.647 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.132763+0000", 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018519273999999999 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.132891+0000", 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012799900000000001 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.648 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:144095 10.8 e266/261 10:12e7ab24:::benchmark_data_smithi016_193276_object36872:head v 266'9211, mlcod=266'9208)", 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:50.119891+0000", 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 54.375956002999999, 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016194262000000001, 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.119891+0000", 2023-10-09T09:15:44.649 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.119891+0000", 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.119889+0000", 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:15:44.650 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.119900+0000", 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0210999999999999e-05 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.119922+0000", 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2268000000000001e-05 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.651 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.119932+0000", 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0131e-05 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.128386+0000", 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0084539850000000007 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.128420+0000", 2023-10-09T09:15:44.652 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3813999999999998e-05 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.135939+0000", 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0075194620000000002 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:50.136085+0000", 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014553299999999999 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.653 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.654 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.654 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:15:44.654 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:15:44.654 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:15:44.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:44.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:44.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:44.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:44.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:44.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:44.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:44.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:44.937 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:44.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:45.041 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:45.069 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:45.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:45.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:45.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:45.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:45.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:45.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:45.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:45.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:45.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:45.483 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:45.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:45.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:45.632 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:45.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:45.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:45.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:45 smithi094 bash[59677]: cluster 2023-10-09T09:15:44.124267+0000 mgr.y (mgr.25421) 1153 : cluster 0 pgmap v1252: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 10345/133977 objects degraded (7.721%); 17819/133977 objects misplaced (13.300%); 14 MiB/s, 230 objects/s recovering 2023-10-09T09:15:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:45 smithi094 bash[59677]: cluster 2023-10-09T09:15:44.750487+0000 mon.a (mon.0) 1538 : cluster 3 Health check update: Degraded data redundancy: 10345/133977 objects degraded (7.721%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:45.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:45.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:45.805 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:45 smithi016 bash[75179]: cluster 2023-10-09T09:15:44.124267+0000 mgr.y (mgr.25421) 1153 : cluster 0 pgmap v1252: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 10345/133977 objects degraded (7.721%); 17819/133977 objects misplaced (13.300%); 14 MiB/s, 230 objects/s recovering 2023-10-09T09:15:45.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:45 smithi016 bash[75179]: cluster 2023-10-09T09:15:44.750487+0000 mon.a (mon.0) 1538 : cluster 3 Health check update: Degraded data redundancy: 10345/133977 objects degraded (7.721%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:45.806 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:45 smithi016 bash[77892]: cluster 2023-10-09T09:15:44.124267+0000 mgr.y (mgr.25421) 1153 : cluster 0 pgmap v1252: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 10345/133977 objects degraded (7.721%); 17819/133977 objects misplaced (13.300%); 14 MiB/s, 230 objects/s recovering 2023-10-09T09:15:45.806 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:45 smithi016 bash[77892]: cluster 2023-10-09T09:15:44.750487+0000 mon.a (mon.0) 1538 : cluster 3 Health check update: Degraded data redundancy: 10345/133977 objects degraded (7.721%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:45.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:45.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:45.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:45.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:46.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:46.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:46.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:46.137 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:15:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:15:45] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-09T09:15:46.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:46.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:46.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:46.398 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:46.431 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:46.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:46.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:46.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:46.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:46.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:46.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:46.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:46.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:46.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:46.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:47.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:47.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:47.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:47.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:47.226 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:47.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:47.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:47.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:47.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:47.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:47.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:47.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:47 smithi094 bash[59677]: cluster 2023-10-09T09:15:46.125130+0000 mgr.y (mgr.25421) 1154 : cluster 0 pgmap v1253: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 10218/129432 objects degraded (7.894%); 15739/129432 objects misplaced (12.160%); 29 MiB/s, 460 objects/s recovering 2023-10-09T09:15:47.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:47.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:47.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:47.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:47.633 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:47.661 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:47.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:47.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:47 smithi016 bash[75179]: cluster 2023-10-09T09:15:46.125130+0000 mgr.y (mgr.25421) 1154 : cluster 0 pgmap v1253: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 10218/129432 objects degraded (7.894%); 15739/129432 objects misplaced (12.160%); 29 MiB/s, 460 objects/s recovering 2023-10-09T09:15:47.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:47 smithi016 bash[77892]: cluster 2023-10-09T09:15:46.125130+0000 mgr.y (mgr.25421) 1154 : cluster 0 pgmap v1253: 89 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 10218/129432 objects degraded (7.894%); 15739/129432 objects misplaced (12.160%); 29 MiB/s, 460 objects/s recovering 2023-10-09T09:15:47.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:47.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:47.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:47.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:47.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:47.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:48.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:48.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:48.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:48.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:48.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:48.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:48.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:48.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:48.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:48.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:48.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:48.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:48.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:48.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:48.526 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:48.554 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:48.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:48.658 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:48.707 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:48.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:48.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:48 smithi094 bash[59677]: cluster 2023-10-09T09:15:47.486375+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e277: 8 total, 8 up, 7 in 2023-10-09T09:15:48.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:48 smithi094 bash[59677]: cluster 2023-10-09T09:15:47.487041+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:48 smithi094 bash[59677]: cluster 2023-10-09T09:15:47.487149+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:48 smithi094 bash[59677]: cluster 2023-10-09T09:15:47.488724+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[75179]: cluster 2023-10-09T09:15:47.486375+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e277: 8 total, 8 up, 7 in 2023-10-09T09:15:48.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[75179]: cluster 2023-10-09T09:15:47.487041+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[75179]: cluster 2023-10-09T09:15:47.487149+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[75179]: cluster 2023-10-09T09:15:47.488724+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[77892]: cluster 2023-10-09T09:15:47.486375+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e277: 8 total, 8 up, 7 in 2023-10-09T09:15:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[77892]: cluster 2023-10-09T09:15:47.487041+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[77892]: cluster 2023-10-09T09:15:47.487149+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:48 smithi016 bash[77892]: cluster 2023-10-09T09:15:47.488724+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T09:15:48.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:15:48.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:48.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:48.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:48.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:48.971 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:49.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:49.157 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:49.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:49.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:49.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:49.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:49.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:49.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:49.546 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:49.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:49.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:49.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:49.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: cluster 2023-10-09T09:15:48.125586+0000 mgr.y (mgr.25421) 1155 : cluster 0 pgmap v1255: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 10218/128373 objects degraded (7.960%); 15563/128373 objects misplaced (12.123%); 28 MiB/s, 454 objects/s recovering 2023-10-09T09:15:49.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: cluster 2023-10-09T09:15:48.489994+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: cluster 2023-10-09T09:15:48.493251+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e278: 8 total, 8 up, 7 in 2023-10-09T09:15:49.698 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: cluster 2023-10-09T09:15:48.493532+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: cluster 2023-10-09T09:15:48.496692+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: cluster 2023-10-09T09:15:48.545607+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: audit 2023-10-09T09:15:49.249623+0000 mon.a (mon.0) 1541 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:49.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: audit 2023-10-09T09:15:49.381058+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:49.699 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[75179]: audit 2023-10-09T09:15:49.391054+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:49.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: cluster 2023-10-09T09:15:48.125586+0000 mgr.y (mgr.25421) 1155 : cluster 0 pgmap v1255: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 10218/128373 objects degraded (7.960%); 15563/128373 objects misplaced (12.123%); 28 MiB/s, 454 objects/s recovering 2023-10-09T09:15:49.699 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: cluster 2023-10-09T09:15:48.489994+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: cluster 2023-10-09T09:15:48.493251+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e278: 8 total, 8 up, 7 in 2023-10-09T09:15:49.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: cluster 2023-10-09T09:15:48.493532+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: cluster 2023-10-09T09:15:48.496692+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: cluster 2023-10-09T09:15:48.545607+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: audit 2023-10-09T09:15:49.249623+0000 mon.a (mon.0) 1541 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:49.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: audit 2023-10-09T09:15:49.381058+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:49.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:49 smithi016 bash[77892]: audit 2023-10-09T09:15:49.391054+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:49.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:49.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:49.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:49.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:49.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:49.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:49.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: cluster 2023-10-09T09:15:48.125586+0000 mgr.y (mgr.25421) 1155 : cluster 0 pgmap v1255: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 10218/128373 objects degraded (7.960%); 15563/128373 objects misplaced (12.123%); 28 MiB/s, 454 objects/s recovering 2023-10-09T09:15:49.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: cluster 2023-10-09T09:15:48.489994+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: cluster 2023-10-09T09:15:48.493251+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e278: 8 total, 8 up, 7 in 2023-10-09T09:15:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: cluster 2023-10-09T09:15:48.493532+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: cluster 2023-10-09T09:15:48.496692+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: cluster 2023-10-09T09:15:48.545607+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T09:15:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: audit 2023-10-09T09:15:49.249623+0000 mon.a (mon.0) 1541 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: audit 2023-10-09T09:15:49.381058+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:49 smithi094 bash[59677]: audit 2023-10-09T09:15:49.391054+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:50.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:50.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:50.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:50.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:50.267 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:50.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:50.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:50.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:50.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:50.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:50.496 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:50.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:50.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:50.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:50.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:50.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:50 smithi016 bash[75179]: cluster 2023-10-09T09:15:49.751616+0000 mon.a (mon.0) 1544 : cluster 3 Health check update: Degraded data redundancy: 10218/128373 objects degraded (7.960%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:50.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:50 smithi016 bash[77892]: cluster 2023-10-09T09:15:49.751616+0000 mon.a (mon.0) 1544 : cluster 3 Health check update: Degraded data redundancy: 10218/128373 objects degraded (7.960%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:50.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:50.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:50.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:50.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:50.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:50.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:50.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:50.903 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:50.911 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:50 smithi094 bash[59677]: cluster 2023-10-09T09:15:49.751616+0000 mon.a (mon.0) 1544 : cluster 3 Health check update: Degraded data redundancy: 10218/128373 objects degraded (7.960%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:50.927 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:50.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:51.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:51.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:51.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:51.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:51.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:51.194 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:51.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:51.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:51.301 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:51.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:51.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:51.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:51.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:51.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:51.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:51.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:51.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:51.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:51.739 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:51.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:51.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:51 smithi016 bash[75179]: cluster 2023-10-09T09:15:50.126105+0000 mgr.y (mgr.25421) 1156 : cluster 0 pgmap v1257: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10218/128373 objects degraded (7.960%); 15563/128373 objects misplaced (12.123%); 16 MiB/s, 259 objects/s recovering 2023-10-09T09:15:51.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:51 smithi016 bash[77892]: cluster 2023-10-09T09:15:50.126105+0000 mgr.y (mgr.25421) 1156 : cluster 0 pgmap v1257: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10218/128373 objects degraded (7.960%); 15563/128373 objects misplaced (12.123%); 16 MiB/s, 259 objects/s recovering 2023-10-09T09:15:51.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:51.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:51.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:51.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:51.971 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:51.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:51 smithi094 bash[59677]: cluster 2023-10-09T09:15:50.126105+0000 mgr.y (mgr.25421) 1156 : cluster 0 pgmap v1257: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10218/128373 objects degraded (7.960%); 15563/128373 objects misplaced (12.123%); 16 MiB/s, 259 objects/s recovering 2023-10-09T09:15:52.012 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:52.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:52.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:52.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:52.273 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:52.378 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:52.415 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:52.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:52.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:52.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:52.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:52.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:52.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:52.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:53.028 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:53.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:53.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:53.158 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:53.195 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:53.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:53.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:53.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:53.431 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:53.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:53.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:53.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:53.608 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:53.714 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:53.722 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:53 smithi094 bash[59677]: cluster 2023-10-09T09:15:52.126838+0000 mgr.y (mgr.25421) 1157 : cluster 0 pgmap v1258: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 10077/123651 objects degraded (8.150%); 14053/123651 objects misplaced (11.365%); 25 MiB/s, 406 objects/s recovering 2023-10-09T09:15:53.739 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:53.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:53.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:53 smithi016 bash[75179]: cluster 2023-10-09T09:15:52.126838+0000 mgr.y (mgr.25421) 1157 : cluster 0 pgmap v1258: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 10077/123651 objects degraded (8.150%); 14053/123651 objects misplaced (11.365%); 25 MiB/s, 406 objects/s recovering 2023-10-09T09:15:53.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:53 smithi016 bash[77892]: cluster 2023-10-09T09:15:52.126838+0000 mgr.y (mgr.25421) 1157 : cluster 0 pgmap v1258: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 10077/123651 objects degraded (8.150%); 14053/123651 objects misplaced (11.365%); 25 MiB/s, 406 objects/s recovering 2023-10-09T09:15:53.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:53.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:53.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:53.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:53.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:53.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:53.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:54.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:54.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:54.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:54.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:54.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:54.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:54.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:54.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:54.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:54.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:54.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:54.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:54.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:54.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:54.632 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:54.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:54.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:54.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[75179]: cluster 2023-10-09T09:15:53.499074+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T09:15:54.745 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[75179]: cluster 2023-10-09T09:15:53.501674+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.745 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[75179]: cluster 2023-10-09T09:15:53.501674+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.745 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[75179]: cluster 2023-10-09T09:15:53.503818+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[77892]: cluster 2023-10-09T09:15:53.499074+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T09:15:54.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[77892]: cluster 2023-10-09T09:15:53.501674+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[77892]: cluster 2023-10-09T09:15:53.501674+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:54 smithi016 bash[77892]: cluster 2023-10-09T09:15:53.503818+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:54.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:54.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:54.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:54 smithi094 bash[59677]: cluster 2023-10-09T09:15:53.499074+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T09:15:54.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:54 smithi094 bash[59677]: cluster 2023-10-09T09:15:53.501674+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.883 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:54 smithi094 bash[59677]: cluster 2023-10-09T09:15:53.501674+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.884 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:54 smithi094 bash[59677]: cluster 2023-10-09T09:15:53.503818+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T09:15:54.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:54.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:55.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:55.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:55.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:55.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:55.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:55.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:55.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:55.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:55.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:55.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:55.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:55.408 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:55.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:55.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:55.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:55.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:55.544 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[75179]: cluster 2023-10-09T09:15:54.127416+0000 mgr.y (mgr.25421) 1158 : cluster 0 pgmap v1260: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 313 op/s; 10077/122127 objects degraded (8.251%); 13759/122127 objects misplaced (11.266%); 14 MiB/s, 225 objects/s recovering 2023-10-09T09:15:55.544 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[75179]: cluster 2023-10-09T09:15:54.513365+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T09:15:55.544 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[75179]: cluster 2023-10-09T09:15:54.514353+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T09:15:55.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[75179]: cluster 2023-10-09T09:15:54.752943+0000 mon.a (mon.0) 1547 : cluster 3 Health check update: Degraded data redundancy: 10077/122127 objects degraded (8.251%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:55.545 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[77892]: cluster 2023-10-09T09:15:54.127416+0000 mgr.y (mgr.25421) 1158 : cluster 0 pgmap v1260: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 313 op/s; 10077/122127 objects degraded (8.251%); 13759/122127 objects misplaced (11.266%); 14 MiB/s, 225 objects/s recovering 2023-10-09T09:15:55.545 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[77892]: cluster 2023-10-09T09:15:54.513365+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T09:15:55.545 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[77892]: cluster 2023-10-09T09:15:54.514353+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T09:15:55.545 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[77892]: cluster 2023-10-09T09:15:54.752943+0000 mon.a (mon.0) 1547 : cluster 3 Health check update: Degraded data redundancy: 10077/122127 objects degraded (8.251%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:55.579 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:55.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:55.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:55.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:55.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:55.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:55 smithi094 bash[59677]: cluster 2023-10-09T09:15:54.127416+0000 mgr.y (mgr.25421) 1158 : cluster 0 pgmap v1260: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 313 op/s; 10077/122127 objects degraded (8.251%); 13759/122127 objects misplaced (11.266%); 14 MiB/s, 225 objects/s recovering 2023-10-09T09:15:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:55 smithi094 bash[59677]: cluster 2023-10-09T09:15:54.513365+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T09:15:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:55 smithi094 bash[59677]: cluster 2023-10-09T09:15:54.514353+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T09:15:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:55 smithi094 bash[59677]: cluster 2023-10-09T09:15:54.752943+0000 mon.a (mon.0) 1547 : cluster 3 Health check update: Degraded data redundancy: 10077/122127 objects degraded (8.251%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:15:55.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:55.821 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:15:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:15:55] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T09:15:55.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:55.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:55.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:55.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:55.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:56.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:56.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:56.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:56.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:56.370 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:56.401 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:56.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:56.504 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:56.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:56.543 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:15:56.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:56.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:56.658 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:56 smithi094 bash[59677]: audit 2023-10-09T09:15:56.428304+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.16:0/1792426077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:15:56.658 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:56 smithi094 bash[59677]: audit 2023-10-09T09:15:56.430149+0000 mon.a (mon.0) 1548 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:15:56.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:56.785 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:56.834 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:56.848 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:56 smithi016 bash[75179]: audit 2023-10-09T09:15:56.428304+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.16:0/1792426077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:15:56.848 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:56 smithi016 bash[75179]: audit 2023-10-09T09:15:56.430149+0000 mon.a (mon.0) 1548 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:15:56.848 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:56 smithi016 bash[77892]: audit 2023-10-09T09:15:56.428304+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.16:0/1792426077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:15:56.849 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:56 smithi016 bash[77892]: audit 2023-10-09T09:15:56.430149+0000 mon.a (mon.0) 1548 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:15:56.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:56.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:56.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:56.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:57.048 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:15:57.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:57.086 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:15:57.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:57.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:57.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:57.123 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:15:57.148 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:15:57.174 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:15:57.202 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:15:57.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:57.236 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:15:57.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:57.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:57.266 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:15:57.269 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.269 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.269 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.269 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.270 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.270 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.270 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.270 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:15:57.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:57.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:57.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:57.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:57.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:57.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:57.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:57.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:57.619 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: cluster 2023-10-09T09:15:56.128311+0000 mgr.y (mgr.25421) 1159 : cluster 0 pgmap v1262: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 9935/117357 objects degraded (8.466%); 12464/117357 objects misplaced (10.621%); 22 MiB/s, 348 objects/s recovering 2023-10-09T09:15:57.619 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: cluster 2023-10-09T09:15:56.511795+0000 mon.a (mon.0) 1549 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:15:57.619 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: cluster 2023-10-09T09:15:56.520903+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T09:15:57.619 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: audit 2023-10-09T09:15:56.523111+0000 mon.a (mon.0) 1550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:15:57.619 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: cluster 2023-10-09T09:15:56.523254+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T09:15:57.619 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: audit 2023-10-09T09:15:56.753239+0000 mon.a (mon.0) 1552 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: audit 2023-10-09T09:15:56.764500+0000 mon.a (mon.0) 1553 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: audit 2023-10-09T09:15:57.006834+0000 mon.a (mon.0) 1554 : audit 0 from='client.? 172.21.15.16:0/1917573470' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: audit 2023-10-09T09:15:57.346673+0000 mon.a (mon.0) 1555 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: audit 2023-10-09T09:15:57.348115+0000 mon.a (mon.0) 1556 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[75179]: audit 2023-10-09T09:15:57.354455+0000 mon.a (mon.0) 1557 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: cluster 2023-10-09T09:15:56.128311+0000 mgr.y (mgr.25421) 1159 : cluster 0 pgmap v1262: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 9935/117357 objects degraded (8.466%); 12464/117357 objects misplaced (10.621%); 22 MiB/s, 348 objects/s recovering 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: cluster 2023-10-09T09:15:56.511795+0000 mon.a (mon.0) 1549 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: cluster 2023-10-09T09:15:56.520903+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T09:15:57.620 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: audit 2023-10-09T09:15:56.523111+0000 mon.a (mon.0) 1550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:15:57.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: cluster 2023-10-09T09:15:56.523254+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T09:15:57.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: audit 2023-10-09T09:15:56.753239+0000 mon.a (mon.0) 1552 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: audit 2023-10-09T09:15:56.764500+0000 mon.a (mon.0) 1553 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: audit 2023-10-09T09:15:57.006834+0000 mon.a (mon.0) 1554 : audit 0 from='client.? 172.21.15.16:0/1917573470' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:15:57.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: audit 2023-10-09T09:15:57.346673+0000 mon.a (mon.0) 1555 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:57.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: audit 2023-10-09T09:15:57.348115+0000 mon.a (mon.0) 1556 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:57.621 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:57 smithi016 bash[77892]: audit 2023-10-09T09:15:57.354455+0000 mon.a (mon.0) 1557 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:57.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:57.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:57.750 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:57.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:57.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:57.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:57.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:57.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:57.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:57.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:57.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: cluster 2023-10-09T09:15:56.128311+0000 mgr.y (mgr.25421) 1159 : cluster 0 pgmap v1262: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 9935/117357 objects degraded (8.466%); 12464/117357 objects misplaced (10.621%); 22 MiB/s, 348 objects/s recovering 2023-10-09T09:15:57.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: cluster 2023-10-09T09:15:56.511795+0000 mon.a (mon.0) 1549 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:15:57.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: cluster 2023-10-09T09:15:56.520903+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T09:15:57.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: audit 2023-10-09T09:15:56.523111+0000 mon.a (mon.0) 1550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:15:57.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: cluster 2023-10-09T09:15:56.523254+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T09:15:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: audit 2023-10-09T09:15:56.753239+0000 mon.a (mon.0) 1552 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: audit 2023-10-09T09:15:56.764500+0000 mon.a (mon.0) 1553 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: audit 2023-10-09T09:15:57.006834+0000 mon.a (mon.0) 1554 : audit 0 from='client.? 172.21.15.16:0/1917573470' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:15:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: audit 2023-10-09T09:15:57.346673+0000 mon.a (mon.0) 1555 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:57.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: audit 2023-10-09T09:15:57.348115+0000 mon.a (mon.0) 1556 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:57.993 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:57 smithi094 bash[59677]: audit 2023-10-09T09:15:57.354455+0000 mon.a (mon.0) 1557 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:15:58.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:58.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:58.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:58.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:58.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:58.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:58.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:58.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:58.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:58.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:58.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:58.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:15:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:15:58.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:58.663 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:15:58.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:58.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:15:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:15:58.902 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:58.936 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:15:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:58.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:15:59.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:59.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:15:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:59.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:15:59.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:59.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:15:59.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:59.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:15:59.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:59.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:59.440 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:59.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:15:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:59.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:15:59.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.581 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:15:59 smithi094 bash[59677]: cluster 2023-10-09T09:15:58.129026+0000 mgr.y (mgr.25421) 1160 : cluster 0 pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 9935/115818 objects degraded (8.578%); 12166/115818 objects misplaced (10.504%); 16 MiB/s, 249 objects/s recovering 2023-10-09T09:15:59.606 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:59.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:59.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.739 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:59.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:59.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.878 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:15:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:15:59.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:15:59.985 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:15:59.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:15:59 smithi016 bash[77892]: cluster 2023-10-09T09:15:58.129026+0000 mgr.y (mgr.25421) 1160 : cluster 0 pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 9935/115818 objects degraded (8.578%); 12166/115818 objects misplaced (10.504%); 16 MiB/s, 249 objects/s recovering 2023-10-09T09:15:59.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:15:59 smithi016 bash[75179]: cluster 2023-10-09T09:15:58.129026+0000 mgr.y (mgr.25421) 1160 : cluster 0 pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 9935/115818 objects degraded (8.578%); 12166/115818 objects misplaced (10.504%); 16 MiB/s, 249 objects/s recovering 2023-10-09T09:16:00.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:00.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:00.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:00.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:00.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:00.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:00.408 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:00.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:00.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:00.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:00 smithi094 bash[59677]: cluster 2023-10-09T09:15:59.754086+0000 mon.a (mon.0) 1558 : cluster 3 Health check update: Degraded data redundancy: 9935/115818 objects degraded (8.578%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:00.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:00.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:00.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:00.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:00.702 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:00.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:00.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:00.954 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:00.978 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:00.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:00.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:00.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:00 smithi016 bash[75179]: cluster 2023-10-09T09:15:59.754086+0000 mon.a (mon.0) 1558 : cluster 3 Health check update: Degraded data redundancy: 9935/115818 objects degraded (8.578%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:00.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:00 smithi016 bash[77892]: cluster 2023-10-09T09:15:59.754086+0000 mon.a (mon.0) 1558 : cluster 3 Health check update: Degraded data redundancy: 9935/115818 objects degraded (8.578%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:01.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:01.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:01.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:01.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:01.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:01.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:01.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:01.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:01.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:01.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:01.495 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:01.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:01.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:01.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:01.679 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:01.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:01.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:01.745 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:01.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:01 smithi016 bash[75179]: cluster 2023-10-09T09:16:00.129646+0000 mgr.y (mgr.25421) 1161 : cluster 0 pgmap v1265: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 317 op/s; 9935/115818 objects degraded (8.578%); 12166/115818 objects misplaced (10.504%); 13 MiB/s, 200 objects/s recovering 2023-10-09T09:16:01.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:01 smithi016 bash[77892]: cluster 2023-10-09T09:16:00.129646+0000 mgr.y (mgr.25421) 1161 : cluster 0 pgmap v1265: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 317 op/s; 9935/115818 objects degraded (8.578%); 12166/115818 objects misplaced (10.504%); 13 MiB/s, 200 objects/s recovering 2023-10-09T09:16:01.785 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:01.794 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:01 smithi094 bash[59677]: cluster 2023-10-09T09:16:00.129646+0000 mgr.y (mgr.25421) 1161 : cluster 0 pgmap v1265: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 317 op/s; 9935/115818 objects degraded (8.578%); 12166/115818 objects misplaced (10.504%); 13 MiB/s, 200 objects/s recovering 2023-10-09T09:16:01.814 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:16:01.819 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:01.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:01.855 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:16:01.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:01.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:01.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:02.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:02.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:02.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:02.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:02.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:02.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:02.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:02.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:02.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:02.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:02.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:02.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:02.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:02.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:02.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:02.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:02.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:02.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:03.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:03.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:03.217 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:03.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:03.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:03.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:16:03.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:16:03.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:03.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:03.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:03.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:03.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:03.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:03.699 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:03.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:03.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:03.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:03.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:03.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:03.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:03.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:03.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:03 smithi094 bash[59677]: audit 2023-10-09T09:16:01.801166+0000 mgr.y (mgr.25421) 1162 : audit 0 from='client.45614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:16:03.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:03 smithi094 bash[59677]: cluster 2023-10-09T09:16:02.130407+0000 mgr.y (mgr.25421) 1163 : cluster 0 pgmap v1266: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 9759/109998 objects degraded (8.872%); 11441/109998 objects misplaced (10.401%); 15 MiB/s, 244 objects/s recovering 2023-10-09T09:16:04.066 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:04.106 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:04.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:04.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:04.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:04.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:03 smithi016 bash[75179]: audit 2023-10-09T09:16:01.801166+0000 mgr.y (mgr.25421) 1162 : audit 0 from='client.45614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:16:04.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:03 smithi016 bash[75179]: cluster 2023-10-09T09:16:02.130407+0000 mgr.y (mgr.25421) 1163 : cluster 0 pgmap v1266: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 9759/109998 objects degraded (8.872%); 11441/109998 objects misplaced (10.401%); 15 MiB/s, 244 objects/s recovering 2023-10-09T09:16:04.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:03 smithi016 bash[77892]: audit 2023-10-09T09:16:01.801166+0000 mgr.y (mgr.25421) 1162 : audit 0 from='client.45614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:16:04.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:03 smithi016 bash[77892]: cluster 2023-10-09T09:16:02.130407+0000 mgr.y (mgr.25421) 1163 : cluster 0 pgmap v1266: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 9759/109998 objects degraded (8.872%); 11441/109998 objects misplaced (10.401%); 15 MiB/s, 244 objects/s recovering 2023-10-09T09:16:04.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:04.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:04.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:04.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:04.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:04.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:04.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:04.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:04.617 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:04.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:04.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:04.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:04.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:04.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:04.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:04.878 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:04.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:04.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:05.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:05.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:05.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:05.276 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1265,"stamp":"2023-10-09T09:16:00.129232+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2529040312,"num_objects":38606,"num_object_clones":0,"num_object_copies":115818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9935,"num_objects_misplaced":12166,"num_objects_unfound":0,"num_objects_dirty":38606,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":176120,"num_write_kb":6864526,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20574,"num_bytes_recovered":1344689574,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122382,"ondisk_log_size":122382,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12539608,"kb_used_data":7404064,"kb_used_omap":44,"kb_used_meta":5135379,"kb_avail":643676456,"statfs":{"total":671965249536,"available":659124690944,"internally_reserved":0,"allocated":7581761536,"data_stored":7576347666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":5258628513},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,3,5,1,1,2,0,2,12],"upper_bound":2048},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-137822208,"num_objects":-2103,"num_object_clones":0,"num_object_copies":-6309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-142,"num_objects_misplaced":-1593,"num_objects_unfound":0,"num_objects_dirty":-2103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2103,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1331,"num_bytes_recovered":87228416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.629799"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":160,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.519100+0000","last_change":"2023-10-09T09:14:48.320488+0000","last_active":"2023-10-09T09:15:53.519100+0000","last_peered":"2023-10-09T09:15:53.519100+0000","last_clean":"2023-10-09T09:15:53.519100+0000","last_became_active":"2023-10-09T09:14:48.320104+0000","last_became_peered":"2023-10-09T09:14:48.320104+0000","last_unstale":"2023-10-09T09:15:53.519100+0000","last_undegraded":"2023-10-09T09:15:53.519100+0000","last_fullsized":"2023-10-09T09:15:53.519100+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520259+0000","last_change":"2023-10-09T09:14:48.316760+0000","last_active":"2023-10-09T09:15:54.520259+0000","last_peered":"2023-10-09T09:15:54.520259+0000","last_clean":"2023-10-09T09:15:54.520259+0000","last_became_active":"2023-10-09T09:14:48.316432+0000","last_became_peered":"2023-10-09T09:14:48.316432+0000","last_unstale":"2023-10-09T09:15:54.520259+0000","last_undegraded":"2023-10-09T09:15:54.520259+0000","last_fullsized":"2023-10-09T09:15:54.520259+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:41.630374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519896+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:15:54.519896+0000","last_peered":"2023-10-09T09:15:54.519896+0000","last_clean":"2023-10-09T09:15:54.519896+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:15:54.519896+0000","last_undegraded":"2023-10-09T09:15:54.519896+0000","last_fullsized":"2023-10-09T09:15:54.519896+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":316,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T09:15:56.542624+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:15:56.542624+0000","last_peered":"2023-10-09T09:15:56.542624+0000","last_clean":"2023-10-09T09:15:56.542624+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:15:56.542624+0000","last_undegraded":"2023-10-09T09:15:56.542624+0000","last_fullsized":"2023-10-09T09:15:56.542624+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":259,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520061+0000","last_change":"2023-10-09T09:14:48.312713+0000","last_active":"2023-10-09T09:15:54.520061+0000","last_peered":"2023-10-09T09:15:54.520061+0000","last_clean":"2023-10-09T09:15:54.520061+0000","last_became_active":"2023-10-09T09:14:48.312429+0000","last_became_peered":"2023-10-09T09:14:48.312429+0000","last_unstale":"2023-10-09T09:15:54.520061+0000","last_undegraded":"2023-10-09T09:15:54.520061+0000","last_fullsized":"2023-10-09T09:15:54.520061+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:56.755105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":174,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508910+0000","last_change":"2023-10-09T09:14:48.310212+0000","last_active":"2023-10-09T09:15:53.508910+0000","last_peered":"2023-10-09T09:15:53.508910+0000","last_clean":"2023-10-09T09:15:53.508910+0000","last_became_active":"2023-10-09T09:14:48.309682+0000","last_became_peered":"2023-10-09T09:14:48.309682+0000","last_unstale":"2023-10-09T09:15:53.508910+0000","last_undegraded":"2023-10-09T09:15:53.508910+0000","last_fullsized":"2023-10-09T09:15:53.508910+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":268,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.525414+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:15:54.525414+0000","last_peered":"2023-10-09T09:15:54.525414+0000","last_clean":"2023-10-09T09:15:54.525414+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:15:54.525414+0000","last_undegraded":"2023-10-09T09:15:54.525414+0000","last_fullsized":"2023-10-09T09:15:54.525414+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519561+0000","last_change":"2023-10-09T09:14:48.306040+0000","last_active":"2023-10-09T09:15:54.519561+0000","last_peered":"2023-10-09T09:15:54.519561+0000","last_clean":"2023-10-09T09:15:54.519561+0000","last_became_active":"2023-10-09T09:14:48.304137+0000","last_became_peered":"2023-10-09T09:14:48.304137+0000","last_unstale":"2023-10-09T09:15:54.519561+0000","last_undegraded":"2023-10-09T09:15:54.519561+0000","last_fullsized":"2023-10-09T09:15:54.519561+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:47.446761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":85,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.512130+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:15:53.512130+0000","last_peered":"2023-10-09T09:15:53.512130+0000","last_clean":"2023-10-09T09:15:53.512130+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:15:53.512130+0000","last_undegraded":"2023-10-09T09:15:53.512130+0000","last_fullsized":"2023-10-09T09:15:53.512130+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":243,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508215+0000","last_change":"2023-10-09T09:13:48.190068+0000","last_active":"2023-10-09T09:15:53.508215+0000","last_peered":"2023-10-09T09:15:53.508215+0000","last_clean":"2023-10-09T09:15:53.508215+0000","last_became_active":"2023-10-09T09:13:48.189740+0000","last_became_peered":"2023-10-09T09:13:48.189740+0000","last_unstale":"2023-10-09T09:15:53.508215+0000","last_undegraded":"2023-10-09T09:15:53.508215+0000","last_fullsized":"2023-10-09T09:15:53.508215+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":66,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.509461+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:15:53.509461+0000","last_peered":"2023-10-09T09:15:53.509461+0000","last_clean":"2023-10-09T09:15:53.509461+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:15:53.509461+0000","last_undegraded":"2023-10-09T09:15:53.509461+0000","last_fullsized":"2023-10-09T09:15:53.509461+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520225+0000","last_change":"2023-10-09T09:14:48.303627+0000","last_active":"2023-10-09T09:15:54.520225+0000","last_peered":"2023-10-09T09:15:54.520225+0000","last_clean":"2023-10-09T09:15:54.520225+0000","last_became_active":"2023-10-09T09:14:48.303287+0000","last_became_peered":"2023-10-09T09:14:48.303287+0000","last_unstale":"2023-10-09T09:15:54.520225+0000","last_undegraded":"2023-10-09T09:15:54.520225+0000","last_fullsized":"2023-10-09T09:15:54.520225+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:27.004752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":181,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.520316+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:15:53.520316+0000","last_peered":"2023-10-09T09:15:53.520316+0000","last_clean":"2023-10-09T09:15:53.520316+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:15:53.520316+0000","last_undegraded":"2023-10-09T09:15:53.520316+0000","last_fullsized":"2023-10-09T09:15:53.520316+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":169,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.556123+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:15:53.556123+0000","last_peered":"2023-10-09T09:15:53.556123+0000","last_clean":"2023-10-09T09:15:53.556123+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:15:53.556123+0000","last_undegraded":"2023-10-09T09:15:53.556123+0000","last_fullsized":"2023-10-09T09:15:53.556123+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":81,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.509254+0000","last_change":"2023-10-09T09:14:19.878196+0000","last_active":"2023-10-09T09:15:53.509254+0000","last_peered":"2023-10-09T09:15:53.509254+0000","last_clean":"2023-10-09T09:15:53.509254+0000","last_became_active":"2023-10-09T09:14:19.750467+0000","last_became_peered":"2023-10-09T09:14:19.750467+0000","last_unstale":"2023-10-09T09:15:53.509254+0000","last_undegraded":"2023-10-09T09:15:53.509254+0000","last_fullsized":"2023-10-09T09:15:53.509254+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520267+0000","last_change":"2023-10-09T09:14:48.319108+0000","last_active":"2023-10-09T09:15:54.520267+0000","last_peered":"2023-10-09T09:15:54.520267+0000","last_clean":"2023-10-09T09:15:54.520267+0000","last_became_active":"2023-10-09T09:14:48.318443+0000","last_became_peered":"2023-10-09T09:14:48.318443+0000","last_unstale":"2023-10-09T09:15:54.520267+0000","last_undegraded":"2023-10-09T09:15:54.520267+0000","last_fullsized":"2023-10-09T09:15:54.520267+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:31.939020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":295,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.527310+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:15:54.527310+0000","last_peered":"2023-10-09T09:15:54.527310+0000","last_clean":"2023-10-09T09:15:54.527310+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:15:54.527310+0000","last_undegraded":"2023-10-09T09:15:54.527310+0000","last_fullsized":"2023-10-09T09:15:54.527310+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":79,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.509331+0000","last_change":"2023-10-09T09:15:23.415336+0000","last_active":"2023-10-09T09:15:53.509331+0000","last_peered":"2023-10-09T09:15:53.509331+0000","last_clean":"2023-10-09T09:15:53.509331+0000","last_became_active":"2023-10-09T09:15:23.414900+0000","last_became_peered":"2023-10-09T09:15:23.414900+0000","last_unstale":"2023-10-09T09:15:53.509331+0000","last_undegraded":"2023-10-09T09:15:53.509331+0000","last_fullsized":"2023-10-09T09:15:53.509331+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":123,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.521295+0000","last_change":"2023-10-09T09:13:48.205269+0000","last_active":"2023-10-09T09:15:53.521295+0000","last_peered":"2023-10-09T09:15:53.521295+0000","last_clean":"2023-10-09T09:15:53.521295+0000","last_became_active":"2023-10-09T09:13:48.204894+0000","last_became_peered":"2023-10-09T09:13:48.204894+0000","last_unstale":"2023-10-09T09:15:53.521295+0000","last_undegraded":"2023-10-09T09:15:53.521295+0000","last_fullsized":"2023-10-09T09:15:53.521295+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":268,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.524826+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:15:54.524826+0000","last_peered":"2023-10-09T09:15:54.524826+0000","last_clean":"2023-10-09T09:15:54.524826+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:15:54.524826+0000","last_undegraded":"2023-10-09T09:15:54.524826+0000","last_fullsized":"2023-10-09T09:15:54.524826+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":196,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508271+0000","last_change":"2023-10-09T09:15:23.420333+0000","last_active":"2023-10-09T09:15:53.508271+0000","last_peered":"2023-10-09T09:15:53.508271+0000","last_clean":"2023-10-09T09:15:53.508271+0000","last_became_active":"2023-10-09T09:15:23.412581+0000","last_became_peered":"2023-10-09T09:15:23.412581+0000","last_unstale":"2023-10-09T09:15:53.508271+0000","last_undegraded":"2023-10-09T09:15:53.508271+0000","last_fullsized":"2023-10-09T09:15:53.508271+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.523842+0000","last_change":"2023-10-09T09:14:48.318889+0000","last_active":"2023-10-09T09:15:54.523842+0000","last_peered":"2023-10-09T09:15:54.523842+0000","last_clean":"2023-10-09T09:15:54.523842+0000","last_became_active":"2023-10-09T09:14:48.318575+0000","last_became_peered":"2023-10-09T09:14:48.318575+0000","last_unstale":"2023-10-09T09:15:54.523842+0000","last_undegraded":"2023-10-09T09:15:54.523842+0000","last_fullsized":"2023-10-09T09:15:54.523842+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:50.182686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"280'11049","reported_seq":11088,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.406749+0000","last_change":"2023-10-09T09:12:00.109899+0000","last_active":"2023-10-09T09:15:55.406749+0000","last_peered":"2023-10-09T09:15:55.406749+0000","last_clean":"2023-10-09T09:15:55.406749+0000","last_became_active":"2023-10-09T09:12:00.109613+0000","last_became_peered":"2023-10-09T09:12:00.109613+0000","last_unstale":"2023-10-09T09:15:55.406749+0000","last_undegraded":"2023-10-09T09:15:55.406749+0000","last_fullsized":"2023-10-09T09:15:55.406749+0000","mapping_epoch":248,"log_start":"252'1800","ondisk_log_start":"252'1800","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":9249,"log_dups_size":0,"ondisk_log_size":9249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159711232,"num_objects":2437,"num_object_clones":0,"num_object_copies":7311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11049,"num_write_kb":431552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":77,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508605+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:15:53.508605+0000","last_peered":"2023-10-09T09:15:53.508605+0000","last_clean":"2023-10-09T09:15:53.508605+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:15:53.508605+0000","last_undegraded":"2023-10-09T09:15:53.508605+0000","last_fullsized":"2023-10-09T09:15:53.508605+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":39,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508965+0000","last_change":"2023-10-09T09:15:23.414649+0000","last_active":"2023-10-09T09:15:53.508965+0000","last_peered":"2023-10-09T09:15:53.508965+0000","last_clean":"2023-10-09T09:15:53.508965+0000","last_became_active":"2023-10-09T09:15:23.414272+0000","last_became_peered":"2023-10-09T09:15:23.414272+0000","last_unstale":"2023-10-09T09:15:53.508965+0000","last_undegraded":"2023-10-09T09:15:53.508965+0000","last_fullsized":"2023-10-09T09:15:53.508965+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"280'11039","reported_seq":12222,"reported_epoch":280,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T09:15:55.461772+0000","last_change":"2023-10-09T09:15:52.750213+0000","last_active":"2023-10-09T09:15:55.461772+0000","last_peered":"2023-10-09T09:15:55.461772+0000","last_clean":"2023-10-09T09:15:22.381843+0000","last_became_active":"2023-10-09T09:15:24.550339+0000","last_became_peered":"2023-10-09T09:15:24.550339+0000","last_unstale":"2023-10-09T09:15:55.461772+0000","last_undegraded":"2023-10-09T09:15:55.461772+0000","last_fullsized":"2023-10-09T09:15:55.461772+0000","mapping_epoch":271,"log_start":"253'3469","ondisk_log_start":"253'3469","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164429824,"num_objects":2509,"num_object_clones":0,"num_object_copies":7527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1951,"num_objects_unfound":0,"num_objects_dirty":2509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11039,"num_write_kb":433536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":566,"num_bytes_recovered":37093376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":2509}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":77,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508726+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:15:53.508726+0000","last_peered":"2023-10-09T09:15:53.508726+0000","last_clean":"2023-10-09T09:15:53.508726+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:15:53.508726+0000","last_undegraded":"2023-10-09T09:15:53.508726+0000","last_fullsized":"2023-10-09T09:15:53.508726+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":169,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.521140+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:15:53.521140+0000","last_peered":"2023-10-09T09:15:53.521140+0000","last_clean":"2023-10-09T09:15:53.521140+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:15:53.521140+0000","last_undegraded":"2023-10-09T09:15:53.521140+0000","last_fullsized":"2023-10-09T09:15:53.521140+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"281'10783","reported_seq":10826,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T09:15:57.111615+0000","last_change":"2023-10-09T09:12:00.136187+0000","last_active":"2023-10-09T09:15:57.111615+0000","last_peered":"2023-10-09T09:15:57.111615+0000","last_clean":"2023-10-09T09:15:57.111615+0000","last_became_active":"2023-10-09T09:12:00.135851+0000","last_became_peered":"2023-10-09T09:12:00.135851+0000","last_unstale":"2023-10-09T09:15:57.111615+0000","last_undegraded":"2023-10-09T09:15:57.111615+0000","last_fullsized":"2023-10-09T09:15:57.111615+0000","mapping_epoch":248,"log_start":"253'3567","ondisk_log_start":"253'3567","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7216,"log_dups_size":0,"ondisk_log_size":7216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:25.402310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150274048,"num_objects":2293,"num_object_clones":0,"num_object_copies":6879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10783,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":153,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508921+0000","last_change":"2023-10-09T09:15:23.437420+0000","last_active":"2023-10-09T09:15:53.508921+0000","last_peered":"2023-10-09T09:15:53.508921+0000","last_clean":"2023-10-09T09:15:53.508921+0000","last_became_active":"2023-10-09T09:15:23.429358+0000","last_became_peered":"2023-10-09T09:15:23.429358+0000","last_unstale":"2023-10-09T09:15:53.508921+0000","last_undegraded":"2023-10-09T09:15:53.508921+0000","last_fullsized":"2023-10-09T09:15:53.508921+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":183,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508749+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:15:53.508749+0000","last_peered":"2023-10-09T09:15:53.508749+0000","last_clean":"2023-10-09T09:15:53.508749+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:15:53.508749+0000","last_undegraded":"2023-10-09T09:15:53.508749+0000","last_fullsized":"2023-10-09T09:15:53.508749+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"281'11031","reported_seq":14768,"reported_epoch":281,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T09:15:57.171488+0000","last_change":"2023-10-09T09:15:23.438134+0000","last_active":"2023-10-09T09:15:57.171488+0000","last_peered":"2023-10-09T09:15:57.171488+0000","last_clean":"2023-10-09T09:14:19.167594+0000","last_became_active":"2023-10-09T09:14:21.157751+0000","last_became_peered":"2023-10-09T09:14:21.157751+0000","last_unstale":"2023-10-09T09:15:57.171488+0000","last_undegraded":"2023-10-09T09:14:20.782662+0000","last_fullsized":"2023-10-09T09:14:20.728852+0000","mapping_epoch":261,"log_start":"253'3828","ondisk_log_start":"253'3828","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7203,"log_dups_size":0,"ondisk_log_size":7203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:23.261828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155779072,"num_objects":2377,"num_object_clones":0,"num_object_copies":7131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1268,"num_objects_misplaced":2183,"num_objects_unfound":0,"num_objects_dirty":2377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11031,"num_write_kb":429056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1842,"num_bytes_recovered":120651776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2183},{"shards":"0,3","objects":-1074}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.511750+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:15:53.511750+0000","last_peered":"2023-10-09T09:15:53.511750+0000","last_clean":"2023-10-09T09:15:53.511750+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:15:53.511750+0000","last_undegraded":"2023-10-09T09:15:53.511750+0000","last_fullsized":"2023-10-09T09:15:53.511750+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":287,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520563+0000","last_change":"2023-10-09T09:14:48.302231+0000","last_active":"2023-10-09T09:15:54.520563+0000","last_peered":"2023-10-09T09:15:54.520563+0000","last_clean":"2023-10-09T09:15:54.520563+0000","last_became_active":"2023-10-09T09:14:48.301767+0000","last_became_peered":"2023-10-09T09:14:48.301767+0000","last_unstale":"2023-10-09T09:15:54.520563+0000","last_undegraded":"2023-10-09T09:15:54.520563+0000","last_fullsized":"2023-10-09T09:15:54.520563+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:55.523804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"281'10982","reported_seq":11055,"reported_epoch":281,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:15:57.100201+0000","last_change":"2023-10-09T09:15:23.424918+0000","last_active":"2023-10-09T09:15:57.100201+0000","last_peered":"2023-10-09T09:15:57.100201+0000","last_clean":"2023-10-09T09:14:47.209721+0000","last_became_active":"2023-10-09T09:14:49.596339+0000","last_became_peered":"2023-10-09T09:14:49.596339+0000","last_unstale":"2023-10-09T09:15:57.100201+0000","last_undegraded":"2023-10-09T09:15:57.100201+0000","last_fullsized":"2023-10-09T09:15:57.100201+0000","mapping_epoch":265,"log_start":"253'3755","ondisk_log_start":"253'3755","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7227,"log_dups_size":0,"ondisk_log_size":7227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:09:16.272858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150994944,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2299,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10982,"num_write_kb":425152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":524288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":2304}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":90,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.525390+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:15:54.525390+0000","last_peered":"2023-10-09T09:15:54.525390+0000","last_clean":"2023-10-09T09:15:54.525390+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:15:54.525390+0000","last_undegraded":"2023-10-09T09:15:54.525390+0000","last_fullsized":"2023-10-09T09:15:54.525390+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":142,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.520225+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:15:53.520225+0000","last_peered":"2023-10-09T09:15:53.520225+0000","last_clean":"2023-10-09T09:15:53.520225+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:15:53.520225+0000","last_undegraded":"2023-10-09T09:15:53.520225+0000","last_fullsized":"2023-10-09T09:15:53.520225+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"280'11042","reported_seq":11855,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:15:55.387858+0000","last_change":"2023-10-09T09:15:23.440414+0000","last_active":"2023-10-09T09:15:55.387858+0000","last_peered":"2023-10-09T09:15:55.387858+0000","last_clean":"2023-10-09T09:14:19.147674+0000","last_became_active":"2023-10-09T09:14:21.157141+0000","last_became_peered":"2023-10-09T09:14:21.157141+0000","last_unstale":"2023-10-09T09:15:55.387858+0000","last_undegraded":"2023-10-09T09:14:20.802763+0000","last_fullsized":"2023-10-09T09:14:20.742865+0000","mapping_epoch":261,"log_start":"252'1800","ondisk_log_start":"252'1800","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":9242,"log_dups_size":0,"ondisk_log_size":9242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158728192,"num_objects":2422,"num_object_clones":0,"num_object_copies":7266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6352,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11042,"num_write_kb":430848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":380,"num_bytes_recovered":24838144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2422},{"shards":"3,5","objects":-2568}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":147,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508660+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:15:53.508660+0000","last_peered":"2023-10-09T09:15:53.508660+0000","last_clean":"2023-10-09T09:15:53.508660+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:15:53.508660+0000","last_undegraded":"2023-10-09T09:15:53.508660+0000","last_fullsized":"2023-10-09T09:15:53.508660+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":160,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.520241+0000","last_change":"2023-10-09T09:14:48.318228+0000","last_active":"2023-10-09T09:15:53.520241+0000","last_peered":"2023-10-09T09:15:53.520241+0000","last_clean":"2023-10-09T09:15:53.520241+0000","last_became_active":"2023-10-09T09:14:48.317743+0000","last_became_peered":"2023-10-09T09:14:48.317743+0000","last_unstale":"2023-10-09T09:15:53.520241+0000","last_undegraded":"2023-10-09T09:15:53.520241+0000","last_fullsized":"2023-10-09T09:15:53.520241+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"280'11120","reported_seq":17461,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.416750+0000","last_change":"2023-10-09T09:15:40.814591+0000","last_active":"2023-10-09T09:15:55.416750+0000","last_peered":"2023-10-09T09:15:55.416750+0000","last_clean":"2023-10-09T09:15:55.416750+0000","last_became_active":"2023-10-09T09:15:40.776768+0000","last_became_peered":"2023-10-09T09:15:40.776768+0000","last_unstale":"2023-10-09T09:15:55.416750+0000","last_undegraded":"2023-10-09T09:15:55.416750+0000","last_fullsized":"2023-10-09T09:15:55.416750+0000","mapping_epoch":275,"log_start":"253'3615","ondisk_log_start":"253'3615","created":248,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159907840,"num_objects":2440,"num_object_clones":0,"num_object_copies":7320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11120,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3138,"num_bytes_recovered":205586432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.521055+0000","last_change":"2023-10-09T09:14:48.321196+0000","last_active":"2023-10-09T09:15:53.521055+0000","last_peered":"2023-10-09T09:15:53.521055+0000","last_clean":"2023-10-09T09:15:53.521055+0000","last_became_active":"2023-10-09T09:14:48.318537+0000","last_became_peered":"2023-10-09T09:14:48.318537+0000","last_unstale":"2023-10-09T09:15:53.521055+0000","last_undegraded":"2023-10-09T09:15:53.521055+0000","last_fullsized":"2023-10-09T09:15:53.521055+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":160,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.523804+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:15:53.523804+0000","last_peered":"2023-10-09T09:15:53.523804+0000","last_clean":"2023-10-09T09:15:53.523804+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:15:53.523804+0000","last_undegraded":"2023-10-09T09:15:53.523804+0000","last_fullsized":"2023-10-09T09:15:53.523804+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"280'10953","reported_seq":10993,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.739925+0000","last_change":"2023-10-09T09:12:00.116690+0000","last_active":"2023-10-09T09:15:55.739925+0000","last_peered":"2023-10-09T09:15:55.739925+0000","last_clean":"2023-10-09T09:15:55.739925+0000","last_became_active":"2023-10-09T09:12:00.116303+0000","last_became_peered":"2023-10-09T09:12:00.116303+0000","last_unstale":"2023-10-09T09:15:55.739925+0000","last_undegraded":"2023-10-09T09:15:55.739925+0000","last_fullsized":"2023-10-09T09:15:55.739925+0000","mapping_epoch":248,"log_start":"254'4527","ondisk_log_start":"254'4527","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":6426,"log_dups_size":0,"ondisk_log_size":6426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162988032,"num_objects":2487,"num_object_clones":0,"num_object_copies":7461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10953,"num_write_kb":430080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519821+0000","last_change":"2023-10-09T09:14:48.306639+0000","last_active":"2023-10-09T09:15:54.519821+0000","last_peered":"2023-10-09T09:15:54.519821+0000","last_clean":"2023-10-09T09:15:54.519821+0000","last_became_active":"2023-10-09T09:14:48.305729+0000","last_became_peered":"2023-10-09T09:14:48.305729+0000","last_unstale":"2023-10-09T09:15:54.519821+0000","last_undegraded":"2023-10-09T09:15:54.519821+0000","last_fullsized":"2023-10-09T09:15:54.519821+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:08.892989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.523929+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:15:54.523929+0000","last_peered":"2023-10-09T09:15:54.523929+0000","last_clean":"2023-10-09T09:15:54.523929+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:15:54.523929+0000","last_undegraded":"2023-10-09T09:15:54.523929+0000","last_fullsized":"2023-10-09T09:15:54.523929+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"280'10982","reported_seq":11022,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.297913+0000","last_change":"2023-10-09T09:12:00.116375+0000","last_active":"2023-10-09T09:15:55.297913+0000","last_peered":"2023-10-09T09:15:55.297913+0000","last_clean":"2023-10-09T09:15:55.297913+0000","last_became_active":"2023-10-09T09:12:00.115851+0000","last_became_peered":"2023-10-09T09:12:00.115851+0000","last_unstale":"2023-10-09T09:15:55.297913+0000","last_undegraded":"2023-10-09T09:15:55.297913+0000","last_fullsized":"2023-10-09T09:15:55.297913+0000","mapping_epoch":248,"log_start":"252'2551","ondisk_log_start":"252'2551","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":8431,"log_dups_size":0,"ondisk_log_size":8431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157810688,"num_objects":2408,"num_object_clones":0,"num_object_copies":7224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10982,"num_write_kb":428480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520385+0000","last_change":"2023-10-09T09:14:48.299630+0000","last_active":"2023-10-09T09:15:54.520385+0000","last_peered":"2023-10-09T09:15:54.520385+0000","last_clean":"2023-10-09T09:15:54.520385+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:15:54.520385+0000","last_undegraded":"2023-10-09T09:15:54.520385+0000","last_fullsized":"2023-10-09T09:15:54.520385+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.511820+0000","last_change":"2023-10-09T09:14:19.753429+0000","last_active":"2023-10-09T09:15:53.511820+0000","last_peered":"2023-10-09T09:15:53.511820+0000","last_clean":"2023-10-09T09:15:53.511820+0000","last_became_active":"2023-10-09T09:14:19.753022+0000","last_became_peered":"2023-10-09T09:14:19.753022+0000","last_unstale":"2023-10-09T09:15:53.511820+0000","last_undegraded":"2023-10-09T09:15:53.511820+0000","last_fullsized":"2023-10-09T09:15:53.511820+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":36,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520195+0000","last_change":"2023-10-09T09:15:23.413972+0000","last_active":"2023-10-09T09:15:54.520195+0000","last_peered":"2023-10-09T09:15:54.520195+0000","last_clean":"2023-10-09T09:15:54.520195+0000","last_became_active":"2023-10-09T09:15:23.413531+0000","last_became_peered":"2023-10-09T09:15:23.413531+0000","last_unstale":"2023-10-09T09:15:54.520195+0000","last_undegraded":"2023-10-09T09:15:54.520195+0000","last_fullsized":"2023-10-09T09:15:54.520195+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:54.516595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"280'10960","reported_seq":11000,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.357266+0000","last_change":"2023-10-09T09:12:00.114449+0000","last_active":"2023-10-09T09:15:55.357266+0000","last_peered":"2023-10-09T09:15:55.357266+0000","last_clean":"2023-10-09T09:15:55.357266+0000","last_became_active":"2023-10-09T09:12:00.113992+0000","last_became_peered":"2023-10-09T09:12:00.113992+0000","last_unstale":"2023-10-09T09:15:55.357266+0000","last_undegraded":"2023-10-09T09:15:55.357266+0000","last_fullsized":"2023-10-09T09:15:55.357266+0000","mapping_epoch":248,"log_start":"253'3419","ondisk_log_start":"253'3419","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155582464,"num_objects":2374,"num_object_clones":0,"num_object_copies":7122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10960,"num_write_kb":426688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":77,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.512004+0000","last_change":"2023-10-09T09:14:48.319163+0000","last_active":"2023-10-09T09:15:53.512004+0000","last_peered":"2023-10-09T09:15:53.512004+0000","last_clean":"2023-10-09T09:15:53.512004+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:15:53.512004+0000","last_undegraded":"2023-10-09T09:15:53.512004+0000","last_fullsized":"2023-10-09T09:15:53.512004+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":142,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.556314+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:15:53.556314+0000","last_peered":"2023-10-09T09:15:53.556314+0000","last_clean":"2023-10-09T09:15:53.556314+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:15:53.556314+0000","last_undegraded":"2023-10-09T09:15:53.556314+0000","last_fullsized":"2023-10-09T09:15:53.556314+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520274+0000","last_change":"2023-10-09T09:14:48.318830+0000","last_active":"2023-10-09T09:15:54.520274+0000","last_peered":"2023-10-09T09:15:54.520274+0000","last_clean":"2023-10-09T09:15:54.520274+0000","last_became_active":"2023-10-09T09:14:48.317507+0000","last_became_peered":"2023-10-09T09:14:48.317507+0000","last_unstale":"2023-10-09T09:15:54.520274+0000","last_undegraded":"2023-10-09T09:15:54.520274+0000","last_fullsized":"2023-10-09T09:15:54.520274+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:28:14.374899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"280'10953","reported_seq":12803,"reported_epoch":280,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T09:15:55.377973+0000","last_change":"2023-10-09T09:15:24.759912+0000","last_active":"2023-10-09T09:15:55.377973+0000","last_peered":"2023-10-09T09:15:55.377973+0000","last_clean":"2023-10-09T09:14:19.067968+0000","last_became_active":"2023-10-09T09:15:24.636106+0000","last_became_peered":"2023-10-09T09:15:24.636106+0000","last_unstale":"2023-10-09T09:15:55.377973+0000","last_undegraded":"2023-10-09T09:15:24.618552+0000","last_fullsized":"2023-10-09T09:15:55.377973+0000","mapping_epoch":271,"log_start":"252'2600","ondisk_log_start":"252'2600","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":8353,"log_dups_size":0,"ondisk_log_size":8353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151715840,"num_objects":2315,"num_object_clones":0,"num_object_copies":6945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2315,"num_objects_misplaced":5733,"num_objects_unfound":0,"num_objects_dirty":2315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10953,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":901,"num_bytes_recovered":59047936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,5],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":2315}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":249,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519485+0000","last_change":"2023-10-09T09:14:48.312633+0000","last_active":"2023-10-09T09:15:54.519485+0000","last_peered":"2023-10-09T09:15:54.519485+0000","last_clean":"2023-10-09T09:15:54.519485+0000","last_became_active":"2023-10-09T09:14:48.312271+0000","last_became_peered":"2023-10-09T09:14:48.312271+0000","last_unstale":"2023-10-09T09:15:54.519485+0000","last_undegraded":"2023-10-09T09:15:54.519485+0000","last_fullsized":"2023-10-09T09:15:54.519485+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:16.219376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519465+0000","last_change":"2023-10-09T09:14:48.306136+0000","last_active":"2023-10-09T09:15:54.519465+0000","last_peered":"2023-10-09T09:15:54.519465+0000","last_clean":"2023-10-09T09:15:54.519465+0000","last_became_active":"2023-10-09T09:14:48.305165+0000","last_became_peered":"2023-10-09T09:14:48.305165+0000","last_unstale":"2023-10-09T09:15:54.519465+0000","last_undegraded":"2023-10-09T09:15:54.519465+0000","last_fullsized":"2023-10-09T09:15:54.519465+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:13.935175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.523753+0000","last_change":"2023-10-09T09:14:48.321029+0000","last_active":"2023-10-09T09:15:54.523753+0000","last_peered":"2023-10-09T09:15:54.523753+0000","last_clean":"2023-10-09T09:15:54.523753+0000","last_became_active":"2023-10-09T09:14:48.320435+0000","last_became_peered":"2023-10-09T09:14:48.320435+0000","last_unstale":"2023-10-09T09:15:54.523753+0000","last_undegraded":"2023-10-09T09:15:54.523753+0000","last_fullsized":"2023-10-09T09:15:54.523753+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:20.309472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"280'10967","reported_seq":16436,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.454119+0000","last_change":"2023-10-09T09:15:54.564593+0000","last_active":"2023-10-09T09:15:55.454119+0000","last_peered":"2023-10-09T09:15:55.454119+0000","last_clean":"2023-10-09T09:15:55.454119+0000","last_became_active":"2023-10-09T09:15:54.521024+0000","last_became_peered":"2023-10-09T09:15:54.521024+0000","last_unstale":"2023-10-09T09:15:55.454119+0000","last_undegraded":"2023-10-09T09:15:55.454119+0000","last_fullsized":"2023-10-09T09:15:55.454119+0000","mapping_epoch":279,"log_start":"253'3398","ondisk_log_start":"253'3398","created":248,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160497664,"num_objects":2449,"num_object_clones":0,"num_object_copies":7347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10967,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2701,"num_bytes_recovered":176947200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":86,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.516699+0000","last_change":"2023-10-09T09:14:19.748416+0000","last_active":"2023-10-09T09:15:53.516699+0000","last_peered":"2023-10-09T09:15:53.516699+0000","last_clean":"2023-10-09T09:15:53.516699+0000","last_became_active":"2023-10-09T09:14:19.748079+0000","last_became_peered":"2023-10-09T09:14:19.748079+0000","last_unstale":"2023-10-09T09:15:53.516699+0000","last_undegraded":"2023-10-09T09:15:53.516699+0000","last_fullsized":"2023-10-09T09:15:53.516699+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":518,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508879+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:15:53.508879+0000","last_peered":"2023-10-09T09:15:53.508879+0000","last_clean":"2023-10-09T09:15:53.508879+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:15:53.508879+0000","last_undegraded":"2023-10-09T09:15:53.508879+0000","last_fullsized":"2023-10-09T09:15:53.508879+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":107,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.512139+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:15:53.512139+0000","last_peered":"2023-10-09T09:15:53.512139+0000","last_clean":"2023-10-09T09:15:53.512139+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:15:53.512139+0000","last_undegraded":"2023-10-09T09:15:53.512139+0000","last_fullsized":"2023-10-09T09:15:53.512139+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":138,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.509090+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:15:53.509090+0000","last_peered":"2023-10-09T09:15:53.509090+0000","last_clean":"2023-10-09T09:15:53.509090+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:15:53.509090+0000","last_undegraded":"2023-10-09T09:15:53.509090+0000","last_fullsized":"2023-10-09T09:15:53.509090+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"281'11201","reported_seq":18359,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T09:15:57.137578+0000","last_change":"2023-10-09T09:15:29.480776+0000","last_active":"2023-10-09T09:15:57.137578+0000","last_peered":"2023-10-09T09:15:57.137578+0000","last_clean":"2023-10-09T09:15:57.137578+0000","last_became_active":"2023-10-09T09:15:29.446526+0000","last_became_peered":"2023-10-09T09:15:29.446526+0000","last_unstale":"2023-10-09T09:15:57.137578+0000","last_undegraded":"2023-10-09T09:15:57.137578+0000","last_fullsized":"2023-10-09T09:15:57.137578+0000","mapping_epoch":273,"log_start":"253'3986","ondisk_log_start":"253'3986","created":248,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7215,"log_dups_size":0,"ondisk_log_size":7215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:31.768973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11201,"num_write_kb":438080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":232062976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519606+0000","last_change":"2023-10-09T09:14:48.301697+0000","last_active":"2023-10-09T09:15:54.519606+0000","last_peered":"2023-10-09T09:15:54.519606+0000","last_clean":"2023-10-09T09:15:54.519606+0000","last_became_active":"2023-10-09T09:14:48.301379+0000","last_became_peered":"2023-10-09T09:14:48.301379+0000","last_unstale":"2023-10-09T09:15:54.519606+0000","last_undegraded":"2023-10-09T09:15:54.519606+0000","last_fullsized":"2023-10-09T09:15:54.519606+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:11.963433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":315,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519555+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:15:54.519555+0000","last_peered":"2023-10-09T09:15:54.519555+0000","last_clean":"2023-10-09T09:15:54.519555+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:15:54.519555+0000","last_undegraded":"2023-10-09T09:15:54.519555+0000","last_fullsized":"2023-10-09T09:15:54.519555+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":311,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.524692+0000","last_change":"2023-10-09T09:14:48.306410+0000","last_active":"2023-10-09T09:15:54.524692+0000","last_peered":"2023-10-09T09:15:54.524692+0000","last_clean":"2023-10-09T09:15:54.524692+0000","last_became_active":"2023-10-09T09:14:48.305480+0000","last_became_peered":"2023-10-09T09:14:48.305480+0000","last_unstale":"2023-10-09T09:15:54.524692+0000","last_undegraded":"2023-10-09T09:15:54.524692+0000","last_fullsized":"2023-10-09T09:15:54.524692+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:28.296221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"280'11099","reported_seq":11140,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.271243+0000","last_change":"2023-10-09T09:12:00.113109+0000","last_active":"2023-10-09T09:15:55.271243+0000","last_peered":"2023-10-09T09:15:55.271243+0000","last_clean":"2023-10-09T09:15:55.271243+0000","last_became_active":"2023-10-09T09:12:00.112495+0000","last_became_peered":"2023-10-09T09:12:00.112495+0000","last_unstale":"2023-10-09T09:15:55.271243+0000","last_undegraded":"2023-10-09T09:15:55.271243+0000","last_fullsized":"2023-10-09T09:15:55.271243+0000","mapping_epoch":248,"log_start":"253'3561","ondisk_log_start":"253'3561","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160825368,"num_objects":2455,"num_object_clones":0,"num_object_copies":7365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2455,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11099,"num_write_kb":433665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.509708+0000","last_change":"2023-10-09T09:09:44.439090+0000","last_active":"2023-10-09T09:15:53.509708+0000","last_peered":"2023-10-09T09:15:53.509708+0000","last_clean":"2023-10-09T09:15:53.509708+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:15:53.509708+0000","last_undegraded":"2023-10-09T09:15:53.509708+0000","last_fullsized":"2023-10-09T09:15:53.509708+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520122+0000","last_change":"2023-10-09T09:14:48.319210+0000","last_active":"2023-10-09T09:15:54.520122+0000","last_peered":"2023-10-09T09:15:54.520122+0000","last_clean":"2023-10-09T09:15:54.520122+0000","last_became_active":"2023-10-09T09:14:48.318880+0000","last_became_peered":"2023-10-09T09:14:48.318880+0000","last_unstale":"2023-10-09T09:15:54.520122+0000","last_undegraded":"2023-10-09T09:15:54.520122+0000","last_fullsized":"2023-10-09T09:15:54.520122+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:12.472280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":291,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.525523+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:15:54.525523+0000","last_peered":"2023-10-09T09:15:54.525523+0000","last_clean":"2023-10-09T09:15:54.525523+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:15:54.525523+0000","last_undegraded":"2023-10-09T09:15:54.525523+0000","last_fullsized":"2023-10-09T09:15:54.525523+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"280'10815","reported_seq":19722,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.732028+0000","last_change":"2023-10-09T09:15:08.403520+0000","last_active":"2023-10-09T09:15:55.732028+0000","last_peered":"2023-10-09T09:15:55.732028+0000","last_clean":"2023-10-09T09:15:55.732028+0000","last_became_active":"2023-10-09T09:15:08.367545+0000","last_became_peered":"2023-10-09T09:15:08.367545+0000","last_unstale":"2023-10-09T09:15:55.732028+0000","last_undegraded":"2023-10-09T09:15:55.732028+0000","last_fullsized":"2023-10-09T09:15:55.732028+0000","mapping_epoch":268,"log_start":"254'4413","ondisk_log_start":"254'4413","created":248,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":6402,"log_dups_size":0,"ondisk_log_size":6402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157351936,"num_objects":2401,"num_object_clones":0,"num_object_copies":7203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10815,"num_write_kb":422912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4420,"num_bytes_recovered":289669120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":47,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.525530+0000","last_change":"2023-10-09T09:14:19.749169+0000","last_active":"2023-10-09T09:15:54.525530+0000","last_peered":"2023-10-09T09:15:54.525530+0000","last_clean":"2023-10-09T09:15:54.525530+0000","last_became_active":"2023-10-09T09:14:19.740888+0000","last_became_peered":"2023-10-09T09:14:19.740888+0000","last_unstale":"2023-10-09T09:15:54.525530+0000","last_undegraded":"2023-10-09T09:15:54.525530+0000","last_fullsized":"2023-10-09T09:15:54.525530+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:01.361061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.523669+0000","last_change":"2023-10-09T09:14:48.317228+0000","last_active":"2023-10-09T09:15:54.523669+0000","last_peered":"2023-10-09T09:15:54.523669+0000","last_clean":"2023-10-09T09:15:54.523669+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:15:54.523669+0000","last_undegraded":"2023-10-09T09:15:54.523669+0000","last_fullsized":"2023-10-09T09:15:54.523669+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.508583+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:15:53.508583+0000","last_peered":"2023-10-09T09:15:53.508583+0000","last_clean":"2023-10-09T09:15:53.508583+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:15:53.508583+0000","last_undegraded":"2023-10-09T09:15:53.508583+0000","last_fullsized":"2023-10-09T09:15:53.508583+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"280'10884","reported_seq":16925,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:55.439187+0000","last_change":"2023-10-09T09:15:48.533323+0000","last_active":"2023-10-09T09:15:55.439187+0000","last_peered":"2023-10-09T09:15:55.439187+0000","last_clean":"2023-10-09T09:15:55.439187+0000","last_became_active":"2023-10-09T09:15:48.501622+0000","last_became_peered":"2023-10-09T09:15:48.501622+0000","last_unstale":"2023-10-09T09:15:55.439187+0000","last_undegraded":"2023-10-09T09:15:55.439187+0000","last_fullsized":"2023-10-09T09:15:55.439187+0000","mapping_epoch":277,"log_start":"253'3383","ondisk_log_start":"253'3383","created":248,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158728192,"num_objects":2422,"num_object_clones":0,"num_object_copies":7266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10884,"num_write_kb":425792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2981,"num_bytes_recovered":195297280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":147,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.509079+0000","last_change":"2023-10-09T09:14:48.319849+0000","last_active":"2023-10-09T09:15:53.509079+0000","last_peered":"2023-10-09T09:15:53.509079+0000","last_clean":"2023-10-09T09:15:53.509079+0000","last_became_active":"2023-10-09T09:14:48.319372+0000","last_became_peered":"2023-10-09T09:14:48.319372+0000","last_unstale":"2023-10-09T09:15:53.509079+0000","last_undegraded":"2023-10-09T09:15:53.509079+0000","last_fullsized":"2023-10-09T09:15:53.509079+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":333,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.524717+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:15:54.524717+0000","last_peered":"2023-10-09T09:15:54.524717+0000","last_clean":"2023-10-09T09:15:54.524717+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:15:54.524717+0000","last_undegraded":"2023-10-09T09:15:54.524717+0000","last_fullsized":"2023-10-09T09:15:54.524717+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":160,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.521220+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:15:53.521220+0000","last_peered":"2023-10-09T09:15:53.521220+0000","last_clean":"2023-10-09T09:15:53.521220+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:15:53.521220+0000","last_undegraded":"2023-10-09T09:15:53.521220+0000","last_fullsized":"2023-10-09T09:15:53.521220+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":302,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.527688+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:15:54.527688+0000","last_peered":"2023-10-09T09:15:54.527688+0000","last_clean":"2023-10-09T09:15:54.527688+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:15:54.527688+0000","last_undegraded":"2023-10-09T09:15:54.527688+0000","last_fullsized":"2023-10-09T09:15:54.527688+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":27,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.520121+0000","last_change":"2023-10-09T09:14:48.318983+0000","last_active":"2023-10-09T09:15:54.520121+0000","last_peered":"2023-10-09T09:15:54.520121+0000","last_clean":"2023-10-09T09:15:54.520121+0000","last_became_active":"2023-10-09T09:14:48.317990+0000","last_became_peered":"2023-10-09T09:14:48.317990+0000","last_unstale":"2023-10-09T09:15:54.520121+0000","last_undegraded":"2023-10-09T09:15:54.520121+0000","last_fullsized":"2023-10-09T09:15:54.520121+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:43.629682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":85,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.511656+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:15:53.511656+0000","last_peered":"2023-10-09T09:15:53.511656+0000","last_clean":"2023-10-09T09:15:53.511656+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:15:53.511656+0000","last_undegraded":"2023-10-09T09:15:53.511656+0000","last_fullsized":"2023-10-09T09:15:53.511656+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":160,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.556190+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:15:53.556190+0000","last_peered":"2023-10-09T09:15:53.556190+0000","last_clean":"2023-10-09T09:15:53.556190+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:15:53.556190+0000","last_undegraded":"2023-10-09T09:15:53.556190+0000","last_fullsized":"2023-10-09T09:15:53.556190+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":72,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.511655+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:15:53.511655+0000","last_peered":"2023-10-09T09:15:53.511655+0000","last_clean":"2023-10-09T09:15:53.511655+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:15:53.511655+0000","last_undegraded":"2023-10-09T09:15:53.511655+0000","last_fullsized":"2023-10-09T09:15:53.511655+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":142,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.518977+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:15:53.518977+0000","last_peered":"2023-10-09T09:15:53.518977+0000","last_clean":"2023-10-09T09:15:53.518977+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:15:53.518977+0000","last_undegraded":"2023-10-09T09:15:53.518977+0000","last_fullsized":"2023-10-09T09:15:53.518977+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":234,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.509369+0000","last_change":"2023-10-09T09:14:48.312443+0000","last_active":"2023-10-09T09:15:53.509369+0000","last_peered":"2023-10-09T09:15:53.509369+0000","last_clean":"2023-10-09T09:15:53.509369+0000","last_became_active":"2023-10-09T09:14:48.312053+0000","last_became_peered":"2023-10-09T09:14:48.312053+0000","last_unstale":"2023-10-09T09:15:53.509369+0000","last_undegraded":"2023-10-09T09:15:53.509369+0000","last_fullsized":"2023-10-09T09:15:53.509369+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":223,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.519456+0000","last_change":"2023-10-09T09:14:19.744537+0000","last_active":"2023-10-09T09:15:54.519456+0000","last_peered":"2023-10-09T09:15:54.519456+0000","last_clean":"2023-10-09T09:15:54.519456+0000","last_became_active":"2023-10-09T09:14:19.744078+0000","last_became_peered":"2023-10-09T09:14:19.744078+0000","last_unstale":"2023-10-09T09:15:54.519456+0000","last_undegraded":"2023-10-09T09:15:54.519456+0000","last_fullsized":"2023-10-09T09:15:54.519456+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:49.733990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":376,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T09:15:54.525479+0000","last_change":"2023-10-09T09:14:19.780202+0000","last_active":"2023-10-09T09:15:54.525479+0000","last_peered":"2023-10-09T09:15:54.525479+0000","last_clean":"2023-10-09T09:15:54.525479+0000","last_became_active":"2023-10-09T09:14:19.758344+0000","last_became_peered":"2023-10-09T09:14:19.758344+0000","last_unstale":"2023-10-09T09:15:54.525479+0000","last_undegraded":"2023-10-09T09:15:54.525479+0000","last_fullsized":"2023-10-09T09:15:54.525479+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:36.052871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":169,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T09:15:53.519003+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:15:53.519003+0000","last_peered":"2023-10-09T09:15:53.519003+0000","last_clean":"2023-10-09T09:15:53.519003+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:15:53.519003+0000","last_undegraded":"2023-10-09T09:15:53.519003+0000","last_fullsized":"2023-10-09T09:15:53.519003+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2528444440,"num_objects":38582,"num_object_clones":0,"num_object_copies":115746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9935,"num_objects_misplaced":12166,"num_objects_unfound":0,"num_objects_dirty":38582,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175860,"num_write_kb":6862081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20479,"num_bytes_recovered":1341718528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8993910784,"data_stored":8993898568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122188,"ondisk_log_size":122188,"up":48,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":19206,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430745,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1130572,"kb_used_data":829252,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92614580,"statfs":{"total":95995035648,"available":94837329920,"internally_reserved":0,"allocated":849154048,"data_stored":848575231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,4,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:15:29 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":2.222,"15min":2.6320000000000001},"min":{"1min":0.32400000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.7990000000000004,"5min":47.776000000000003,"15min":119.498},"last":2.4510000000000001},{"interface":"front","average":{"1min":1.6619999999999999,"5min":2.0209999999999999,"15min":2.5600000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.26400000000000001},"max":{"1min":5.5430000000000001,"5min":47.689,"15min":119.532},"last":0.46800000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:15:12 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.425,"15min":1.0940000000000001},"min":{"1min":0.40999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.2069999999999999,"5min":12.682,"15min":12.682},"last":2.524},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0640000000000001,"15min":0.93000000000000005},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.5960000000000001,"5min":7.875,"15min":7.875},"last":0.442}]},{"osd":2,"last update":"Mon Oct 9 09:15:29 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.994,"15min":2.2679999999999998},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":5.6879999999999997,"5min":45.345999999999997,"15min":126.072},"last":1.615},{"interface":"front","average":{"1min":0.94299999999999995,"5min":2.1509999999999998,"15min":2.3530000000000002},"min":{"1min":0.42399999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.129,"5min":46.228000000000002,"15min":120.729},"last":2.5019999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:15:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":2.5339999999999998,"15min":3.4420000000000002},"min":{"1min":0.46200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":5.0330000000000004,"5min":62.060000000000002,"15min":191.30799999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":2.899,"15min":3.5419999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.318},"max":{"1min":2.3479999999999999,"5min":63.268999999999998,"15min":191.47900000000001},"last":2.5409999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:14:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.8680000000000001,"15min":1.7350000000000001},"min":{"1min":0.41299999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.8339999999999996,"5min":13.895,"15min":138.34299999999999},"last":1.54},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.952,"15min":1.762},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":8.9879999999999995,"5min":16.433,"15min":137.04900000000001},"last":2.556}]},{"osd":5,"last update":"Mon Oct 9 09:14:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.6040000000000001,"15min":1.091},"min":{"1min":0.34599999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.1720000000000006,"5min":9.468,"15min":9.468},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.0899999999999999,"15min":1.2769999999999999},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.7859999999999996,"5min":13.82,"15min":13.82},"last":2.6589999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:14:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":2.0779999999999998,"15min":1.3560000000000001},"min":{"1min":0.39300000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7480000000000002,"5min":21.145,"15min":21.145},"last":1.577},{"interface":"front","average":{"1min":1.2989999999999999,"5min":2.1469999999999998,"15min":1.431},"min":{"1min":0.53600000000000003,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":2.8039999999999998,"5min":19.068999999999999,"15min":19.068999999999999},"last":2.4820000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626978,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2013460,"kb_used_data":1105996,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91731692,"statfs":{"total":95995035648,"available":93933252608,"internally_reserved":0,"allocated":1132539904,"data_stored":1131961220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,1,0,2,12],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":4.2880000000000003,"15min":2.581},"min":{"1min":0.33500000000000002,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":10.683999999999999,"5min":92.138999999999996,"15min":92.138999999999996},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.4620000000000002,"5min":4.3570000000000002,"15min":2.6509999999999998},"min":{"1min":0.48999999999999999,"5min":0.317,"15min":0.24099999999999999},"max":{"1min":18.045000000000002,"5min":91.709999999999994,"15min":91.709999999999994},"last":0.63400000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.6220000000000001,"15min":1.5760000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.314},"max":{"1min":6.7460000000000004,"5min":12.369,"15min":109.453},"last":0.621},{"interface":"front","average":{"1min":1.371,"5min":1.3600000000000001,"15min":1.534},"min":{"1min":0.372,"5min":0.34999999999999998,"15min":0.314},"max":{"1min":6.1470000000000002,"5min":12.385,"15min":111.47499999999999},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:15:42 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.712,"15min":1.544},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.590999999999999,"5min":12.234,"15min":18.756},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.714,"5min":1.883,"15min":1.609},"min":{"1min":0.34399999999999997,"5min":0.33700000000000002,"15min":0.30199999999999999},"max":{"1min":18.170000000000002,"5min":18.170000000000002,"15min":18.841999999999999},"last":0.68100000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.774,"15min":1.55},"min":{"1min":0.34300000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.8109999999999999,"5min":12.35,"15min":16.977},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.454,"5min":1.3879999999999999,"15min":1.643},"min":{"1min":0.36699999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4790000000000001,"5min":12.628,"15min":20.573},"last":0.59999999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":1.7470000000000001,"15min":1.4370000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":6.9039999999999999,"5min":12.606,"15min":32.280999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.48,"5min":1.8420000000000001,"15min":1.419},"min":{"1min":0.41899999999999998,"5min":0.32500000000000001,"15min":0.317},"max":{"1min":11.170999999999999,"5min":16.648,"15min":34.689},"last":0.439}]},{"osd":5,"last update":"Mon Oct 9 09:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.978,"15min":1.4690000000000001},"min":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.30199999999999999},"max":{"1min":6.1040000000000001,"5min":12.359,"15min":12.359},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.3570000000000002,"5min":1.9119999999999999,"15min":1.371},"min":{"1min":0.372,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.452999999999999,"5min":13.609,"15min":13.609},"last":0.50700000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":4.4429999999999996,"5min":2.5,"15min":1.677},"min":{"1min":0.58999999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":29.530000000000001,"5min":29.530000000000001,"15min":29.530000000000001},"last":0.90600000000000003},{"interface":"front","average":{"1min":2.7559999999999998,"5min":1.8280000000000001,"15min":1.389},"min":{"1min":0.40500000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.1669999999999998,"5min":12.667999999999999,"15min":13.523999999999999},"last":0.66700000000000004}]}]},{"osd":1,"up_from":83,"seq":356482285926,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":749828,"kb_used_data":314748,"kb_used_omap":9,"kb_used_meta":435062,"kb_avail":92995324,"statfs":{"total":95995035648,"available":95227211776,"internally_reserved":0,"allocated":322301952,"data_stored":321271976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":445504489},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.0509999999999999,"15min":1.968},"min":{"1min":0.25,"5min":0.223,"15min":0.189},"max":{"1min":15.298,"5min":15.298,"15min":94.915999999999997},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.694,"5min":1.4890000000000001,"15min":2.157},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.21199999999999999},"max":{"1min":19.899000000000001,"5min":19.899000000000001,"15min":98.227999999999994},"last":0.31900000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:15:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.80100000000000005,"15min":1.595},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.23799999999999999},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":46.354999999999997},"last":0.372},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.65500000000000003,"15min":1.427},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.252},"max":{"1min":1.48,"5min":3.0499999999999998,"15min":46.241999999999997},"last":0.35399999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:15:13 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.84199999999999997,"15min":1.954},"min":{"1min":0.42299999999999999,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":3.2989999999999999,"5min":3.2989999999999999,"15min":67.807000000000002},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.83799999999999997,"15min":1.8859999999999999},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.22600000000000001},"max":{"1min":3.319,"5min":3.9750000000000001,"15min":68.097999999999999},"last":0.65000000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.3999999999999999,"15min":1.1919999999999999},"min":{"1min":0.42799999999999999,"5min":0.376,"15min":0.30099999999999999},"max":{"1min":7.2140000000000004,"5min":7.5780000000000003,"15min":8.7080000000000002},"last":0.627},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.2310000000000001,"15min":1.113},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.33100000000000002},"max":{"1min":4.6139999999999999,"5min":6.593,"15min":8.1750000000000007},"last":2.9860000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.369,"15min":1.1060000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33800000000000002},"max":{"1min":6.1929999999999996,"5min":7.6600000000000001,"15min":11.446999999999999},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.222,"15min":1.0920000000000001},"min":{"1min":0.46400000000000002,"5min":0.41699999999999998,"15min":0.35299999999999998},"max":{"1min":6.2080000000000002,"5min":7.6399999999999997,"15min":8.6639999999999997},"last":1.089}]},{"osd":6,"last update":"Mon Oct 9 09:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.6040000000000001,"15min":1.359},"min":{"1min":0.42299999999999999,"5min":0.39700000000000002,"15min":0.33200000000000002},"max":{"1min":4.0099999999999998,"5min":7.9480000000000004,"15min":7.9480000000000004},"last":3.7200000000000002},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.7649999999999999,"15min":1.3480000000000001},"min":{"1min":0.42499999999999999,"5min":0.40400000000000003,"15min":0.39000000000000001},"max":{"1min":8.9079999999999995,"5min":8.9079999999999995,"15min":11.337},"last":3.7040000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.5449999999999999,"15min":1.1950000000000001},"min":{"1min":0.42199999999999999,"5min":0.41799999999999998,"15min":0.38500000000000001},"max":{"1min":9.2959999999999994,"5min":10.326000000000001,"15min":10.326000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.863,"5min":1.544,"15min":1.2629999999999999},"min":{"1min":0.505,"5min":0.46800000000000003,"15min":0.38900000000000001},"max":{"1min":5.8239999999999998,"5min":7.6379999999999999,"15min":7.6379999999999999},"last":2.081}]}]},{"osd":0,"up_from":78,"seq":335007449452,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2399508,"kb_used_data":1541388,"kb_used_omap":28,"kb_used_meta":858083,"kb_avail":91345644,"statfs":{"total":95995035648,"available":93537939456,"internally_reserved":0,"allocated":1578381312,"data_stored":1577336468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":878677840},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:15:17 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.0629999999999999,"15min":1.1519999999999999},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.193},"max":{"1min":2.6739999999999999,"5min":9.7620000000000005,"15min":9.7620000000000005},"last":0.313},{"interface":"front","average":{"1min":1.5,"5min":1.149,"15min":1.1339999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.21199999999999999},"max":{"1min":6.6040000000000001,"5min":9.2609999999999992,"15min":18.661000000000001},"last":0.503}]},{"osd":2,"last update":"Mon Oct 9 09:14:54 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":2.2029999999999998,"15min":2.7549999999999999},"min":{"1min":0.27200000000000002,"5min":0.25900000000000001,"15min":0.23400000000000001},"max":{"1min":5.4569999999999999,"5min":74.200999999999993,"15min":74.200999999999993},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.585,"5min":2.2650000000000001,"15min":3.6269999999999998},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.24399999999999999},"max":{"1min":5.3639999999999999,"5min":73.983999999999995,"15min":219.00700000000001},"last":0.625}]},{"osd":3,"last update":"Mon Oct 9 09:15:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.726,"15min":2.798},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.8239999999999998,"5min":20.364000000000001,"15min":219.041},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.794,"5min":1.597,"15min":2.2839999999999998},"min":{"1min":0.38800000000000001,"5min":0.36499999999999999,"15min":0.25},"max":{"1min":5.407,"5min":18.895,"15min":137.94999999999999},"last":0.59099999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:14:54 2023","interfaces":[{"interface":"back","average":{"1min":3.9260000000000002,"5min":2,"15min":1.589},"min":{"1min":0.52500000000000002,"5min":0.34000000000000002,"15min":0.30199999999999999},"max":{"1min":14.163,"5min":14.163,"15min":14.163},"last":2.2690000000000001},{"interface":"front","average":{"1min":2.766,"5min":2.0800000000000001,"15min":1.778},"min":{"1min":0.36799999999999999,"5min":0.33000000000000002,"15min":0.29099999999999998},"max":{"1min":10.786,"5min":10.786,"15min":10.786},"last":1.349}]},{"osd":5,"last update":"Mon Oct 9 09:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.7589999999999999,"15min":1.6020000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.867,"5min":8.9079999999999995,"15min":9.3030000000000008},"last":1.915},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.512,"15min":1.536},"min":{"1min":0.40699999999999997,"5min":0.38400000000000001,"15min":0.33700000000000002},"max":{"1min":5.4379999999999997,"5min":9.173,"15min":16.364000000000001},"last":1.3109999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":1.75,"15min":2.2749999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":9.6699999999999999,"5min":9.6699999999999999,"15min":98.864000000000004},"last":2.25},{"interface":"front","average":{"1min":2.306,"5min":1.867,"15min":2.343},"min":{"1min":0.52200000000000002,"5min":0.44600000000000001,"15min":0.36699999999999999},"max":{"1min":5.4020000000000001,"5min":9.2070000000000007,"15min":98.878},"last":2.218}]},{"osd":7,"last update":"Mon Oct 9 09:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":1.6719999999999999,"15min":1.546},"min":{"1min":0.58999999999999997,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":8.2010000000000005,"5min":9.2349999999999994,"15min":10.157999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":3.206,"5min":2.0350000000000001,"15min":1.8109999999999999},"min":{"1min":0.41199999999999998,"5min":0.40799999999999997,"15min":0.40100000000000002},"max":{"1min":10.734,"5min":10.734,"15min":14.359},"last":0.54800000000000004}]}]},{"osd":2,"up_from":0,"seq":377957122400,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":93,"seq":399431958874,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761996,"kb_used_data":923844,"kb_used_omap":7,"kb_used_meta":838072,"kb_avail":91983156,"statfs":{"total":95995035648,"available":94190751744,"internally_reserved":0,"allocated":946016256,"data_stored":944977357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":858186344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:15:06 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.4689999999999999,"15min":2.6259999999999999},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":39.667000000000002,"5min":130.45500000000001,"15min":233.22900000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":3.0219999999999998,"5min":2.7000000000000002,"15min":3.004},"min":{"1min":0.222,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":39.808999999999997,"5min":129.74799999999999,"15min":237.447},"last":0.39800000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:15:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.94699999999999995,"15min":2.3210000000000002},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":2.4500000000000002,"5min":10.662000000000001,"15min":326.75700000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.76100000000000001,"15min":2.1880000000000002},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.22900000000000001},"max":{"1min":2.4969999999999999,"5min":3.1219999999999999,"15min":327.024},"last":0.35199999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:15:06 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":2.2589999999999999,"15min":2.238},"min":{"1min":0.39200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.6880000000000002,"5min":129.67699999999999,"15min":129.67699999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.181,"5min":2.1080000000000001,"15min":2.242},"min":{"1min":0.29999999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.0490000000000004,"5min":130.40000000000001,"15min":130.40000000000001},"last":0.309}]},{"osd":4,"last update":"Mon Oct 9 09:15:06 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.2330000000000001,"15min":1.2769999999999999},"min":{"1min":0.45000000000000001,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":6.6790000000000003,"5min":6.6790000000000003,"15min":8.9879999999999995},"last":4.4109999999999996},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.625,"15min":1.7170000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.313},"max":{"1min":5.8819999999999997,"5min":7.2649999999999997,"15min":10.505000000000001},"last":12.617000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:15:06 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.4990000000000001,"15min":1.71},"min":{"1min":0.41999999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":4.7140000000000004,"5min":7.819,"15min":16.132999999999999},"last":0.502},{"interface":"front","average":{"1min":2.359,"5min":1.992,"15min":1.643},"min":{"1min":0.42399999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.3070000000000004,"5min":14.361000000000001,"15min":14.361000000000001},"last":0.38700000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:15:06 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.7689999999999999,"15min":1.8919999999999999},"min":{"1min":0.48099999999999998,"5min":0.44,"15min":0.308},"max":{"1min":5.9669999999999996,"5min":9.1159999999999997,"15min":12.105},"last":2.5470000000000002},{"interface":"front","average":{"1min":1.629,"5min":1.5800000000000001,"15min":1.702},"min":{"1min":0.56499999999999995,"5min":0.41099999999999998,"15min":0.38500000000000001},"max":{"1min":5.9329999999999998,"5min":9.0199999999999996,"15min":9.5090000000000003},"last":2.5670000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:15:06 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.488,"15min":1.5609999999999999},"min":{"1min":0.49099999999999999,"5min":0.44600000000000001,"15min":0.41499999999999998},"max":{"1min":4.2839999999999998,"5min":5.7809999999999997,"15min":7.5899999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.508,"5min":1.3899999999999999,"15min":1.4119999999999999},"min":{"1min":0.49399999999999999,"5min":0.436,"15min":0.38300000000000001},"max":{"1min":6.0890000000000004,"5min":6.0890000000000004,"15min":9.1560000000000006},"last":0.441}]}]},{"osd":4,"up_from":30,"seq":128849019444,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2483796,"kb_used_data":1589708,"kb_used_omap":0,"kb_used_meta":894080,"kb_avail":91261356,"statfs":{"total":95995035648,"available":93451628544,"internally_reserved":0,"allocated":1627860992,"data_stored":1627281567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915537920},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,2,1,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:15:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":2.9670000000000001,"15min":2.605},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":6.5570000000000004,"5min":116.59999999999999,"15min":141.07400000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.8720000000000001,"5min":2.645,"15min":2.6150000000000002},"min":{"1min":0.315,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":7.952,"5min":116.843,"15min":141.15600000000001},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.679,"5min":1.446,"15min":1.022},"min":{"1min":0.33200000000000002,"5min":0.317,"15min":0.29899999999999999},"max":{"1min":7.8529999999999998,"5min":13.853999999999999,"15min":13.853999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.714,"5min":1.3999999999999999,"15min":1.1259999999999999},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":11.391999999999999},"last":0.41299999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:15:38 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.778,"15min":2.3610000000000002},"min":{"1min":0.36099999999999999,"5min":0.32800000000000001,"15min":0.29299999999999998},"max":{"1min":6.7030000000000003,"5min":116.7,"15min":142.816},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.6830000000000001,"5min":2.8090000000000002,"15min":2.3849999999999998},"min":{"1min":0.34699999999999998,"5min":0.316,"15min":0.315},"max":{"1min":7.9059999999999997,"5min":116.63800000000001,"15min":143.81800000000001},"last":0.67700000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":2.4609999999999999,"15min":4.1989999999999998},"min":{"1min":0.38700000000000001,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":7.891,"5min":82.219999999999999,"15min":165.995},"last":0.378},{"interface":"front","average":{"1min":3.343,"5min":2.5230000000000001,"15min":4.5890000000000004},"min":{"1min":0.39400000000000002,"5min":0.35899999999999999,"15min":0.314},"max":{"1min":12.207000000000001,"5min":81.757999999999996,"15min":165.53200000000001},"last":0.64600000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:15:22 2023","interfaces":[{"interface":"back","average":{"1min":3.6339999999999999,"5min":2.2509999999999999,"15min":1.429},"min":{"1min":0.52700000000000002,"5min":0.30299999999999999,"15min":0.30099999999999999},"max":{"1min":13.907,"5min":13.907,"15min":13.907},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.7829999999999999,"15min":1.2390000000000001},"min":{"1min":0.376,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":7.9900000000000002,"5min":11.042999999999999,"15min":11.042999999999999},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:15:04 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.0179999999999998,"15min":1.552},"min":{"1min":0.44900000000000001,"5min":0.313,"15min":0.313},"max":{"1min":9.3659999999999997,"5min":13.804,"15min":74.700999999999993},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.843,"5min":1.9159999999999999,"15min":1.5760000000000001},"min":{"1min":0.437,"5min":0.35999999999999999,"15min":0.33000000000000002},"max":{"1min":11.448,"5min":13.444000000000001,"15min":73.572999999999993},"last":5.9009999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:15:38 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":2.105,"15min":1.3859999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.32800000000000001},"max":{"1min":8.2650000000000006,"5min":11.169,"15min":11.169},"last":6.782},{"interface":"front","average":{"1min":2.673,"5min":2.0990000000000002,"15min":1.4239999999999999},"min":{"1min":0.376,"5min":0.37,"15min":0.36199999999999999},"max":{"1min":11.064,"5min":13.916,"15min":13.916},"last":1.8660000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823212,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2000448,"kb_used_data":1099128,"kb_used_omap":0,"kb_used_meta":901312,"kb_avail":91744704,"statfs":{"total":95995035648,"available":93946576896,"internally_reserved":0,"allocated":1125507072,"data_stored":1124943847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922943488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":3.6139999999999999,"15min":3.1840000000000002},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":11.131,"5min":128.96199999999999,"15min":156.97800000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":3,"5min":3.4550000000000001,"15min":3.2280000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":12.127000000000001,"5min":128.93700000000001,"15min":157.45699999999999},"last":0.60099999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4889999999999999,"5min":1.8169999999999999,"15min":1.349},"min":{"1min":0.312,"5min":0.312,"15min":0.307},"max":{"1min":11.105,"5min":11.105,"15min":36.963999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.3940000000000001,"5min":1.677,"15min":1.3280000000000001},"min":{"1min":0.36299999999999999,"5min":0.314,"15min":0.313},"max":{"1min":10.959,"5min":10.959,"15min":36.158000000000001},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":3.407,"15min":2.919},"min":{"1min":0.37,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.058,"5min":128.77199999999999,"15min":156.69499999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.496,"5min":3.3069999999999999,"15min":2.7770000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.994,"5min":128.72999999999999,"15min":156.87799999999999},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 09:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":1.734,"15min":3.0539999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.077999999999999,"5min":40.857999999999997,"15min":161.678},"last":0.56999999999999995},{"interface":"front","average":{"1min":3.3439999999999999,"5min":2.2229999999999999,"15min":3.3300000000000001},"min":{"1min":0.41699999999999998,"5min":0.31,"15min":0.31},"max":{"1min":11.019,"5min":40.621000000000002,"15min":159.43899999999999},"last":0.628}]},{"osd":4,"last update":"Mon Oct 9 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":2.577,"15min":1.5920000000000001},"min":{"1min":0.31,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.4889999999999999,"5min":13.904,"15min":13.904},"last":0.93700000000000006},{"interface":"front","average":{"1min":3.8820000000000001,"5min":2.1120000000000001,"15min":1.397},"min":{"1min":0.504,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":22.228000000000002,"5min":22.228000000000002,"15min":22.228000000000002},"last":0.68799999999999994}]},{"osd":6,"last update":"Mon Oct 9 09:14:58 2023","interfaces":[{"interface":"back","average":{"1min":3.0659999999999998,"5min":2.121,"15min":1.508},"min":{"1min":0.36499999999999999,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":11.125,"5min":11.125,"15min":16.052},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.641,"5min":2.024,"15min":1.452},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.183,"5min":11.863,"15min":11.863},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.911,"15min":1.2589999999999999},"min":{"1min":0.36099999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":11.210000000000001,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":3.0819999999999999,"5min":2.0760000000000001,"15min":1.363},"min":{"1min":0.41899999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":11.090999999999999,"5min":11.090999999999999,"15min":11.090999999999999},"last":0.495}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1556156416,"data_stored":1556152344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":315097088,"data_stored":315097088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1497169920,"data_stored":1497169920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":934739968,"data_stored":934739968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1619202048,"data_stored":1619197976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1114308608,"data_stored":1114308608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1117327360,"data_stored":1117323288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":839909376,"data_stored":839909376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:16:05.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-09T09:16:05.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[75179]: cluster 2023-10-09T09:16:04.130947+0000 mgr.y (mgr.25421) 1164 : cluster 0 pgmap v1267: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 9759/107997 objects degraded (9.036%); 11119/107997 objects misplaced (10.296%); 14 MiB/s, 220 objects/s recovering 2023-10-09T09:16:05.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[75179]: audit 2023-10-09T09:16:04.257409+0000 mon.a (mon.0) 1559 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[75179]: audit 2023-10-09T09:16:04.258602+0000 mon.a (mon.0) 1560 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[75179]: cluster 2023-10-09T09:16:04.755013+0000 mon.a (mon.0) 1561 : cluster 3 Health check update: Degraded data redundancy: 9759/107997 objects degraded (9.036%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[77892]: cluster 2023-10-09T09:16:04.130947+0000 mgr.y (mgr.25421) 1164 : cluster 0 pgmap v1267: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 9759/107997 objects degraded (9.036%); 11119/107997 objects misplaced (10.296%); 14 MiB/s, 220 objects/s recovering 2023-10-09T09:16:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[77892]: audit 2023-10-09T09:16:04.257409+0000 mon.a (mon.0) 1559 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[77892]: audit 2023-10-09T09:16:04.258602+0000 mon.a (mon.0) 1560 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[77892]: cluster 2023-10-09T09:16:04.755013+0000 mon.a (mon.0) 1561 : cluster 3 Health check update: Degraded data redundancy: 9759/107997 objects degraded (9.036%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:05.300 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:05.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:05.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:05.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:05.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:05 smithi094 bash[59677]: cluster 2023-10-09T09:16:04.130947+0000 mgr.y (mgr.25421) 1164 : cluster 0 pgmap v1267: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 9759/107997 objects degraded (9.036%); 11119/107997 objects misplaced (10.296%); 14 MiB/s, 220 objects/s recovering 2023-10-09T09:16:05.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:05 smithi094 bash[59677]: audit 2023-10-09T09:16:04.257409+0000 mon.a (mon.0) 1559 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:05.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:05 smithi094 bash[59677]: audit 2023-10-09T09:16:04.258602+0000 mon.a (mon.0) 1560 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:05.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:05 smithi094 bash[59677]: cluster 2023-10-09T09:16:04.755013+0000 mon.a (mon.0) 1561 : cluster 3 Health check update: Degraded data redundancy: 9759/107997 objects degraded (9.036%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:05.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:05.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:05.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:05.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:05.837 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:05.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:05.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.952 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:16:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:16:05] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-09T09:16:05.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:05.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:05.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:06.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:06.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:06.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:06.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:06.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:06.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:06.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:06.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.513 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:06.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:06.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:06.656 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:06.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:06.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:06.761 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:06.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:06.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:06.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:06.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:06.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:07.027 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:07.062 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:07.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:07.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:07.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:07.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:07.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:07.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:07.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:07.306 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:07 smithi094 bash[59677]: cluster 2023-10-09T09:16:06.131525+0000 mgr.y (mgr.25421) 1165 : cluster 0 pgmap v1268: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9604/102096 objects degraded (9.407%); 10478/102096 objects misplaced (10.263%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T09:16:07.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:07.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:07.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:07.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:07 smithi016 bash[75179]: cluster 2023-10-09T09:16:06.131525+0000 mgr.y (mgr.25421) 1165 : cluster 0 pgmap v1268: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9604/102096 objects degraded (9.407%); 10478/102096 objects misplaced (10.263%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T09:16:07.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:07 smithi016 bash[77892]: cluster 2023-10-09T09:16:06.131525+0000 mgr.y (mgr.25421) 1165 : cluster 0 pgmap v1268: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9604/102096 objects degraded (9.407%); 10478/102096 objects misplaced (10.263%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T09:16:07.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:07.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:07.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:07.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:07.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:07.703 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:07.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:07.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:07.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:07.838 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:07.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:07.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:07.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:08.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:08.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:08.115 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:08.118 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:08.118 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:08.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:08.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:08.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:08.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:08.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:08.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:08.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:08.526 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:08.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:08.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:08.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:08.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:08.766 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:08.799 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:08.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:08.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:08.905 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:08.936 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:08.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:08.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:09.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:09.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:09.171 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:09.194 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:09.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:09.298 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:09.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:09.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:09.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:09 smithi016 bash[75179]: cluster 2023-10-09T09:16:08.132224+0000 mgr.y (mgr.25421) 1166 : cluster 0 pgmap v1269: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 9604/100140 objects degraded (9.591%); 10161/100140 objects misplaced (10.147%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T09:16:09.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:09 smithi016 bash[77892]: cluster 2023-10-09T09:16:08.132224+0000 mgr.y (mgr.25421) 1166 : cluster 0 pgmap v1269: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 9604/100140 objects degraded (9.591%); 10161/100140 objects misplaced (10.147%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T09:16:09.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:09.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:09.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:09 smithi094 bash[59677]: cluster 2023-10-09T09:16:08.132224+0000 mgr.y (mgr.25421) 1166 : cluster 0 pgmap v1269: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 9604/100140 objects degraded (9.591%); 10161/100140 objects misplaced (10.147%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T09:16:09.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:09.591 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:09.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:09.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:09.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:09.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:09.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:09.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:09.944 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:09.986 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:09.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:09.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:10.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:10.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:10.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:10.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:10.145 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:10.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:10.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:10.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:10.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:10.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:10.354 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:10 smithi016 bash[75179]: cluster 2023-10-09T09:16:09.755959+0000 mon.a (mon.0) 1562 : cluster 3 Health check update: Degraded data redundancy: 9604/100140 objects degraded (9.591%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:10.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:10 smithi016 bash[77892]: cluster 2023-10-09T09:16:09.755959+0000 mon.a (mon.0) 1562 : cluster 3 Health check update: Degraded data redundancy: 9604/100140 objects degraded (9.591%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:10.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:10.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:10.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:10.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:10.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:10.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:10.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:10.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:10.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:10.732 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:10 smithi094 bash[59677]: cluster 2023-10-09T09:16:09.755959+0000 mon.a (mon.0) 1562 : cluster 3 Health check update: Degraded data redundancy: 9604/100140 objects degraded (9.591%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:10.756 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:10.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:10.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:10.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:10.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:10.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:10.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:11.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:11.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:11.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:11.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:11.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:11.265 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:11 smithi094 bash[59677]: cluster 2023-10-09T09:16:10.132719+0000 mgr.y (mgr.25421) 1167 : cluster 0 pgmap v1270: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 9604/100140 objects degraded (9.591%); 10161/100140 objects misplaced (10.147%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T09:16:11.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:11.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:11.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:11.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:11 smithi016 bash[75179]: cluster 2023-10-09T09:16:10.132719+0000 mgr.y (mgr.25421) 1167 : cluster 0 pgmap v1270: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 9604/100140 objects degraded (9.591%); 10161/100140 objects misplaced (10.147%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T09:16:11.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:11 smithi016 bash[77892]: cluster 2023-10-09T09:16:10.132719+0000 mgr.y (mgr.25421) 1167 : cluster 0 pgmap v1270: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 9604/100140 objects degraded (9.591%); 10161/100140 objects misplaced (10.147%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T09:16:11.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:11.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:11.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:11.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:11.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:11.770 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:11.798 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:11.901 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:11.936 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:11.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:12.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:12.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:12.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:12.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:12.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:12.211 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:12.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:12.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:12.318 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:12.357 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:12.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:12.462 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:12.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:12.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:12.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:12.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:12.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:12.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:12.649 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:16:12.650 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:16:12.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:12.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:12.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:12.865 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:12.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:12.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:12.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:12.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:13.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:13.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:13.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:13.259 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:13.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:13.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:13.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:13.403 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:13 smithi016 bash[75179]: cluster 2023-10-09T09:16:12.133408+0000 mgr.y (mgr.25421) 1168 : cluster 0 pgmap v1271: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 9445/93705 objects degraded (10.080%); 9585/93705 objects misplaced (10.229%); 11 MiB/s, 168 objects/s recovering 2023-10-09T09:16:13.404 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:13 smithi016 bash[77892]: cluster 2023-10-09T09:16:12.133408+0000 mgr.y (mgr.25421) 1168 : cluster 0 pgmap v1271: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 9445/93705 objects degraded (10.080%); 9585/93705 objects misplaced (10.229%); 11 MiB/s, 168 objects/s recovering 2023-10-09T09:16:13.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:13.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:13.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:13.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:13.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:13.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:13 smithi094 bash[59677]: cluster 2023-10-09T09:16:12.133408+0000 mgr.y (mgr.25421) 1168 : cluster 0 pgmap v1271: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 9445/93705 objects degraded (10.080%); 9585/93705 objects misplaced (10.229%); 11 MiB/s, 168 objects/s recovering 2023-10-09T09:16:13.687 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:13.790 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:13.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:13.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:13.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:13.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:13.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:14.056 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:14.093 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:14.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:14.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:14.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:14.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:14.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:14.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:14.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:14.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:14.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:14.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:14.579 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:14.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:14.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:14.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:14.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:14.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:14.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:15.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:15.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:15.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:15.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:15.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:15.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:15.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:15.374 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:15.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:15.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:15 smithi016 bash[75179]: cluster 2023-10-09T09:16:14.134009+0000 mgr.y (mgr.25421) 1169 : cluster 0 pgmap v1272: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9445/91671 objects degraded (10.303%); 9242/91671 objects misplaced (10.082%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:16:15.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:15 smithi016 bash[75179]: cluster 2023-10-09T09:16:14.757060+0000 mon.a (mon.0) 1563 : cluster 3 Health check update: Degraded data redundancy: 9445/91671 objects degraded (10.303%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:15 smithi016 bash[77892]: cluster 2023-10-09T09:16:14.134009+0000 mgr.y (mgr.25421) 1169 : cluster 0 pgmap v1272: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9445/91671 objects degraded (10.303%); 9242/91671 objects misplaced (10.082%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:16:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:15 smithi016 bash[77892]: cluster 2023-10-09T09:16:14.757060+0000 mon.a (mon.0) 1563 : cluster 3 Health check update: Degraded data redundancy: 9445/91671 objects degraded (10.303%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:15.502 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:15.540 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:15.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:15 smithi094 bash[59677]: cluster 2023-10-09T09:16:14.134009+0000 mgr.y (mgr.25421) 1169 : cluster 0 pgmap v1272: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9445/91671 objects degraded (10.303%); 9242/91671 objects misplaced (10.082%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:16:15.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:15 smithi094 bash[59677]: cluster 2023-10-09T09:16:14.757060+0000 mon.a (mon.0) 1563 : cluster 3 Health check update: Degraded data redundancy: 9445/91671 objects degraded (10.303%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:15.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:15.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:15.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:15.775 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:15.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:15.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:15.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:15.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:15.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:15.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:15.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:16:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:16:15] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-09T09:16:16.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:16.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:16.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:16.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:16.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:16.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:16.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:16.344 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:16.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:16.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:16.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:16.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:16.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:16.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:16.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:16.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:16.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:16.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:16.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:16.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:16.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:16.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:16.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:16.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:16.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:17.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:17.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:17.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:17.233 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:17.280 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:17.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:17.328 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:17 smithi016 bash[75179]: cluster 2023-10-09T09:16:16.134745+0000 mgr.y (mgr.25421) 1170 : cluster 0 pgmap v1273: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 9274/85368 objects degraded (10.864%); 8682/85368 objects misplaced (10.170%); 10 MiB/s, 161 objects/s recovering 2023-10-09T09:16:17.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:17.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:17.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:17.520 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:17 smithi094 bash[59677]: cluster 2023-10-09T09:16:16.134745+0000 mgr.y (mgr.25421) 1170 : cluster 0 pgmap v1273: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 9274/85368 objects degraded (10.864%); 8682/85368 objects misplaced (10.170%); 10 MiB/s, 161 objects/s recovering 2023-10-09T09:16:17.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:17.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:17.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:17.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:17.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:17 smithi016 bash[77892]: cluster 2023-10-09T09:16:16.134745+0000 mgr.y (mgr.25421) 1170 : cluster 0 pgmap v1273: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 9274/85368 objects degraded (10.864%); 8682/85368 objects misplaced (10.170%); 10 MiB/s, 161 objects/s recovering 2023-10-09T09:16:17.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:17.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:17.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:17.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:17.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:17.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:17.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:18.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:16:18.046 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:18.075 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:18.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:18.179 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:18.205 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:18.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:18.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:18.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:18.344 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:18.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:18.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:18.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:18.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:18.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:18.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:18.607 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:18.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:18.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:18.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:18.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:18.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:18.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:18.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:18.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:18.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:18.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:18.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:19.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:19.101 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:19.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:19.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:19.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:19.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:19.363 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:19.399 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:19.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[75179]: cluster 2023-10-09T09:16:18.135491+0000 mgr.y (mgr.25421) 1171 : cluster 0 pgmap v1274: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 9274/83268 objects degraded (11.138%); 8350/83268 objects misplaced (10.028%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T09:16:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[75179]: cluster 2023-10-09T09:16:18.293391+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T09:16:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[75179]: cluster 2023-10-09T09:16:18.294162+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[75179]: cluster 2023-10-09T09:16:18.295087+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[75179]: cluster 2023-10-09T09:16:18.302509+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[75179]: cluster 2023-10-09T09:16:18.304822+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[75179]: audit 2023-10-09T09:16:19.250945+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[77892]: cluster 2023-10-09T09:16:18.135491+0000 mgr.y (mgr.25421) 1171 : cluster 0 pgmap v1274: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 9274/83268 objects degraded (11.138%); 8350/83268 objects misplaced (10.028%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T09:16:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[77892]: cluster 2023-10-09T09:16:18.293391+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T09:16:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[77892]: cluster 2023-10-09T09:16:18.294162+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[77892]: cluster 2023-10-09T09:16:18.295087+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[77892]: cluster 2023-10-09T09:16:18.302509+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[77892]: cluster 2023-10-09T09:16:18.304822+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:19 smithi016 bash[77892]: audit 2023-10-09T09:16:19.250945+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:19.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:19.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:19.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:19.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:19.656 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:19 smithi094 bash[59677]: cluster 2023-10-09T09:16:18.135491+0000 mgr.y (mgr.25421) 1171 : cluster 0 pgmap v1274: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 9274/83268 objects degraded (11.138%); 8350/83268 objects misplaced (10.028%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T09:16:19.656 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:19 smithi094 bash[59677]: cluster 2023-10-09T09:16:18.293391+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T09:16:19.656 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:19 smithi094 bash[59677]: cluster 2023-10-09T09:16:18.294162+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.657 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:19 smithi094 bash[59677]: cluster 2023-10-09T09:16:18.295087+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.657 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:19 smithi094 bash[59677]: cluster 2023-10-09T09:16:18.302509+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.657 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:19 smithi094 bash[59677]: cluster 2023-10-09T09:16:18.304822+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T09:16:19.657 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:19 smithi094 bash[59677]: audit 2023-10-09T09:16:19.250945+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:19.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:19.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:19.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:19.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:19.921 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:19.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:19.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:20.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:20.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:20.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:20.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:20.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:20.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:20.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:20.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:20.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:20.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:20.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:20.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:20.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[75179]: cluster 2023-10-09T09:16:19.297847+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[75179]: cluster 2023-10-09T09:16:19.299352+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T09:16:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[75179]: cluster 2023-10-09T09:16:19.301227+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[75179]: cluster 2023-10-09T09:16:19.311840+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[75179]: cluster 2023-10-09T09:16:19.757969+0000 mon.a (mon.0) 1567 : cluster 3 Health check update: Degraded data redundancy: 9274/83268 objects degraded (11.138%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[77892]: cluster 2023-10-09T09:16:19.297847+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[77892]: cluster 2023-10-09T09:16:19.299352+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T09:16:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[77892]: cluster 2023-10-09T09:16:19.301227+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[77892]: cluster 2023-10-09T09:16:19.311840+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:20 smithi016 bash[77892]: cluster 2023-10-09T09:16:19.757969+0000 mon.a (mon.0) 1567 : cluster 3 Health check update: Degraded data redundancy: 9274/83268 objects degraded (11.138%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:20.508 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:20.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:20.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:20.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:20.621 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:20 smithi094 bash[59677]: cluster 2023-10-09T09:16:19.297847+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.622 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:20 smithi094 bash[59677]: cluster 2023-10-09T09:16:19.299352+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T09:16:20.622 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:20 smithi094 bash[59677]: cluster 2023-10-09T09:16:19.301227+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.622 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:20 smithi094 bash[59677]: cluster 2023-10-09T09:16:19.311840+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T09:16:20.622 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:20 smithi094 bash[59677]: cluster 2023-10-09T09:16:19.757969+0000 mon.a (mon.0) 1567 : cluster 3 Health check update: Degraded data redundancy: 9274/83268 objects degraded (11.138%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:20.638 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:20.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:20.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:20.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:20.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:20.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:20.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:20.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:20.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:20.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:21.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:21.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:21.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:21.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:21.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:21.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:21.301 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:21 smithi094 bash[59677]: cluster 2023-10-09T09:16:20.136039+0000 mgr.y (mgr.25421) 1172 : cluster 0 pgmap v1277: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 9274/83268 objects degraded (11.138%); 8350/83268 objects misplaced (10.028%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T09:16:21.394 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:21.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:21.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:21 smithi016 bash[75179]: cluster 2023-10-09T09:16:20.136039+0000 mgr.y (mgr.25421) 1172 : cluster 0 pgmap v1277: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 9274/83268 objects degraded (11.138%); 8350/83268 objects misplaced (10.028%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T09:16:21.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:21 smithi016 bash[77892]: cluster 2023-10-09T09:16:20.136039+0000 mgr.y (mgr.25421) 1172 : cluster 0 pgmap v1277: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 9274/83268 objects degraded (11.138%); 8350/83268 objects misplaced (10.028%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T09:16:21.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:21.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:21.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:21.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:21.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:21.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:21.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:21.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:21.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:21.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:21.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:21.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:22.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:22.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:22.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:22.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:22.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:22.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:22.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:22.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:22.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:22.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:22.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:22.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:22.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:22.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:22.572 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:22.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:22.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:22.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:22.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:22.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:22.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:22.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:22.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:22.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:23.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:23.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:23.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:23.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:23.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:23.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:23.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:23.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:23.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:23.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:23.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:23.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:23.540 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:23.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:23.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:23.577 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:23 smithi094 bash[59677]: cluster 2023-10-09T09:16:22.136840+0000 mgr.y (mgr.25421) 1173 : cluster 0 pgmap v1278: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 9132/77901 objects degraded (11.723%); 8097/77901 objects misplaced (10.394%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T09:16:23.677 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:23.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:23.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:23.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:23.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:23 smithi016 bash[75179]: cluster 2023-10-09T09:16:22.136840+0000 mgr.y (mgr.25421) 1173 : cluster 0 pgmap v1278: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 9132/77901 objects degraded (11.723%); 8097/77901 objects misplaced (10.394%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T09:16:23.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:23 smithi016 bash[77892]: cluster 2023-10-09T09:16:22.136840+0000 mgr.y (mgr.25421) 1173 : cluster 0 pgmap v1278: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 9132/77901 objects degraded (11.723%); 8097/77901 objects misplaced (10.394%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T09:16:23.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:23.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:23.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:23.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:23.998 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:24.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:24.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:24.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:24.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:24.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:24.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:24.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:24.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:24.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:24.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:24.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:24.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:24.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:24.797 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:24.902 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:24.933 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:24.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:24.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:25.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:25.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:25.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:25.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:25.215 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:25.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:25.306 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:25 smithi016 bash[77892]: cluster 2023-10-09T09:16:24.137454+0000 mgr.y (mgr.25421) 1174 : cluster 0 pgmap v1279: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 9132/76095 objects degraded (12.001%); 7732/76095 objects misplaced (10.161%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:16:25.307 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:25 smithi016 bash[77892]: cluster 2023-10-09T09:16:24.759079+0000 mon.a (mon.0) 1568 : cluster 3 Health check update: Degraded data redundancy: 9132/76095 objects degraded (12.001%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:25.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:25.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:25.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:25.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:25.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:25.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:25.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:25 smithi094 bash[59677]: cluster 2023-10-09T09:16:24.137454+0000 mgr.y (mgr.25421) 1174 : cluster 0 pgmap v1279: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 9132/76095 objects degraded (12.001%); 7732/76095 objects misplaced (10.161%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:16:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:25 smithi094 bash[59677]: cluster 2023-10-09T09:16:24.759079+0000 mon.a (mon.0) 1568 : cluster 3 Health check update: Degraded data redundancy: 9132/76095 objects degraded (12.001%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:25.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:25.599 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:25 smithi016 bash[75179]: cluster 2023-10-09T09:16:24.137454+0000 mgr.y (mgr.25421) 1174 : cluster 0 pgmap v1279: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 9132/76095 objects degraded (12.001%); 7732/76095 objects misplaced (10.161%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:16:25.599 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:25 smithi016 bash[75179]: cluster 2023-10-09T09:16:24.759079+0000 mon.a (mon.0) 1568 : cluster 3 Health check update: Degraded data redundancy: 9132/76095 objects degraded (12.001%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:25.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:25.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:25.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:25.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:25.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:25.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:25.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:25.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:25.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:25.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:25.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:25.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:16:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:16:25] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T09:16:26.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:26.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:26.158 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:26.159 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:26.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:26.265 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:26.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:26.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:26.400 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:26.436 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:26.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:26.539 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:26.571 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:26.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:26.676 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:26.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:26.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:26.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:26.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:26.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:26.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:27.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:27.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:27.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:27.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:27.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:27.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:27.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:27.334 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:27.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:27.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:27.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:27.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:27 smithi016 bash[75179]: cluster 2023-10-09T09:16:26.138103+0000 mgr.y (mgr.25421) 1175 : cluster 0 pgmap v1280: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8966/69798 objects degraded (12.846%); 7566/69798 objects misplaced (10.840%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:16:27.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:27 smithi016 bash[77892]: cluster 2023-10-09T09:16:26.138103+0000 mgr.y (mgr.25421) 1175 : cluster 0 pgmap v1280: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8966/69798 objects degraded (12.846%); 7566/69798 objects misplaced (10.840%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:16:27.511 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:27.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:27.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:27.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:27.624 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:27 smithi094 bash[59677]: cluster 2023-10-09T09:16:26.138103+0000 mgr.y (mgr.25421) 1175 : cluster 0 pgmap v1280: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8966/69798 objects degraded (12.846%); 7566/69798 objects misplaced (10.840%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T09:16:27.655 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:27.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:27.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:27.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:27.793 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:27.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:27.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:27.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:27.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:27.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:28.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:28.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:28.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:28.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:28.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:28.329 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:16:28.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:28.422 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:28 smithi094 bash[59677]: audit 2023-10-09T09:16:27.616115+0000 mon.a (mon.0) 1569 : audit 1 from='client.? 172.21.15.16:0/2383565015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:16:28.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:28.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:28 smithi016 bash[75179]: audit 2023-10-09T09:16:27.616115+0000 mon.a (mon.0) 1569 : audit 1 from='client.? 172.21.15.16:0/2383565015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:16:28.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:28 smithi016 bash[77892]: audit 2023-10-09T09:16:27.616115+0000 mon.a (mon.0) 1569 : audit 1 from='client.? 172.21.15.16:0/2383565015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:16:28.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:28.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:28.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:28.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:28.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:28.980 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:28.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:28.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:29.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:29.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:29.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:29.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:29.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:29.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:29.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:29.355 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:29.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:29.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[75179]: cluster 2023-10-09T09:16:28.138704+0000 mgr.y (mgr.25421) 1176 : cluster 0 pgmap v1281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 8966/67626 objects degraded (13.258%); 7199/67626 objects misplaced (10.645%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T09:16:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[75179]: cluster 2023-10-09T09:16:28.308432+0000 mon.a (mon.0) 1570 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:16:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[75179]: cluster 2023-10-09T09:16:28.321643+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T09:16:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[75179]: audit 2023-10-09T09:16:28.322660+0000 mon.a (mon.0) 1571 : audit 1 from='client.? 172.21.15.16:0/2383565015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:16:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[75179]: cluster 2023-10-09T09:16:28.322721+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T09:16:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[75179]: cluster 2023-10-09T09:16:28.329488+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T09:16:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[77892]: cluster 2023-10-09T09:16:28.138704+0000 mgr.y (mgr.25421) 1176 : cluster 0 pgmap v1281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 8966/67626 objects degraded (13.258%); 7199/67626 objects misplaced (10.645%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T09:16:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[77892]: cluster 2023-10-09T09:16:28.308432+0000 mon.a (mon.0) 1570 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:16:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[77892]: cluster 2023-10-09T09:16:28.321643+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T09:16:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[77892]: audit 2023-10-09T09:16:28.322660+0000 mon.a (mon.0) 1571 : audit 1 from='client.? 172.21.15.16:0/2383565015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:16:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[77892]: cluster 2023-10-09T09:16:28.322721+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T09:16:29.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:29 smithi016 bash[77892]: cluster 2023-10-09T09:16:28.329488+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T09:16:29.501 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:29.531 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:29.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:29.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:29 smithi094 bash[59677]: cluster 2023-10-09T09:16:28.138704+0000 mgr.y (mgr.25421) 1176 : cluster 0 pgmap v1281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 8966/67626 objects degraded (13.258%); 7199/67626 objects misplaced (10.645%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T09:16:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:29 smithi094 bash[59677]: cluster 2023-10-09T09:16:28.308432+0000 mon.a (mon.0) 1570 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:16:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:29 smithi094 bash[59677]: cluster 2023-10-09T09:16:28.321643+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T09:16:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:29 smithi094 bash[59677]: audit 2023-10-09T09:16:28.322660+0000 mon.a (mon.0) 1571 : audit 1 from='client.? 172.21.15.16:0/2383565015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:16:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:29 smithi094 bash[59677]: cluster 2023-10-09T09:16:28.322721+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T09:16:29.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:29 smithi094 bash[59677]: cluster 2023-10-09T09:16:28.329488+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T09:16:29.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:29.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:29.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:29.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:29.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:29.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:29.948 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:29.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:29.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:30.053 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:30.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:30.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:30.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:30.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:30.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:30.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:30.280 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:16:30.308 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:16:30.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:30.338 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:16:30.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:30.375 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:16:30.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:30.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:30.410 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:16:30.442 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:16:30.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:30.485 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:30 smithi016 bash[75179]: cluster 2023-10-09T09:16:29.760310+0000 mon.a (mon.0) 1573 : cluster 3 Health check update: Degraded data redundancy: 8966/67626 objects degraded (13.258%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:30.485 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:30 smithi016 bash[75179]: audit 2023-10-09T09:16:30.242501+0000 mon.a (mon.0) 1574 : audit 0 from='client.? 172.21.15.16:0/3453006381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:16:30.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:30 smithi016 bash[77892]: cluster 2023-10-09T09:16:29.760310+0000 mon.a (mon.0) 1573 : cluster 3 Health check update: Degraded data redundancy: 8966/67626 objects degraded (13.258%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:30.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:30 smithi016 bash[77892]: audit 2023-10-09T09:16:30.242501+0000 mon.a (mon.0) 1574 : audit 0 from='client.? 172.21.15.16:0/3453006381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:16:30.495 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:16:30.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:30.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:30.539 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:16:30.541 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.541 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.541 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.541 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.542 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.542 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.542 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.542 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:16:30.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:30.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:30.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:30.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:30 smithi094 bash[59677]: cluster 2023-10-09T09:16:29.760310+0000 mon.a (mon.0) 1573 : cluster 3 Health check update: Degraded data redundancy: 8966/67626 objects degraded (13.258%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:30.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:30 smithi094 bash[59677]: audit 2023-10-09T09:16:30.242501+0000 mon.a (mon.0) 1574 : audit 0 from='client.? 172.21.15.16:0/3453006381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:16:30.742 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:30.785 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:30.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:30.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:30.923 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:30.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:31.030 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:31.058 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:31.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:31.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:31.165 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:31.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:31.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:31.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:31.426 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:31.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:31.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:31.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:31.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:31.668 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:31.676 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:31 smithi094 bash[59677]: cluster 2023-10-09T09:16:30.139149+0000 mgr.y (mgr.25421) 1177 : cluster 0 pgmap v1283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 8966/67626 objects degraded (13.258%); 7199/67626 objects misplaced (10.645%); 5.1 MiB/s, 80 objects/s recovering 2023-10-09T09:16:31.712 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:31.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:31.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:31 smithi016 bash[77892]: cluster 2023-10-09T09:16:30.139149+0000 mgr.y (mgr.25421) 1177 : cluster 0 pgmap v1283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 8966/67626 objects degraded (13.258%); 7199/67626 objects misplaced (10.645%); 5.1 MiB/s, 80 objects/s recovering 2023-10-09T09:16:31.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:31 smithi016 bash[75179]: cluster 2023-10-09T09:16:30.139149+0000 mgr.y (mgr.25421) 1177 : cluster 0 pgmap v1283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 8966/67626 objects degraded (13.258%); 7199/67626 objects misplaced (10.645%); 5.1 MiB/s, 80 objects/s recovering 2023-10-09T09:16:31.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:31.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:31.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:31.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:31.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:31.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:31.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:32.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:32.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:32.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:32.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:32.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:32.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:32.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:32.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:32.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:32.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:32.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:32.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:32.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:32.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:32.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:32.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:32.726 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:32.767 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:32.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:32.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:32.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:32.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:32.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:33.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:33.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:33.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:33.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:33.190 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:33.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:33.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:33.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:33.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:33 smithi094 bash[59677]: cluster 2023-10-09T09:16:32.139784+0000 mgr.y (mgr.25421) 1178 : cluster 0 pgmap v1284: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 8826/60990 objects degraded (14.471%); 7060/60990 objects misplaced (11.576%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:16:33.419 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:33.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:33.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:33.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:33.563 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:33 smithi016 bash[75179]: cluster 2023-10-09T09:16:32.139784+0000 mgr.y (mgr.25421) 1178 : cluster 0 pgmap v1284: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 8826/60990 objects degraded (14.471%); 7060/60990 objects misplaced (11.576%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:16:33.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:33 smithi016 bash[77892]: cluster 2023-10-09T09:16:32.139784+0000 mgr.y (mgr.25421) 1178 : cluster 0 pgmap v1284: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 8826/60990 objects degraded (14.471%); 7060/60990 objects misplaced (11.576%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:16:33.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:33.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:33.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:33.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:33.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:33.810 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:33.838 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:33.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:33.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:33.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:33.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:34.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:34.132 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:34.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:34.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:34.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:34.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:34.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:34.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:34.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:34.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:34.532 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:34.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:34.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:34.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:34.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:34.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:34.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:34.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:34.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:16:34.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:34.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:34.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:34.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:34.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:34.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:34.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:35.070 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:35.106 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:35.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:35.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:35.246 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:35.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:35.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:35.281 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 10 pgp_num to 16 2023-10-09T09:16:35.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:35.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:35.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:35.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:35 smithi094 bash[59677]: cluster 2023-10-09T09:16:34.140491+0000 mgr.y (mgr.25421) 1179 : cluster 0 pgmap v1285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8826/58791 objects degraded (15.013%); 6705/58791 objects misplaced (11.405%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:16:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:35 smithi094 bash[59677]: audit 2023-10-09T09:16:34.255872+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:35 smithi094 bash[59677]: audit 2023-10-09T09:16:34.255980+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.16:0/93785135' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:16:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:35 smithi094 bash[59677]: audit 2023-10-09T09:16:34.257644+0000 mon.a (mon.0) 1576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:16:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:35 smithi094 bash[59677]: audit 2023-10-09T09:16:34.257989+0000 mon.a (mon.0) 1577 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:35.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:35 smithi094 bash[59677]: cluster 2023-10-09T09:16:34.762623+0000 mon.a (mon.0) 1578 : cluster 3 Health check update: Degraded data redundancy: 8826/58791 objects degraded (15.013%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:35.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:35.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:35.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:35.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[77892]: cluster 2023-10-09T09:16:34.140491+0000 mgr.y (mgr.25421) 1179 : cluster 0 pgmap v1285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8826/58791 objects degraded (15.013%); 6705/58791 objects misplaced (11.405%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:16:35.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[77892]: audit 2023-10-09T09:16:34.255872+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:35.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[77892]: audit 2023-10-09T09:16:34.255980+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.16:0/93785135' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:16:35.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[77892]: audit 2023-10-09T09:16:34.257644+0000 mon.a (mon.0) 1576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:16:35.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[77892]: audit 2023-10-09T09:16:34.257989+0000 mon.a (mon.0) 1577 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:35.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[77892]: cluster 2023-10-09T09:16:34.762623+0000 mon.a (mon.0) 1578 : cluster 3 Health check update: Degraded data redundancy: 8826/58791 objects degraded (15.013%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:35.611 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[75179]: cluster 2023-10-09T09:16:34.140491+0000 mgr.y (mgr.25421) 1179 : cluster 0 pgmap v1285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8826/58791 objects degraded (15.013%); 6705/58791 objects misplaced (11.405%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:16:35.611 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[75179]: audit 2023-10-09T09:16:34.255872+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:35.611 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[75179]: audit 2023-10-09T09:16:34.255980+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.16:0/93785135' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:16:35.611 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[75179]: audit 2023-10-09T09:16:34.257644+0000 mon.a (mon.0) 1576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:16:35.611 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[75179]: audit 2023-10-09T09:16:34.257989+0000 mon.a (mon.0) 1577 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:35.611 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[75179]: cluster 2023-10-09T09:16:34.762623+0000 mon.a (mon.0) 1578 : cluster 3 Health check update: Degraded data redundancy: 8826/58791 objects degraded (15.013%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:35.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:35.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:35.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:35.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:35.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:35.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:35.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:35.867 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:16:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:16:35] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T09:16:35.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:35.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:35.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:36.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:36.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:16:36.137 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:36.164 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:36.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:36.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:36.275 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: audit 2023-10-09T09:16:35.266829+0000 mon.a (mon.0) 1579 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:16:36.275 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: cluster 2023-10-09T09:16:35.266914+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T09:16:36.275 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: cluster 2023-10-09T09:16:35.268884+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.275 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: cluster 2023-10-09T09:16:35.275214+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: cluster 2023-10-09T09:16:35.277488+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: cluster 2023-10-09T09:16:35.279359+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: cluster 2023-10-09T09:16:35.280679+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:36 smithi094 bash[59677]: cluster 2023-10-09T09:16:35.281978+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:36.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:36.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:36.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:36.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: audit 2023-10-09T09:16:35.266829+0000 mon.a (mon.0) 1579 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: cluster 2023-10-09T09:16:35.266914+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: cluster 2023-10-09T09:16:35.268884+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: cluster 2023-10-09T09:16:35.275214+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: cluster 2023-10-09T09:16:35.277488+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: cluster 2023-10-09T09:16:35.279359+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: cluster 2023-10-09T09:16:35.280679+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[75179]: cluster 2023-10-09T09:16:35.281978+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: audit 2023-10-09T09:16:35.266829+0000 mon.a (mon.0) 1579 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: cluster 2023-10-09T09:16:35.266914+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T09:16:36.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: cluster 2023-10-09T09:16:35.268884+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: cluster 2023-10-09T09:16:35.275214+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: cluster 2023-10-09T09:16:35.277488+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: cluster 2023-10-09T09:16:35.279359+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: cluster 2023-10-09T09:16:35.280679+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:36 smithi016 bash[77892]: cluster 2023-10-09T09:16:35.281978+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T09:16:36.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:36.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:36.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:36.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:36.703 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:36.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:36.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:36.853 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:36.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:36.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:36.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:36.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:37.097 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:37.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:37.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:37.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:37.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:37.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:37.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:37.361 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:37.370 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:37 smithi094 bash[59677]: cluster 2023-10-09T09:16:36.141288+0000 mgr.y (mgr.25421) 1180 : cluster 0 pgmap v1287: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 8656/52305 objects degraded (16.549%); 6535/52305 objects misplaced (12.494%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:16:37.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:37.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:37.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:37 smithi016 bash[77892]: cluster 2023-10-09T09:16:36.141288+0000 mgr.y (mgr.25421) 1180 : cluster 0 pgmap v1287: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 8656/52305 objects degraded (16.549%); 6535/52305 objects misplaced (12.494%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:16:37.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:37 smithi016 bash[75179]: cluster 2023-10-09T09:16:36.141288+0000 mgr.y (mgr.25421) 1180 : cluster 0 pgmap v1287: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 8656/52305 objects degraded (16.549%); 6535/52305 objects misplaced (12.494%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:16:37.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:37.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:37.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:37.639 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:37.681 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:37.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:37.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:37.819 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:37.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:37.922 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:37.956 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:37.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:37.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:38.063 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:38.096 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:38.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:38.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:38.241 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:38.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:38.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:38.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:38.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:38.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:38.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:38.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:38.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:38.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:38.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:38.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:38.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:38.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:38.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:38.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:38.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:38.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:38.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:38.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:38.943 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:16:38.943 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:16:38.943 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T09:16:38.943 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T09:16:38.943 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:16:39.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:39.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:39.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:39.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:39.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:39.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:39.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:39.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:39.433 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:39 smithi016 bash[75179]: cluster 2023-10-09T09:16:38.141987+0000 mgr.y (mgr.25421) 1181 : cluster 0 pgmap v1288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 8656/50256 objects degraded (17.224%); 6155/50256 objects misplaced (12.247%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:16:39.434 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:39 smithi016 bash[77892]: cluster 2023-10-09T09:16:38.141987+0000 mgr.y (mgr.25421) 1181 : cluster 0 pgmap v1288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 8656/50256 objects degraded (17.224%); 6155/50256 objects misplaced (12.247%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:16:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:39.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:39.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:39 smithi094 bash[59677]: cluster 2023-10-09T09:16:38.141987+0000 mgr.y (mgr.25421) 1181 : cluster 0 pgmap v1288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 8656/50256 objects degraded (17.224%); 6155/50256 objects misplaced (12.247%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:16:39.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:39.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:39.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:39.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:39.805 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:39.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:39.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:39.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:39.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:39.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:40.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:40.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:40.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:40.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:40.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:40.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:40.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:40.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:40.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:40 smithi094 bash[59677]: cluster 2023-10-09T09:16:39.763756+0000 mon.a (mon.0) 1581 : cluster 3 Health check update: Degraded data redundancy: 8656/50256 objects degraded (17.224%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:40.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:40.526 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:40.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:40.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:40.666 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:40.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:40.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:40.689 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:40 smithi016 bash[75179]: cluster 2023-10-09T09:16:39.763756+0000 mon.a (mon.0) 1581 : cluster 3 Health check update: Degraded data redundancy: 8656/50256 objects degraded (17.224%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:40.690 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:40 smithi016 bash[77892]: cluster 2023-10-09T09:16:39.763756+0000 mon.a (mon.0) 1581 : cluster 3 Health check update: Degraded data redundancy: 8656/50256 objects degraded (17.224%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:40.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:16:40.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:40.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:40.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:40.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:40.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:40.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:41.057 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:41.093 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:41.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:41.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:41.232 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:41.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:41.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:41.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:41.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:41.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:41.485 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:41 smithi094 bash[59677]: cluster 2023-10-09T09:16:40.142499+0000 mgr.y (mgr.25421) 1182 : cluster 0 pgmap v1289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 8656/50256 objects degraded (17.224%); 6155/50256 objects misplaced (12.247%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T09:16:41.513 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:41.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:41.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:41.629 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:41 smithi016 bash[75179]: cluster 2023-10-09T09:16:40.142499+0000 mgr.y (mgr.25421) 1182 : cluster 0 pgmap v1289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 8656/50256 objects degraded (17.224%); 6155/50256 objects misplaced (12.247%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T09:16:41.630 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:41 smithi016 bash[77892]: cluster 2023-10-09T09:16:40.142499+0000 mgr.y (mgr.25421) 1182 : cluster 0 pgmap v1289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 8656/50256 objects degraded (17.224%); 6155/50256 objects misplaced (12.247%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T09:16:41.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:41.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:41.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:41.751 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:41.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:41.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:41.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:41.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:41.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:41.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:42.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:42.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:42.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:42.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:42.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:42.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:42.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:42.321 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:42.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:42.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:42.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:42.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:42.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:42.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:42.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:42.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:42.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:42.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:42.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:42.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:42.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:43.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:43.122 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:16:43.122 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:16:43.123 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T09:16:43.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:16:43.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:43.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:43.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:43.206 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:43.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:43.267 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:43.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:43.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:43.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:43.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:43.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:43.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:43.514 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:43 smithi094 bash[59677]: cluster 2023-10-09T09:16:42.143470+0000 mgr.y (mgr.25421) 1183 : cluster 0 pgmap v1290: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 911 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 8497/43776 objects degraded (19.410%); 5996/43776 objects misplaced (13.697%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T09:16:43.536 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:43.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:43.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:43.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:43.653 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:43 smithi016 bash[75179]: cluster 2023-10-09T09:16:42.143470+0000 mgr.y (mgr.25421) 1183 : cluster 0 pgmap v1290: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 911 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 8497/43776 objects degraded (19.410%); 5996/43776 objects misplaced (13.697%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T09:16:43.654 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:43 smithi016 bash[77892]: cluster 2023-10-09T09:16:42.143470+0000 mgr.y (mgr.25421) 1183 : cluster 0 pgmap v1290: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 911 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 8497/43776 objects degraded (19.410%); 5996/43776 objects misplaced (13.697%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T09:16:43.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:43.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:43.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:43.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:43.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:43.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:43.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:43.941 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:43.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:44.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:44.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:44.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:44.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:44.173 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:44.201 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:44.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:44.222 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:44.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:44.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:44.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:44.434 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:44.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:44.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:44.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:44.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:44.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:44.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:44.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:44.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:44.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:44.841 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:44.880 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:44.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:44.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:45.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:45.122 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:45.157 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:45.239 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:45.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:45.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:45.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:45.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:45.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:45.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:45.524 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:45 smithi016 bash[75179]: cluster 2023-10-09T09:16:44.144224+0000 mgr.y (mgr.25421) 1184 : cluster 0 pgmap v1291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 864 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 8497/41532 objects degraded (20.459%); 5603/41532 objects misplaced (13.491%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:16:45.525 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:45 smithi016 bash[75179]: cluster 2023-10-09T09:16:44.765059+0000 mon.a (mon.0) 1582 : cluster 3 Health check update: Degraded data redundancy: 8497/41532 objects degraded (20.459%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:45.525 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:45 smithi016 bash[77892]: cluster 2023-10-09T09:16:44.144224+0000 mgr.y (mgr.25421) 1184 : cluster 0 pgmap v1291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 864 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 8497/41532 objects degraded (20.459%); 5603/41532 objects misplaced (13.491%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:16:45.525 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:45 smithi016 bash[77892]: cluster 2023-10-09T09:16:44.765059+0000 mon.a (mon.0) 1582 : cluster 3 Health check update: Degraded data redundancy: 8497/41532 objects degraded (20.459%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:45.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:45.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:45.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:45.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:45 smithi094 bash[59677]: cluster 2023-10-09T09:16:44.144224+0000 mgr.y (mgr.25421) 1184 : cluster 0 pgmap v1291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 864 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 8497/41532 objects degraded (20.459%); 5603/41532 objects misplaced (13.491%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:16:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:45 smithi094 bash[59677]: cluster 2023-10-09T09:16:44.765059+0000 mon.a (mon.0) 1582 : cluster 3 Health check update: Degraded data redundancy: 8497/41532 objects degraded (20.459%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:45.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:45.779 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:16:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:16:45] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T09:16:45.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:45.899 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:45.927 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:45.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:46.030 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:46.077 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:46.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:46.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:46.205 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:46.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:46.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:46.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:46.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:46.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:46.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:46.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:46.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:46.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:46.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:46.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:46.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:46.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:46.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:46.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:46.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:46.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:46.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:46.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:46.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:46.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:46.978 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:47.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:47.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:47.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:47.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:47.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:47.271 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:16:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:47.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:47.382 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:47 smithi016 bash[75179]: cluster 2023-10-09T09:16:46.144888+0000 mgr.y (mgr.25421) 1185 : cluster 0 pgmap v1292: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 729 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 8336/35028 objects degraded (23.798%); 5442/35028 objects misplaced (15.536%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:16:47.383 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:47 smithi016 bash[77892]: cluster 2023-10-09T09:16:46.144888+0000 mgr.y (mgr.25421) 1185 : cluster 0 pgmap v1292: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 729 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 8336/35028 objects degraded (23.798%); 5442/35028 objects misplaced (15.536%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:16:47.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:47.507 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:47.534 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:47.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:47.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:47.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:47 smithi094 bash[59677]: cluster 2023-10-09T09:16:46.144888+0000 mgr.y (mgr.25421) 1185 : cluster 0 pgmap v1292: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 729 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 8336/35028 objects degraded (23.798%); 5442/35028 objects misplaced (15.536%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T09:16:47.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:47.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:47.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:47.889 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:47.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:47.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:48.027 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:48.058 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:48.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:48.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:48.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:48.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:48.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:48.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:48.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:48.568 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:48.598 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:48.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:48.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:48.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:48.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:48.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:48.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:48.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:48.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:48.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:48.995 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:48.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:49.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:49.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:49.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:49.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:49.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:49.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:49.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:49.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:49.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:49.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:49.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:49.442 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:49 smithi016 bash[75179]: cluster 2023-10-09T09:16:48.145611+0000 mgr.y (mgr.25421) 1186 : cluster 0 pgmap v1293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 684 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 8336/32859 objects degraded (25.369%); 5066/32859 objects misplaced (15.417%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:16:49.442 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:49 smithi016 bash[75179]: audit 2023-10-09T09:16:49.251149+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:49.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:49 smithi016 bash[77892]: cluster 2023-10-09T09:16:48.145611+0000 mgr.y (mgr.25421) 1186 : cluster 0 pgmap v1293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 684 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 8336/32859 objects degraded (25.369%); 5066/32859 objects misplaced (15.417%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:16:49.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:49 smithi016 bash[77892]: audit 2023-10-09T09:16:49.251149+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:49.507 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:49.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:49.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:49.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:49.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:49.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:49 smithi094 bash[59677]: cluster 2023-10-09T09:16:48.145611+0000 mgr.y (mgr.25421) 1186 : cluster 0 pgmap v1293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 684 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 8336/32859 objects degraded (25.369%); 5066/32859 objects misplaced (15.417%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:16:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:49 smithi094 bash[59677]: audit 2023-10-09T09:16:49.251149+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:49.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:49.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:49.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:49.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:49.910 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:49.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:49.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:50.046 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:50.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:50.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:50.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:50.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:50.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:50.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:50.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:50.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:50.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:50 smithi016 bash[75179]: cluster 2023-10-09T09:16:49.766146+0000 mon.a (mon.0) 1584 : cluster 3 Health check update: Degraded data redundancy: 8336/32859 objects degraded (25.369%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:50.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:50 smithi016 bash[77892]: cluster 2023-10-09T09:16:49.766146+0000 mon.a (mon.0) 1584 : cluster 3 Health check update: Degraded data redundancy: 8336/32859 objects degraded (25.369%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:50.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:50.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:50.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:50.699 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:50 smithi094 bash[59677]: cluster 2023-10-09T09:16:49.766146+0000 mon.a (mon.0) 1584 : cluster 3 Health check update: Degraded data redundancy: 8336/32859 objects degraded (25.369%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:50.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:50.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:50.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:50.944 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:50.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:50.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:50.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:51.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:51.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:51.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:51.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:51.234 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:51.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:51.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:51.347 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:51 smithi016 bash[75179]: cluster 2023-10-09T09:16:50.146105+0000 mgr.y (mgr.25421) 1187 : cluster 0 pgmap v1294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 684 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 8336/32859 objects degraded (25.369%); 5066/32859 objects misplaced (15.417%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:16:51.347 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:51 smithi016 bash[77892]: cluster 2023-10-09T09:16:50.146105+0000 mgr.y (mgr.25421) 1187 : cluster 0 pgmap v1294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 684 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 8336/32859 objects degraded (25.369%); 5066/32859 objects misplaced (15.417%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:16:51.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:51.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:51.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:51.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:51 smithi094 bash[59677]: cluster 2023-10-09T09:16:50.146105+0000 mgr.y (mgr.25421) 1187 : cluster 0 pgmap v1294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 684 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 8336/32859 objects degraded (25.369%); 5066/32859 objects misplaced (15.417%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:16:51.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:51.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:51.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:51.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:51.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:51.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:51.754 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:51.784 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:51.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:51.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:51.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:51.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:51.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:52.019 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:52.056 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:52.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:52.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:52.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:52.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:52.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:52.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:52.332 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:52.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:52.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:52.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:52.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:52.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:52.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:52.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:52.782 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:52.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:52.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:52.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:52.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:52.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:53.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:53.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:53.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:53.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:53.206 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:53.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:53.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:53.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:53.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:53.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:53.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:53.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:53.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:53 smithi094 bash[59677]: cluster 2023-10-09T09:16:52.146751+0000 mgr.y (mgr.25421) 1188 : cluster 0 pgmap v1295: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 546 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 8172/26256 objects degraded (31.124%); 4901/26256 objects misplaced (18.666%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:16:53.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:53.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:53.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:53.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:53.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:53 smithi016 bash[75179]: cluster 2023-10-09T09:16:52.146751+0000 mgr.y (mgr.25421) 1188 : cluster 0 pgmap v1295: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 546 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 8172/26256 objects degraded (31.124%); 4901/26256 objects misplaced (18.666%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:16:53.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:53 smithi016 bash[77892]: cluster 2023-10-09T09:16:52.146751+0000 mgr.y (mgr.25421) 1188 : cluster 0 pgmap v1295: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 546 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 8172/26256 objects degraded (31.124%); 4901/26256 objects misplaced (18.666%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:16:53.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:53.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:53.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:53.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:53.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:53.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:53.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:53.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:54.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:54.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:54.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:54.105 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:54.144 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:54.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:54.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:54.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:54.288 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:54.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:54.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:54.396 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:54.430 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:54.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:54.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:54.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:54.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:54.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:54.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:54.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:54.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:54.820 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:54.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:54.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:54.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:54.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:55.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:55.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:55.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:55.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:55.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:55.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:55.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:55.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:55.509 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:55.519 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:55 smithi094 bash[59677]: cluster 2023-10-09T09:16:54.147239+0000 mgr.y (mgr.25421) 1189 : cluster 0 pgmap v1296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 503 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 8172/24186 objects degraded (33.788%); 4546/24186 objects misplaced (18.796%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T09:16:55.520 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:55 smithi094 bash[59677]: cluster 2023-10-09T09:16:54.767016+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: Degraded data redundancy: 8172/24186 objects degraded (33.788%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:55.549 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:55.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:55.666 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:55 smithi016 bash[75179]: cluster 2023-10-09T09:16:54.147239+0000 mgr.y (mgr.25421) 1189 : cluster 0 pgmap v1296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 503 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 8172/24186 objects degraded (33.788%); 4546/24186 objects misplaced (18.796%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T09:16:55.666 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:55 smithi016 bash[75179]: cluster 2023-10-09T09:16:54.767016+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: Degraded data redundancy: 8172/24186 objects degraded (33.788%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:55.667 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:55 smithi016 bash[77892]: cluster 2023-10-09T09:16:54.147239+0000 mgr.y (mgr.25421) 1189 : cluster 0 pgmap v1296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 503 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 8172/24186 objects degraded (33.788%); 4546/24186 objects misplaced (18.796%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T09:16:55.667 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:55 smithi016 bash[77892]: cluster 2023-10-09T09:16:54.767016+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: Degraded data redundancy: 8172/24186 objects degraded (33.788%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:16:55.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:55.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:55.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:55.916 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:55.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:55.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:55.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:16:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:16:55] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T09:16:56.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:56.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:56.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:56.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:56.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:56.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:56.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:56.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:56.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:56.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:56.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:56.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:56.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:56.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:56.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:56.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:56.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:56.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:56.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:56.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:56.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:56.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:56.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:56.892 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:56.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:56.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:56.997 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:57.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:57.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:57.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:57.168 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:16:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:16:57.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:57.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:57.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:57.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:57.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:57.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:57 smithi016 bash[75179]: cluster 2023-10-09T09:16:56.148313+0000 mgr.y (mgr.25421) 1190 : cluster 0 pgmap v1297: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 375 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 7982/18027 objects degraded (44.278%); 4357/18027 objects misplaced (24.169%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:16:57.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:57 smithi016 bash[77892]: cluster 2023-10-09T09:16:56.148313+0000 mgr.y (mgr.25421) 1190 : cluster 0 pgmap v1297: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 375 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 7982/18027 objects degraded (44.278%); 4357/18027 objects misplaced (24.169%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:16:57.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:57.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:57.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:57.699 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:57.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:57 smithi094 bash[59677]: cluster 2023-10-09T09:16:56.148313+0000 mgr.y (mgr.25421) 1190 : cluster 0 pgmap v1297: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 375 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 7982/18027 objects degraded (44.278%); 4357/18027 objects misplaced (24.169%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:16:57.728 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:57.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:57.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:57.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:57.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:57.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:57.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:57.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:58.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:58.144 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:16:58.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:58.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:16:58.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:58.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:58.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:58.382 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:58.391 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:58 smithi094 bash[59677]: audit 2023-10-09T09:16:57.442556+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:16:58.391 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:58 smithi094 bash[59677]: audit 2023-10-09T09:16:57.920806+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:58.391 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:58 smithi094 bash[59677]: audit 2023-10-09T09:16:57.927256+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:58.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:16:58.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:58.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:16:58.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:58.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:58 smithi016 bash[75179]: audit 2023-10-09T09:16:57.442556+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:16:58.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:58 smithi016 bash[75179]: audit 2023-10-09T09:16:57.920806+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:58.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:58 smithi016 bash[75179]: audit 2023-10-09T09:16:57.927256+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:58.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:58 smithi016 bash[77892]: audit 2023-10-09T09:16:57.442556+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:16:58.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:58 smithi016 bash[77892]: audit 2023-10-09T09:16:57.920806+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:58.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:58 smithi016 bash[77892]: audit 2023-10-09T09:16:57.927256+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:16:58.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:16:58.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:58.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:16:58.653 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:58.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:58.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:58.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:58.854 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:58.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:58.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:58.960 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:58.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:59.101 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:59.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:59.232 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:59.270 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:59.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:59.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:59.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:59.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:16:59.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:59.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:16:59.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:16:59.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:59.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:16:59 smithi016 bash[75179]: cluster 2023-10-09T09:16:58.148918+0000 mgr.y (mgr.25421) 1191 : cluster 0 pgmap v1298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 329 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 7982/15840 objects degraded (50.391%); 4044/15840 objects misplaced (25.530%); 4.8 MiB/s, 79 objects/s recovering 2023-10-09T09:16:59.643 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:16:59 smithi016 bash[77892]: cluster 2023-10-09T09:16:58.148918+0000 mgr.y (mgr.25421) 1191 : cluster 0 pgmap v1298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 329 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 7982/15840 objects degraded (50.391%); 4044/15840 objects misplaced (25.530%); 4.8 MiB/s, 79 objects/s recovering 2023-10-09T09:16:59.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:16:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:16:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:16:59 smithi094 bash[59677]: cluster 2023-10-09T09:16:58.148918+0000 mgr.y (mgr.25421) 1191 : cluster 0 pgmap v1298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 329 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 7982/15840 objects degraded (50.391%); 4044/15840 objects misplaced (25.530%); 4.8 MiB/s, 79 objects/s recovering 2023-10-09T09:16:59.766 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:59.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:16:59.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:16:59.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:59.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:16:59.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:16:59.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:00.038 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:00.079 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:00.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:00.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:00.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:00.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:00.309 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:00.343 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:00.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:00.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:00.445 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:00.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:00.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:00.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:00.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:00 smithi016 bash[75179]: cluster 2023-10-09T09:16:59.767881+0000 mon.a (mon.0) 1589 : cluster 3 Health check update: Degraded data redundancy: 7982/15840 objects degraded (50.391%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:00.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:00 smithi016 bash[77892]: cluster 2023-10-09T09:16:59.767881+0000 mon.a (mon.0) 1589 : cluster 3 Health check update: Degraded data redundancy: 7982/15840 objects degraded (50.391%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:00.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:00.594 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:00 smithi094 bash[59677]: cluster 2023-10-09T09:16:59.767881+0000 mon.a (mon.0) 1589 : cluster 3 Health check update: Degraded data redundancy: 7982/15840 objects degraded (50.391%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:00.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:00.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:00.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:00.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:00.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:00.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:00.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:00.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:00.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:00.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:00.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:00.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:00.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:01.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:01.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:01.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:02.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:01 smithi016 bash[75179]: cluster 2023-10-09T09:17:00.149348+0000 mgr.y (mgr.25421) 1192 : cluster 0 pgmap v1299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 329 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 7982/15840 objects degraded (50.391%); 4044/15840 objects misplaced (25.530%); 3.7 MiB/s, 62 objects/s recovering 2023-10-09T09:17:02.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:01 smithi016 bash[77892]: cluster 2023-10-09T09:17:00.149348+0000 mgr.y (mgr.25421) 1192 : cluster 0 pgmap v1299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 329 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 7982/15840 objects degraded (50.391%); 4044/15840 objects misplaced (25.530%); 3.7 MiB/s, 62 objects/s recovering 2023-10-09T09:17:02.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:01 smithi094 bash[59677]: cluster 2023-10-09T09:17:00.149348+0000 mgr.y (mgr.25421) 1192 : cluster 0 pgmap v1299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 329 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 7982/15840 objects degraded (50.391%); 4044/15840 objects misplaced (25.530%); 3.7 MiB/s, 62 objects/s recovering 2023-10-09T09:17:02.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:02.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:02.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:02.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:02.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:02.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:02.686 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:02.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:02.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:02.865 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:03.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:03.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:03.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:03.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:03.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:03.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:03.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:03.334 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:03 smithi094 bash[59677]: cluster 2023-10-09T09:17:02.150553+0000 mgr.y (mgr.25421) 1193 : cluster 0 pgmap v1300: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 204 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 7807/9843 objects degraded (79.315%); 3869/9843 objects misplaced (39.307%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T09:17:03.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:03.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:03.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:03.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:03 smithi016 bash[75179]: cluster 2023-10-09T09:17:02.150553+0000 mgr.y (mgr.25421) 1193 : cluster 0 pgmap v1300: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 204 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 7807/9843 objects degraded (79.315%); 3869/9843 objects misplaced (39.307%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T09:17:03.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:03 smithi016 bash[77892]: cluster 2023-10-09T09:17:02.150553+0000 mgr.y (mgr.25421) 1193 : cluster 0 pgmap v1300: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 204 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 7807/9843 objects degraded (79.315%); 3869/9843 objects misplaced (39.307%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T09:17:03.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:03.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:03.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:03.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:03.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:03.816 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:03.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:03.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:03.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:03.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:04.080 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:04.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:04.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:04.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:04.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:04.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:04.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:04.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:04.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:04.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:04.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:04.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:04.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:04.625 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:04.634 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:04 smithi094 bash[59677]: audit 2023-10-09T09:17:04.251186+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:04.634 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:04 smithi094 bash[59677]: audit 2023-10-09T09:17:04.283116+0000 mon.a (mon.0) 1591 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:04.634 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:04 smithi094 bash[59677]: audit 2023-10-09T09:17:04.291729+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:04.653 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:04.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:04.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:04.724 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:04 smithi016 bash[77892]: audit 2023-10-09T09:17:04.251186+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:04.724 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:04 smithi016 bash[77892]: audit 2023-10-09T09:17:04.283116+0000 mon.a (mon.0) 1591 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:04.724 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:04 smithi016 bash[77892]: audit 2023-10-09T09:17:04.291729+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:04.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:04 smithi016 bash[75179]: audit 2023-10-09T09:17:04.251186+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:04.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:04 smithi016 bash[75179]: audit 2023-10-09T09:17:04.283116+0000 mon.a (mon.0) 1591 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:04.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:04 smithi016 bash[75179]: audit 2023-10-09T09:17:04.291729+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:04.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:04.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:04.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:04.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:04.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:04.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:04.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:05.061 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:05.096 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:05.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:05.206 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:05.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:05.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:05.344 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:05 smithi094 bash[59677]: cluster 2023-10-09T09:17:04.151546+0000 mgr.y (mgr.25421) 1194 : cluster 0 pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 163 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7807/7869 objects degraded (99.212%); 3722/7869 objects misplaced (47.300%); 3.7 MiB/s, 81 objects/s recovering 2023-10-09T09:17:05.345 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:05 smithi094 bash[59677]: cluster 2023-10-09T09:17:04.769177+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 7807/7869 objects degraded (99.212%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:05.345 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:05 smithi094 bash[59677]: audit 2023-10-09T09:17:05.305419+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.16:0/1754200405' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:05.345 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:05 smithi094 bash[59677]: audit 2023-10-09T09:17:05.305860+0000 mon.a (mon.0) 1594 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:05.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:05.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:05.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:05.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:05.479 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:05.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:05.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:05.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:05.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:05.626 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:17:05.644 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:05.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:05.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:05.659 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:17:05.695 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:17:05.716 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[75179]: cluster 2023-10-09T09:17:04.151546+0000 mgr.y (mgr.25421) 1194 : cluster 0 pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 163 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7807/7869 objects degraded (99.212%); 3722/7869 objects misplaced (47.300%); 3.7 MiB/s, 81 objects/s recovering 2023-10-09T09:17:05.716 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[75179]: cluster 2023-10-09T09:17:04.769177+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 7807/7869 objects degraded (99.212%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:05.716 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[75179]: audit 2023-10-09T09:17:05.305419+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.16:0/1754200405' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:05.716 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[75179]: audit 2023-10-09T09:17:05.305860+0000 mon.a (mon.0) 1594 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:05.716 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[77892]: cluster 2023-10-09T09:17:04.151546+0000 mgr.y (mgr.25421) 1194 : cluster 0 pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 163 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7807/7869 objects degraded (99.212%); 3722/7869 objects misplaced (47.300%); 3.7 MiB/s, 81 objects/s recovering 2023-10-09T09:17:05.717 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[77892]: cluster 2023-10-09T09:17:04.769177+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 7807/7869 objects degraded (99.212%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:05.717 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[77892]: audit 2023-10-09T09:17:05.305419+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.16:0/1754200405' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:05.717 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[77892]: audit 2023-10-09T09:17:05.305860+0000 mon.a (mon.0) 1594 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:05.743 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:17:05.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:05.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:05.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:05.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:05.788 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:17:05.789 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:17:05.827 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:17:05.868 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:17:05.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:05.903 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:17:05.907 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.908 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.908 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:17:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:05.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:05.972 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:17:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:17:05] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.33.4" 2023-10-09T09:17:06.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:06.039 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:06.061 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 107220 objects 2023-10-09T09:17:06.061 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :195.052 2023-10-09T09:17:06.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:06.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:06.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:06.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:06.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:06.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:06.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:06.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:06.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:06.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:06.555 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:06.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:06.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:06.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[75179]: audit 2023-10-09T09:17:05.571384+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.16:0/3942475076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:06.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[75179]: cluster 2023-10-09T09:17:05.771919+0000 mon.a (mon.0) 1595 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:17:06.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[75179]: audit 2023-10-09T09:17:05.779215+0000 mon.a (mon.0) 1596 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:17:06.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[75179]: cluster 2023-10-09T09:17:05.779488+0000 mon.a (mon.0) 1597 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T09:17:06.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[75179]: cluster 2023-10-09T09:17:05.787868+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T09:17:06.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[77892]: audit 2023-10-09T09:17:05.571384+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.16:0/3942475076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:06.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[77892]: cluster 2023-10-09T09:17:05.771919+0000 mon.a (mon.0) 1595 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:17:06.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[77892]: audit 2023-10-09T09:17:05.779215+0000 mon.a (mon.0) 1596 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:17:06.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[77892]: cluster 2023-10-09T09:17:05.779488+0000 mon.a (mon.0) 1597 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T09:17:06.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:06 smithi016 bash[77892]: cluster 2023-10-09T09:17:05.787868+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T09:17:06.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:06.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:06.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:06 smithi094 bash[59677]: audit 2023-10-09T09:17:05.571384+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.16:0/3942475076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:06 smithi094 bash[59677]: cluster 2023-10-09T09:17:05.771919+0000 mon.a (mon.0) 1595 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:17:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:06 smithi094 bash[59677]: audit 2023-10-09T09:17:05.779215+0000 mon.a (mon.0) 1596 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:17:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:06 smithi094 bash[59677]: cluster 2023-10-09T09:17:05.779488+0000 mon.a (mon.0) 1597 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T09:17:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:06 smithi094 bash[59677]: cluster 2023-10-09T09:17:05.787868+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T09:17:06.820 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:06.847 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:06.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:06.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:06.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:06.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:07.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:07.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:07.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:07.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:07.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:07.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:07.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:07.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:07.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:07.371 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:07.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:07.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:07.410 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:17:07.410 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:17:07.475 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:07.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:07 smithi094 bash[59677]: cluster 2023-10-09T09:17:05.779829+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T09:17:07.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:07 smithi094 bash[59677]: cluster 2023-10-09T09:17:06.152119+0000 mgr.y (mgr.25421) 1195 : cluster 0 pgmap v1303: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 41 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 7637/2034 objects degraded (375.467%); 3552/2034 objects misplaced (174.631%); 3.1 MiB/s, 76 objects/s recovering 2023-10-09T09:17:07.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:07 smithi094 bash[59677]: audit 2023-10-09T09:17:06.929472+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:07.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:07 smithi094 bash[59677]: audit 2023-10-09T09:17:06.935374+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:07.499 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:07.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[75179]: cluster 2023-10-09T09:17:05.779829+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T09:17:07.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[75179]: cluster 2023-10-09T09:17:06.152119+0000 mgr.y (mgr.25421) 1195 : cluster 0 pgmap v1303: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 41 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 7637/2034 objects degraded (375.467%); 3552/2034 objects misplaced (174.631%); 3.1 MiB/s, 76 objects/s recovering 2023-10-09T09:17:07.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[75179]: audit 2023-10-09T09:17:06.929472+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:07.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[75179]: audit 2023-10-09T09:17:06.935374+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:07.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[77892]: cluster 2023-10-09T09:17:05.779829+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T09:17:07.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[77892]: cluster 2023-10-09T09:17:06.152119+0000 mgr.y (mgr.25421) 1195 : cluster 0 pgmap v1303: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 41 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 7637/2034 objects degraded (375.467%); 3552/2034 objects misplaced (174.631%); 3.1 MiB/s, 76 objects/s recovering 2023-10-09T09:17:07.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[77892]: audit 2023-10-09T09:17:06.929472+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:07.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:07 smithi016 bash[77892]: audit 2023-10-09T09:17:06.935374+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:07.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:07.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:07.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:07.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:07.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:07.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:07.758 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:07.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:07.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:07.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:07.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:07.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:07.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:08.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:08.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:08.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:08.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:08.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:08.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:08.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:08.382 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:08.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:08.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:08.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:08 smithi016 bash[77892]: audit 2023-10-09T09:17:07.405018+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:17:08.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:08 smithi016 bash[77892]: audit 2023-10-09T09:17:07.405975+0000 mon.a (mon.0) 1601 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:17:08.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:08 smithi016 bash[77892]: audit 2023-10-09T09:17:07.412018+0000 mon.a (mon.0) 1602 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:08.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:08 smithi016 bash[75179]: audit 2023-10-09T09:17:07.405018+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:17:08.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:08 smithi016 bash[75179]: audit 2023-10-09T09:17:07.405975+0000 mon.a (mon.0) 1601 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:17:08.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:08 smithi016 bash[75179]: audit 2023-10-09T09:17:07.412018+0000 mon.a (mon.0) 1602 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:08.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:08.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:08.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:08.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:08 smithi094 bash[59677]: audit 2023-10-09T09:17:07.405018+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:17:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:08 smithi094 bash[59677]: audit 2023-10-09T09:17:07.405975+0000 mon.a (mon.0) 1601 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:17:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:08 smithi094 bash[59677]: audit 2023-10-09T09:17:07.412018+0000 mon.a (mon.0) 1602 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:17:08.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:08.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:08.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:08.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:08.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:08.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:09.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:09.067 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:09.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:09.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:09.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:09.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:09.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:09.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:09.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:09.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:09.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:09.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:09.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:09 smithi016 bash[75179]: cluster 2023-10-09T09:17:08.152680+0000 mgr.y (mgr.25421) 1196 : cluster 0 pgmap v1304: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 12 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 7637/606 objects degraded (1260.231%); 3435/606 objects misplaced (566.832%); 2.2 MiB/s, 105 objects/s recovering 2023-10-09T09:17:09.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:09 smithi016 bash[77892]: cluster 2023-10-09T09:17:08.152680+0000 mgr.y (mgr.25421) 1196 : cluster 0 pgmap v1304: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 12 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 7637/606 objects degraded (1260.231%); 3435/606 objects misplaced (566.832%); 2.2 MiB/s, 105 objects/s recovering 2023-10-09T09:17:09.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:09.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:09.675 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:17:09.682 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:17:09.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:09.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:09.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:09 smithi094 bash[59677]: cluster 2023-10-09T09:17:08.152680+0000 mgr.y (mgr.25421) 1196 : cluster 0 pgmap v1304: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 12 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 7637/606 objects degraded (1260.231%); 3435/606 objects misplaced (566.832%); 2.2 MiB/s, 105 objects/s recovering 2023-10-09T09:17:09.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:09.844 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:09.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:09.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:09.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:10.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:10.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:10.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:10.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:10.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:10.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:10.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:10.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:10.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:10.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:10.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:10.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:10.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:10.613 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:10.621 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:10 smithi016 bash[75179]: audit 2023-10-09T09:17:09.656364+0000 mgr.y (mgr.25421) 1197 : audit 0 from='client.35784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:17:10.621 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:10 smithi016 bash[75179]: cluster 2023-10-09T09:17:09.770701+0000 mon.a (mon.0) 1603 : cluster 3 Health check update: Degraded data redundancy: 7637/606 objects degraded (1260.231%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:10.622 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:10 smithi016 bash[77892]: audit 2023-10-09T09:17:09.656364+0000 mgr.y (mgr.25421) 1197 : audit 0 from='client.35784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:17:10.622 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:10 smithi016 bash[77892]: cluster 2023-10-09T09:17:09.770701+0000 mon.a (mon.0) 1603 : cluster 3 Health check update: Degraded data redundancy: 7637/606 objects degraded (1260.231%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:10.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:10.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:10.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:10.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:10 smithi094 bash[59677]: audit 2023-10-09T09:17:09.656364+0000 mgr.y (mgr.25421) 1197 : audit 0 from='client.35784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:17:10.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:10 smithi094 bash[59677]: cluster 2023-10-09T09:17:09.770701+0000 mon.a (mon.0) 1603 : cluster 3 Health check update: Degraded data redundancy: 7637/606 objects degraded (1260.231%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:10.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:10.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:10.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:10.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:10.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:10.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:10.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:11.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:11.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:11.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:11.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:11.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:11.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:11.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:11.410 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:11.437 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:11.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:11.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:11.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:11.686 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:11 smithi094 bash[59677]: cluster 2023-10-09T09:17:10.153355+0000 mgr.y (mgr.25421) 1198 : cluster 0 pgmap v1305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 12 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 7637/606 objects degraded (1260.231%); 3435/606 objects misplaced (566.832%); 2.2 MiB/s, 105 objects/s recovering 2023-10-09T09:17:11.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:11 smithi016 bash[75179]: cluster 2023-10-09T09:17:10.153355+0000 mgr.y (mgr.25421) 1198 : cluster 0 pgmap v1305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 12 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 7637/606 objects degraded (1260.231%); 3435/606 objects misplaced (566.832%); 2.2 MiB/s, 105 objects/s recovering 2023-10-09T09:17:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:11 smithi016 bash[77892]: cluster 2023-10-09T09:17:10.153355+0000 mgr.y (mgr.25421) 1198 : cluster 0 pgmap v1305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 12 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 7637/606 objects degraded (1260.231%); 3435/606 objects misplaced (566.832%); 2.2 MiB/s, 105 objects/s recovering 2023-10-09T09:17:11.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:11.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:11.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:11.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:11.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:11.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:12.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:12.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:12.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:12.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:12.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:12.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:12.460 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:12.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:12.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:12.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:12.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:12.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:12.724 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:12.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:12.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:12.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:12.892 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:12.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:12.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:13.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:13.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:13.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:13.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:13.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:13.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:13.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:13.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:13.463 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:13.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:13.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:13.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:13.655 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:13 smithi016 bash[75179]: cluster 2023-10-09T09:17:12.154531+0000 mgr.y (mgr.25421) 1199 : cluster 0 pgmap v1306: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 7620/72 objects degraded (10583.333%); 3418/72 objects misplaced (4747.222%); 1.2 MiB/s, 90 objects/s recovering 2023-10-09T09:17:13.656 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:13 smithi016 bash[77892]: cluster 2023-10-09T09:17:12.154531+0000 mgr.y (mgr.25421) 1199 : cluster 0 pgmap v1306: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 7620/72 objects degraded (10583.333%); 3418/72 objects misplaced (4747.222%); 1.2 MiB/s, 90 objects/s recovering 2023-10-09T09:17:13.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:17:13.692 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:13 smithi094 bash[59677]: cluster 2023-10-09T09:17:12.154531+0000 mgr.y (mgr.25421) 1199 : cluster 0 pgmap v1306: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 7620/72 objects degraded (10583.333%); 3418/72 objects misplaced (4747.222%); 1.2 MiB/s, 90 objects/s recovering 2023-10-09T09:17:13.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:13.708 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1304,"stamp":"2023-10-09T09:17:08.152281+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":12195768,"num_objects":202,"num_object_clones":0,"num_object_copies":606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7637,"num_objects_misplaced":3435,"num_objects_unfound":0,"num_objects_dirty":202,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":214524,"num_write_kb":6864526,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27487,"num_bytes_recovered":1748325798,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123986,"ondisk_log_size":123986,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5590140,"kb_used_data":402116,"kb_used_omap":44,"kb_used_meta":5187859,"kb_avail":650625924,"statfs":{"total":671965249536,"available":666240946176,"internally_reserved":0,"allocated":411766784,"data_stored":406300165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":5312368033},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,2,5,4,4,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-332791808,"num_objects":-5078,"num_object_clones":0,"num_object_copies":-15234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-345,"num_objects_misplaced":-609,"num_objects_unfound":0,"num_objects_dirty":-5078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5078,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1059,"num_bytes_recovered":22609920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003803"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":166,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280331+0000","last_change":"2023-10-09T09:14:48.320488+0000","last_active":"2023-10-09T09:16:35.280331+0000","last_peered":"2023-10-09T09:16:35.280331+0000","last_clean":"2023-10-09T09:16:35.280331+0000","last_became_active":"2023-10-09T09:14:48.320104+0000","last_became_peered":"2023-10-09T09:14:48.320104+0000","last_unstale":"2023-10-09T09:16:35.280331+0000","last_undegraded":"2023-10-09T09:16:35.280331+0000","last_fullsized":"2023-10-09T09:16:35.280331+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.268234+0000","last_change":"2023-10-09T09:14:48.316760+0000","last_active":"2023-10-09T09:16:35.268234+0000","last_peered":"2023-10-09T09:16:35.268234+0000","last_clean":"2023-10-09T09:16:35.268234+0000","last_became_active":"2023-10-09T09:14:48.316432+0000","last_became_peered":"2023-10-09T09:14:48.316432+0000","last_unstale":"2023-10-09T09:16:35.268234+0000","last_undegraded":"2023-10-09T09:16:35.268234+0000","last_fullsized":"2023-10-09T09:16:35.268234+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:41.630374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.267875+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:16:35.267875+0000","last_peered":"2023-10-09T09:16:35.267875+0000","last_clean":"2023-10-09T09:16:35.267875+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:16:35.267875+0000","last_undegraded":"2023-10-09T09:16:35.267875+0000","last_fullsized":"2023-10-09T09:16:35.267875+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":321,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.277931+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:16:35.277931+0000","last_peered":"2023-10-09T09:16:35.277931+0000","last_clean":"2023-10-09T09:16:35.277931+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:16:35.277931+0000","last_undegraded":"2023-10-09T09:16:35.277931+0000","last_fullsized":"2023-10-09T09:16:35.277931+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":265,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.271069+0000","last_change":"2023-10-09T09:14:48.312713+0000","last_active":"2023-10-09T09:16:35.271069+0000","last_peered":"2023-10-09T09:16:35.271069+0000","last_clean":"2023-10-09T09:16:35.271069+0000","last_became_active":"2023-10-09T09:14:48.312429+0000","last_became_peered":"2023-10-09T09:14:48.312429+0000","last_unstale":"2023-10-09T09:16:35.271069+0000","last_undegraded":"2023-10-09T09:16:35.271069+0000","last_fullsized":"2023-10-09T09:16:35.271069+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:56.755105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":179,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.337730+0000","last_change":"2023-10-09T09:14:48.310212+0000","last_active":"2023-10-09T09:16:28.337730+0000","last_peered":"2023-10-09T09:16:28.337730+0000","last_clean":"2023-10-09T09:16:28.337730+0000","last_became_active":"2023-10-09T09:14:48.309682+0000","last_became_peered":"2023-10-09T09:14:48.309682+0000","last_unstale":"2023-10-09T09:16:28.337730+0000","last_undegraded":"2023-10-09T09:16:28.337730+0000","last_fullsized":"2023-10-09T09:16:28.337730+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":273,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.275160+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:16:35.275160+0000","last_peered":"2023-10-09T09:16:35.275160+0000","last_clean":"2023-10-09T09:16:35.275160+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:16:35.275160+0000","last_undegraded":"2023-10-09T09:16:35.275160+0000","last_fullsized":"2023-10-09T09:16:35.275160+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.265643+0000","last_change":"2023-10-09T09:14:48.306040+0000","last_active":"2023-10-09T09:16:35.265643+0000","last_peered":"2023-10-09T09:16:35.265643+0000","last_clean":"2023-10-09T09:16:35.265643+0000","last_became_active":"2023-10-09T09:14:48.304137+0000","last_became_peered":"2023-10-09T09:14:48.304137+0000","last_unstale":"2023-10-09T09:16:35.265643+0000","last_undegraded":"2023-10-09T09:16:35.265643+0000","last_fullsized":"2023-10-09T09:16:35.265643+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:47.446761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":90,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.351656+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:16:28.351656+0000","last_peered":"2023-10-09T09:16:28.351656+0000","last_clean":"2023-10-09T09:16:28.351656+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:16:28.351656+0000","last_undegraded":"2023-10-09T09:16:28.351656+0000","last_fullsized":"2023-10-09T09:16:28.351656+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":248,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.334213+0000","last_change":"2023-10-09T09:13:48.190068+0000","last_active":"2023-10-09T09:16:28.334213+0000","last_peered":"2023-10-09T09:16:28.334213+0000","last_clean":"2023-10-09T09:16:28.334213+0000","last_became_active":"2023-10-09T09:13:48.189740+0000","last_became_peered":"2023-10-09T09:13:48.189740+0000","last_unstale":"2023-10-09T09:16:28.334213+0000","last_undegraded":"2023-10-09T09:16:28.334213+0000","last_fullsized":"2023-10-09T09:16:28.334213+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":71,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.359258+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:16:28.359258+0000","last_peered":"2023-10-09T09:16:28.359258+0000","last_clean":"2023-10-09T09:16:28.359258+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:16:28.359258+0000","last_undegraded":"2023-10-09T09:16:28.359258+0000","last_fullsized":"2023-10-09T09:16:28.359258+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.268238+0000","last_change":"2023-10-09T09:14:48.303627+0000","last_active":"2023-10-09T09:16:35.268238+0000","last_peered":"2023-10-09T09:16:35.268238+0000","last_clean":"2023-10-09T09:16:35.268238+0000","last_became_active":"2023-10-09T09:14:48.303287+0000","last_became_peered":"2023-10-09T09:14:48.303287+0000","last_unstale":"2023-10-09T09:16:35.268238+0000","last_undegraded":"2023-10-09T09:16:35.268238+0000","last_fullsized":"2023-10-09T09:16:35.268238+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:27.004752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":187,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.279073+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:16:35.279073+0000","last_peered":"2023-10-09T09:16:35.279073+0000","last_clean":"2023-10-09T09:16:35.279073+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:16:35.279073+0000","last_undegraded":"2023-10-09T09:16:35.279073+0000","last_fullsized":"2023-10-09T09:16:35.279073+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":175,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280782+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:16:35.280782+0000","last_peered":"2023-10-09T09:16:35.280782+0000","last_clean":"2023-10-09T09:16:35.280782+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:16:35.280782+0000","last_undegraded":"2023-10-09T09:16:35.280782+0000","last_fullsized":"2023-10-09T09:16:35.280782+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":86,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.358534+0000","last_change":"2023-10-09T09:14:19.878196+0000","last_active":"2023-10-09T09:16:28.358534+0000","last_peered":"2023-10-09T09:16:28.358534+0000","last_clean":"2023-10-09T09:16:28.358534+0000","last_became_active":"2023-10-09T09:14:19.750467+0000","last_became_peered":"2023-10-09T09:14:19.750467+0000","last_unstale":"2023-10-09T09:16:28.358534+0000","last_undegraded":"2023-10-09T09:16:28.358534+0000","last_fullsized":"2023-10-09T09:16:28.358534+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.268547+0000","last_change":"2023-10-09T09:14:48.319108+0000","last_active":"2023-10-09T09:16:35.268547+0000","last_peered":"2023-10-09T09:16:35.268547+0000","last_clean":"2023-10-09T09:16:35.268547+0000","last_became_active":"2023-10-09T09:14:48.318443+0000","last_became_peered":"2023-10-09T09:14:48.318443+0000","last_unstale":"2023-10-09T09:16:35.268547+0000","last_undegraded":"2023-10-09T09:16:35.268547+0000","last_fullsized":"2023-10-09T09:16:35.268547+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:31.939020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":301,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.271473+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:16:35.271473+0000","last_peered":"2023-10-09T09:16:35.271473+0000","last_clean":"2023-10-09T09:16:35.271473+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:16:35.271473+0000","last_undegraded":"2023-10-09T09:16:35.271473+0000","last_fullsized":"2023-10-09T09:16:35.271473+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":84,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.359012+0000","last_change":"2023-10-09T09:15:23.415336+0000","last_active":"2023-10-09T09:16:28.359012+0000","last_peered":"2023-10-09T09:16:28.359012+0000","last_clean":"2023-10-09T09:16:28.359012+0000","last_became_active":"2023-10-09T09:15:23.414900+0000","last_became_peered":"2023-10-09T09:15:23.414900+0000","last_unstale":"2023-10-09T09:16:28.359012+0000","last_undegraded":"2023-10-09T09:16:28.359012+0000","last_fullsized":"2023-10-09T09:16:28.359012+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":129,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280164+0000","last_change":"2023-10-09T09:13:48.205269+0000","last_active":"2023-10-09T09:16:35.280164+0000","last_peered":"2023-10-09T09:16:35.280164+0000","last_clean":"2023-10-09T09:16:35.280164+0000","last_became_active":"2023-10-09T09:13:48.204894+0000","last_became_peered":"2023-10-09T09:13:48.204894+0000","last_unstale":"2023-10-09T09:16:35.280164+0000","last_undegraded":"2023-10-09T09:16:35.280164+0000","last_fullsized":"2023-10-09T09:16:35.280164+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":273,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.278002+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:16:35.278002+0000","last_peered":"2023-10-09T09:16:35.278002+0000","last_clean":"2023-10-09T09:16:35.278002+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:16:35.278002+0000","last_undegraded":"2023-10-09T09:16:35.278002+0000","last_fullsized":"2023-10-09T09:16:35.278002+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":201,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.333709+0000","last_change":"2023-10-09T09:15:23.420333+0000","last_active":"2023-10-09T09:16:28.333709+0000","last_peered":"2023-10-09T09:16:28.333709+0000","last_clean":"2023-10-09T09:16:28.333709+0000","last_became_active":"2023-10-09T09:15:23.412581+0000","last_became_peered":"2023-10-09T09:15:23.412581+0000","last_unstale":"2023-10-09T09:16:28.333709+0000","last_undegraded":"2023-10-09T09:16:28.333709+0000","last_fullsized":"2023-10-09T09:16:28.333709+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.268738+0000","last_change":"2023-10-09T09:14:48.318889+0000","last_active":"2023-10-09T09:16:35.268738+0000","last_peered":"2023-10-09T09:16:35.268738+0000","last_clean":"2023-10-09T09:16:35.268738+0000","last_became_active":"2023-10-09T09:14:48.318575+0000","last_became_peered":"2023-10-09T09:14:48.318575+0000","last_unstale":"2023-10-09T09:16:35.268738+0000","last_undegraded":"2023-10-09T09:16:35.268738+0000","last_fullsized":"2023-10-09T09:16:35.268738+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:50.182686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"285'13469","reported_seq":13514,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.515654+0000","last_change":"2023-10-09T09:12:00.109899+0000","last_active":"2023-10-09T09:17:05.515654+0000","last_peered":"2023-10-09T09:17:05.515654+0000","last_clean":"2023-10-09T09:17:05.515654+0000","last_became_active":"2023-10-09T09:12:00.109613+0000","last_became_peered":"2023-10-09T09:12:00.109613+0000","last_unstale":"2023-10-09T09:17:05.515654+0000","last_undegraded":"2023-10-09T09:17:05.515654+0000","last_fullsized":"2023-10-09T09:17:05.515654+0000","mapping_epoch":248,"log_start":"253'3400","ondisk_log_start":"253'3400","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":10069,"log_dups_size":0,"ondisk_log_size":10069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13469,"num_write_kb":431552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":82,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.359182+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:16:28.359182+0000","last_peered":"2023-10-09T09:16:28.359182+0000","last_clean":"2023-10-09T09:16:28.359182+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:16:28.359182+0000","last_undegraded":"2023-10-09T09:16:28.359182+0000","last_fullsized":"2023-10-09T09:16:28.359182+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":44,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.358526+0000","last_change":"2023-10-09T09:15:23.414649+0000","last_active":"2023-10-09T09:16:28.358526+0000","last_peered":"2023-10-09T09:16:28.358526+0000","last_clean":"2023-10-09T09:16:28.358526+0000","last_became_active":"2023-10-09T09:15:23.414272+0000","last_became_peered":"2023-10-09T09:15:23.414272+0000","last_unstale":"2023-10-09T09:16:28.358526+0000","last_undegraded":"2023-10-09T09:16:28.358526+0000","last_fullsized":"2023-10-09T09:16:28.358526+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"285'13522","reported_seq":18078,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.491698+0000","last_change":"2023-10-09T09:16:19.372168+0000","last_active":"2023-10-09T09:17:05.491698+0000","last_peered":"2023-10-09T09:17:05.491698+0000","last_clean":"2023-10-09T09:17:05.491698+0000","last_became_active":"2023-10-09T09:16:19.310639+0000","last_became_peered":"2023-10-09T09:16:19.310639+0000","last_unstale":"2023-10-09T09:17:05.491698+0000","last_undegraded":"2023-10-09T09:17:05.491698+0000","last_fullsized":"2023-10-09T09:17:05.491698+0000","mapping_epoch":282,"log_start":"255'5969","ondisk_log_start":"255'5969","created":248,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1703936,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13522,"num_write_kb":433536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2244,"num_bytes_recovered":147062784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":82,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.359192+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:16:28.359192+0000","last_peered":"2023-10-09T09:16:28.359192+0000","last_clean":"2023-10-09T09:16:28.359192+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:16:28.359192+0000","last_undegraded":"2023-10-09T09:16:28.359192+0000","last_fullsized":"2023-10-09T09:16:28.359192+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":175,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280010+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:16:35.280010+0000","last_peered":"2023-10-09T09:16:35.280010+0000","last_clean":"2023-10-09T09:16:35.280010+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:16:35.280010+0000","last_undegraded":"2023-10-09T09:16:35.280010+0000","last_fullsized":"2023-10-09T09:16:35.280010+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"286'13076","reported_seq":13125,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T09:17:06.033145+0000","last_change":"2023-10-09T09:12:00.136187+0000","last_active":"2023-10-09T09:17:06.033145+0000","last_peered":"2023-10-09T09:17:06.033145+0000","last_clean":"2023-10-09T09:17:06.033145+0000","last_became_active":"2023-10-09T09:12:00.135851+0000","last_became_peered":"2023-10-09T09:12:00.135851+0000","last_unstale":"2023-10-09T09:17:06.033145+0000","last_undegraded":"2023-10-09T09:17:06.033145+0000","last_fullsized":"2023-10-09T09:17:06.033145+0000","mapping_epoch":248,"log_start":"256'5867","ondisk_log_start":"256'5867","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:25.402310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13076,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":158,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.337288+0000","last_change":"2023-10-09T09:15:23.437420+0000","last_active":"2023-10-09T09:16:28.337288+0000","last_peered":"2023-10-09T09:16:28.337288+0000","last_clean":"2023-10-09T09:16:28.337288+0000","last_became_active":"2023-10-09T09:15:23.429358+0000","last_became_peered":"2023-10-09T09:15:23.429358+0000","last_unstale":"2023-10-09T09:16:28.337288+0000","last_undegraded":"2023-10-09T09:16:28.337288+0000","last_fullsized":"2023-10-09T09:16:28.337288+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":188,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.339418+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:16:28.339418+0000","last_peered":"2023-10-09T09:16:28.339418+0000","last_clean":"2023-10-09T09:16:28.339418+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:16:28.339418+0000","last_undegraded":"2023-10-09T09:16:28.339418+0000","last_fullsized":"2023-10-09T09:16:28.339418+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"286'13408","reported_seq":23025,"reported_epoch":286,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T09:17:07.304967+0000","last_change":"2023-10-09T09:15:23.438134+0000","last_active":"2023-10-09T09:17:07.304967+0000","last_peered":"2023-10-09T09:17:07.304967+0000","last_clean":"2023-10-09T09:14:19.167594+0000","last_became_active":"2023-10-09T09:14:21.157751+0000","last_became_peered":"2023-10-09T09:14:21.157751+0000","last_unstale":"2023-10-09T09:17:07.304967+0000","last_undegraded":"2023-10-09T09:14:20.782662+0000","last_fullsized":"2023-10-09T09:14:20.728852+0000","mapping_epoch":261,"log_start":"256'6228","ondisk_log_start":"256'6228","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7180,"log_dups_size":0,"ondisk_log_size":7180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:23.261828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1268,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13408,"num_write_kb":429056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4779,"num_bytes_recovered":263716864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,3","objects":-1268}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.349589+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:16:28.349589+0000","last_peered":"2023-10-09T09:16:28.349589+0000","last_clean":"2023-10-09T09:16:28.349589+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:16:28.349589+0000","last_undegraded":"2023-10-09T09:16:28.349589+0000","last_fullsized":"2023-10-09T09:16:28.349589+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":292,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.272249+0000","last_change":"2023-10-09T09:14:48.302231+0000","last_active":"2023-10-09T09:16:35.272249+0000","last_peered":"2023-10-09T09:16:35.272249+0000","last_clean":"2023-10-09T09:16:35.272249+0000","last_became_active":"2023-10-09T09:14:48.301767+0000","last_became_peered":"2023-10-09T09:14:48.301767+0000","last_unstale":"2023-10-09T09:16:35.272249+0000","last_undegraded":"2023-10-09T09:16:35.272249+0000","last_fullsized":"2023-10-09T09:16:35.272249+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:55.523804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"286'13286","reported_seq":13365,"reported_epoch":286,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:17:06.018591+0000","last_change":"2023-10-09T09:15:23.424918+0000","last_active":"2023-10-09T09:17:06.018591+0000","last_peered":"2023-10-09T09:17:06.018591+0000","last_clean":"2023-10-09T09:14:47.209721+0000","last_became_active":"2023-10-09T09:14:49.596339+0000","last_became_peered":"2023-10-09T09:14:49.596339+0000","last_unstale":"2023-10-09T09:17:06.018591+0000","last_undegraded":"2023-10-09T09:17:06.018591+0000","last_fullsized":"2023-10-09T09:17:06.018591+0000","mapping_epoch":265,"log_start":"256'6055","ondisk_log_start":"256'6055","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7231,"log_dups_size":0,"ondisk_log_size":7231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:09:16.272858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13286,"num_write_kb":425152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":524288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4,3],"avail_no_missing":["0","1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":95,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.272363+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:16:35.272363+0000","last_peered":"2023-10-09T09:16:35.272363+0000","last_clean":"2023-10-09T09:16:35.272363+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:16:35.272363+0000","last_undegraded":"2023-10-09T09:16:35.272363+0000","last_fullsized":"2023-10-09T09:16:35.272363+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":148,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.278908+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:16:35.278908+0000","last_peered":"2023-10-09T09:16:35.278908+0000","last_clean":"2023-10-09T09:16:35.278908+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:16:35.278908+0000","last_undegraded":"2023-10-09T09:16:35.278908+0000","last_fullsized":"2023-10-09T09:16:35.278908+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"285'13439","reported_seq":14257,"reported_epoch":285,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:17:05.431533+0000","last_change":"2023-10-09T09:15:23.440414+0000","last_active":"2023-10-09T09:17:05.431533+0000","last_peered":"2023-10-09T09:17:05.431533+0000","last_clean":"2023-10-09T09:14:19.147674+0000","last_became_active":"2023-10-09T09:14:21.157141+0000","last_became_peered":"2023-10-09T09:14:21.157141+0000","last_unstale":"2023-10-09T09:17:05.431533+0000","last_undegraded":"2023-10-09T09:14:20.802763+0000","last_fullsized":"2023-10-09T09:14:20.742865+0000","mapping_epoch":261,"log_start":"253'3400","ondisk_log_start":"253'3400","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6352,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13439,"num_write_kb":430848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":380,"num_bytes_recovered":24838144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":25},{"shards":"3,5","objects":-4965}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":152,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.339344+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:16:28.339344+0000","last_peered":"2023-10-09T09:16:28.339344+0000","last_clean":"2023-10-09T09:16:28.339344+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:16:28.339344+0000","last_undegraded":"2023-10-09T09:16:28.339344+0000","last_fullsized":"2023-10-09T09:16:28.339344+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":166,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.279374+0000","last_change":"2023-10-09T09:14:48.318228+0000","last_active":"2023-10-09T09:16:35.279374+0000","last_peered":"2023-10-09T09:16:35.279374+0000","last_clean":"2023-10-09T09:16:35.279374+0000","last_became_active":"2023-10-09T09:14:48.317743+0000","last_became_peered":"2023-10-09T09:14:48.317743+0000","last_unstale":"2023-10-09T09:16:35.279374+0000","last_undegraded":"2023-10-09T09:16:35.279374+0000","last_fullsized":"2023-10-09T09:16:35.279374+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"285'13549","reported_seq":19895,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.496599+0000","last_change":"2023-10-09T09:15:40.814591+0000","last_active":"2023-10-09T09:17:05.496599+0000","last_peered":"2023-10-09T09:17:05.496599+0000","last_clean":"2023-10-09T09:17:05.496599+0000","last_became_active":"2023-10-09T09:15:40.776768+0000","last_became_peered":"2023-10-09T09:15:40.776768+0000","last_unstale":"2023-10-09T09:17:05.496599+0000","last_undegraded":"2023-10-09T09:17:05.496599+0000","last_fullsized":"2023-10-09T09:17:05.496599+0000","mapping_epoch":275,"log_start":"256'6015","ondisk_log_start":"256'6015","created":248,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7534,"log_dups_size":0,"ondisk_log_size":7534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":720896,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13549,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3138,"num_bytes_recovered":205586432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":193,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.279928+0000","last_change":"2023-10-09T09:14:48.321196+0000","last_active":"2023-10-09T09:16:35.279928+0000","last_peered":"2023-10-09T09:16:35.279928+0000","last_clean":"2023-10-09T09:16:35.279928+0000","last_became_active":"2023-10-09T09:14:48.318537+0000","last_became_peered":"2023-10-09T09:14:48.318537+0000","last_unstale":"2023-10-09T09:16:35.279928+0000","last_undegraded":"2023-10-09T09:16:35.279928+0000","last_fullsized":"2023-10-09T09:16:35.279928+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":166,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.297609+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:16:35.297609+0000","last_peered":"2023-10-09T09:16:35.297609+0000","last_clean":"2023-10-09T09:16:35.297609+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:16:35.297609+0000","last_undegraded":"2023-10-09T09:16:35.297609+0000","last_fullsized":"2023-10-09T09:16:35.297609+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"286'13440","reported_seq":13486,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.992686+0000","last_change":"2023-10-09T09:12:00.116690+0000","last_active":"2023-10-09T09:17:05.992686+0000","last_peered":"2023-10-09T09:17:05.992686+0000","last_clean":"2023-10-09T09:17:05.992686+0000","last_became_active":"2023-10-09T09:12:00.116303+0000","last_became_peered":"2023-10-09T09:12:00.116303+0000","last_unstale":"2023-10-09T09:17:05.992686+0000","last_undegraded":"2023-10-09T09:17:05.992686+0000","last_fullsized":"2023-10-09T09:17:05.992686+0000","mapping_epoch":248,"log_start":"258'7027","ondisk_log_start":"258'7027","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":6413,"log_dups_size":0,"ondisk_log_size":6413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13440,"num_write_kb":430080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:01.680610+0000","last_change":"2023-10-09T09:14:48.306639+0000","last_active":"2023-10-09T09:17:01.680610+0000","last_peered":"2023-10-09T09:17:01.680610+0000","last_clean":"2023-10-09T09:17:01.680610+0000","last_became_active":"2023-10-09T09:14:48.305729+0000","last_became_peered":"2023-10-09T09:14:48.305729+0000","last_unstale":"2023-10-09T09:17:01.680610+0000","last_undegraded":"2023-10-09T09:17:01.680610+0000","last_fullsized":"2023-10-09T09:17:01.680610+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:08.892989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.268830+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:16:35.268830+0000","last_peered":"2023-10-09T09:16:35.268830+0000","last_clean":"2023-10-09T09:16:35.268830+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:16:35.268830+0000","last_undegraded":"2023-10-09T09:16:35.268830+0000","last_fullsized":"2023-10-09T09:16:35.268830+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"285'13373","reported_seq":13418,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.393152+0000","last_change":"2023-10-09T09:12:00.116375+0000","last_active":"2023-10-09T09:17:05.393152+0000","last_peered":"2023-10-09T09:17:05.393152+0000","last_clean":"2023-10-09T09:17:05.393152+0000","last_became_active":"2023-10-09T09:12:00.115851+0000","last_became_peered":"2023-10-09T09:12:00.115851+0000","last_unstale":"2023-10-09T09:17:05.393152+0000","last_undegraded":"2023-10-09T09:17:05.393152+0000","last_fullsized":"2023-10-09T09:17:05.393152+0000","mapping_epoch":248,"log_start":"255'4951","ondisk_log_start":"255'4951","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":8422,"log_dups_size":0,"ondisk_log_size":8422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13373,"num_write_kb":428480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:56.679964+0000","last_change":"2023-10-09T09:14:48.299630+0000","last_active":"2023-10-09T09:16:56.679964+0000","last_peered":"2023-10-09T09:16:56.679964+0000","last_clean":"2023-10-09T09:16:56.679964+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:16:56.679964+0000","last_undegraded":"2023-10-09T09:16:56.679964+0000","last_fullsized":"2023-10-09T09:16:56.679964+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":82,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.349308+0000","last_change":"2023-10-09T09:14:19.753429+0000","last_active":"2023-10-09T09:16:28.349308+0000","last_peered":"2023-10-09T09:16:28.349308+0000","last_clean":"2023-10-09T09:16:28.349308+0000","last_became_active":"2023-10-09T09:14:19.753022+0000","last_became_peered":"2023-10-09T09:14:19.753022+0000","last_unstale":"2023-10-09T09:16:28.349308+0000","last_undegraded":"2023-10-09T09:16:28.349308+0000","last_fullsized":"2023-10-09T09:16:28.349308+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":42,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:56.679870+0000","last_change":"2023-10-09T09:15:23.413972+0000","last_active":"2023-10-09T09:16:56.679870+0000","last_peered":"2023-10-09T09:16:56.679870+0000","last_clean":"2023-10-09T09:16:56.679870+0000","last_became_active":"2023-10-09T09:15:23.413531+0000","last_became_peered":"2023-10-09T09:15:23.413531+0000","last_unstale":"2023-10-09T09:16:56.679870+0000","last_undegraded":"2023-10-09T09:16:56.679870+0000","last_fullsized":"2023-10-09T09:16:56.679870+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:54.516595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"285'13320","reported_seq":13365,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.556050+0000","last_change":"2023-10-09T09:12:00.114449+0000","last_active":"2023-10-09T09:17:05.556050+0000","last_peered":"2023-10-09T09:17:05.556050+0000","last_clean":"2023-10-09T09:17:05.556050+0000","last_became_active":"2023-10-09T09:12:00.113992+0000","last_became_peered":"2023-10-09T09:12:00.113992+0000","last_unstale":"2023-10-09T09:17:05.556050+0000","last_undegraded":"2023-10-09T09:17:05.556050+0000","last_fullsized":"2023-10-09T09:17:05.556050+0000","mapping_epoch":248,"log_start":"255'5819","ondisk_log_start":"255'5819","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13320,"num_write_kb":426688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":82,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.349741+0000","last_change":"2023-10-09T09:14:48.319163+0000","last_active":"2023-10-09T09:16:28.349741+0000","last_peered":"2023-10-09T09:16:28.349741+0000","last_clean":"2023-10-09T09:16:28.349741+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:16:28.349741+0000","last_undegraded":"2023-10-09T09:16:28.349741+0000","last_fullsized":"2023-10-09T09:16:28.349741+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":148,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280995+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:16:35.280995+0000","last_peered":"2023-10-09T09:16:35.280995+0000","last_clean":"2023-10-09T09:16:35.280995+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:16:35.280995+0000","last_undegraded":"2023-10-09T09:16:35.280995+0000","last_fullsized":"2023-10-09T09:16:35.280995+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:56.679932+0000","last_change":"2023-10-09T09:14:48.318830+0000","last_active":"2023-10-09T09:16:56.679932+0000","last_peered":"2023-10-09T09:16:56.679932+0000","last_clean":"2023-10-09T09:16:56.679932+0000","last_became_active":"2023-10-09T09:14:48.317507+0000","last_became_peered":"2023-10-09T09:14:48.317507+0000","last_unstale":"2023-10-09T09:16:56.679932+0000","last_undegraded":"2023-10-09T09:16:56.679932+0000","last_fullsized":"2023-10-09T09:16:56.679932+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:28:14.374899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"285'13251","reported_seq":19703,"reported_epoch":285,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T09:17:05.483711+0000","last_change":"2023-10-09T09:15:24.759912+0000","last_active":"2023-10-09T09:17:05.483711+0000","last_peered":"2023-10-09T09:17:05.483711+0000","last_clean":"2023-10-09T09:14:19.067968+0000","last_became_active":"2023-10-09T09:15:24.636106+0000","last_became_peered":"2023-10-09T09:15:24.636106+0000","last_unstale":"2023-10-09T09:17:05.483711+0000","last_undegraded":"2023-10-09T09:15:24.618552+0000","last_fullsized":"2023-10-09T09:17:05.483711+0000","mapping_epoch":271,"log_start":"255'4900","ondisk_log_start":"255'4900","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":8351,"log_dups_size":0,"ondisk_log_size":8351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17,"num_objects_misplaced":3435,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13251,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3199,"num_bytes_recovered":209649664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,5],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":17}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":255,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.271672+0000","last_change":"2023-10-09T09:14:48.312633+0000","last_active":"2023-10-09T09:16:35.271672+0000","last_peered":"2023-10-09T09:16:35.271672+0000","last_clean":"2023-10-09T09:16:35.271672+0000","last_became_active":"2023-10-09T09:14:48.312271+0000","last_became_peered":"2023-10-09T09:14:48.312271+0000","last_unstale":"2023-10-09T09:16:35.271672+0000","last_undegraded":"2023-10-09T09:16:35.271672+0000","last_fullsized":"2023-10-09T09:16:35.271672+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:16.219376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:56.679799+0000","last_change":"2023-10-09T09:14:48.306136+0000","last_active":"2023-10-09T09:16:56.679799+0000","last_peered":"2023-10-09T09:16:56.679799+0000","last_clean":"2023-10-09T09:16:56.679799+0000","last_became_active":"2023-10-09T09:14:48.305165+0000","last_became_peered":"2023-10-09T09:14:48.305165+0000","last_unstale":"2023-10-09T09:16:56.679799+0000","last_undegraded":"2023-10-09T09:16:56.679799+0000","last_fullsized":"2023-10-09T09:16:56.679799+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:13.935175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:01.680774+0000","last_change":"2023-10-09T09:14:48.321029+0000","last_active":"2023-10-09T09:17:01.680774+0000","last_peered":"2023-10-09T09:17:01.680774+0000","last_clean":"2023-10-09T09:17:01.680774+0000","last_became_active":"2023-10-09T09:14:48.320435+0000","last_became_peered":"2023-10-09T09:14:48.320435+0000","last_unstale":"2023-10-09T09:17:01.680774+0000","last_undegraded":"2023-10-09T09:17:01.680774+0000","last_fullsized":"2023-10-09T09:17:01.680774+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:20.309472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"285'13398","reported_seq":18872,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.469142+0000","last_change":"2023-10-09T09:15:54.564593+0000","last_active":"2023-10-09T09:17:05.469142+0000","last_peered":"2023-10-09T09:17:05.469142+0000","last_clean":"2023-10-09T09:17:05.469142+0000","last_became_active":"2023-10-09T09:15:54.521024+0000","last_became_peered":"2023-10-09T09:15:54.521024+0000","last_unstale":"2023-10-09T09:17:05.469142+0000","last_undegraded":"2023-10-09T09:17:05.469142+0000","last_fullsized":"2023-10-09T09:17:05.469142+0000","mapping_epoch":279,"log_start":"255'5798","ondisk_log_start":"255'5798","created":248,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7600,"log_dups_size":0,"ondisk_log_size":7600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13398,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2701,"num_bytes_recovered":176947200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":91,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.352562+0000","last_change":"2023-10-09T09:14:19.748416+0000","last_active":"2023-10-09T09:16:28.352562+0000","last_peered":"2023-10-09T09:16:28.352562+0000","last_clean":"2023-10-09T09:16:28.352562+0000","last_became_active":"2023-10-09T09:14:19.748079+0000","last_became_peered":"2023-10-09T09:14:19.748079+0000","last_unstale":"2023-10-09T09:16:28.352562+0000","last_undegraded":"2023-10-09T09:16:28.352562+0000","last_fullsized":"2023-10-09T09:16:28.352562+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":523,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.358439+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:16:28.358439+0000","last_peered":"2023-10-09T09:16:28.358439+0000","last_clean":"2023-10-09T09:16:28.358439+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:16:28.358439+0000","last_undegraded":"2023-10-09T09:16:28.358439+0000","last_fullsized":"2023-10-09T09:16:28.358439+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":112,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.358045+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:16:28.358045+0000","last_peered":"2023-10-09T09:16:28.358045+0000","last_clean":"2023-10-09T09:16:28.358045+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:16:28.358045+0000","last_undegraded":"2023-10-09T09:16:28.358045+0000","last_fullsized":"2023-10-09T09:16:28.358045+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":143,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.337695+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:16:28.337695+0000","last_peered":"2023-10-09T09:16:28.337695+0000","last_clean":"2023-10-09T09:16:28.337695+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:16:28.337695+0000","last_undegraded":"2023-10-09T09:16:28.337695+0000","last_fullsized":"2023-10-09T09:16:28.337695+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"286'13690","reported_seq":20854,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.876192+0000","last_change":"2023-10-09T09:15:29.480776+0000","last_active":"2023-10-09T09:17:05.876192+0000","last_peered":"2023-10-09T09:17:05.876192+0000","last_clean":"2023-10-09T09:17:05.876192+0000","last_became_active":"2023-10-09T09:15:29.446526+0000","last_became_peered":"2023-10-09T09:15:29.446526+0000","last_unstale":"2023-10-09T09:17:05.876192+0000","last_undegraded":"2023-10-09T09:17:05.876192+0000","last_fullsized":"2023-10-09T09:17:05.876192+0000","mapping_epoch":273,"log_start":"256'6486","ondisk_log_start":"256'6486","created":248,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7204,"log_dups_size":0,"ondisk_log_size":7204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:31.768973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13690,"num_write_kb":438080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":232062976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:56.679832+0000","last_change":"2023-10-09T09:14:48.301697+0000","last_active":"2023-10-09T09:16:56.679832+0000","last_peered":"2023-10-09T09:16:56.679832+0000","last_clean":"2023-10-09T09:16:56.679832+0000","last_became_active":"2023-10-09T09:14:48.301379+0000","last_became_peered":"2023-10-09T09:14:48.301379+0000","last_unstale":"2023-10-09T09:16:56.679832+0000","last_undegraded":"2023-10-09T09:16:56.679832+0000","last_fullsized":"2023-10-09T09:16:56.679832+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:11.963433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":321,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:02.303621+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:17:02.303621+0000","last_peered":"2023-10-09T09:17:02.303621+0000","last_clean":"2023-10-09T09:17:02.303621+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:17:02.303621+0000","last_undegraded":"2023-10-09T09:17:02.303621+0000","last_fullsized":"2023-10-09T09:17:02.303621+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":317,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.277753+0000","last_change":"2023-10-09T09:14:48.306410+0000","last_active":"2023-10-09T09:16:35.277753+0000","last_peered":"2023-10-09T09:16:35.277753+0000","last_clean":"2023-10-09T09:16:35.277753+0000","last_became_active":"2023-10-09T09:14:48.305480+0000","last_became_peered":"2023-10-09T09:14:48.305480+0000","last_unstale":"2023-10-09T09:16:35.277753+0000","last_undegraded":"2023-10-09T09:16:35.277753+0000","last_fullsized":"2023-10-09T09:16:35.277753+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:28.296221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"285'13539","reported_seq":13585,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.561648+0000","last_change":"2023-10-09T09:12:00.113109+0000","last_active":"2023-10-09T09:17:05.561648+0000","last_peered":"2023-10-09T09:17:05.561648+0000","last_clean":"2023-10-09T09:17:05.561648+0000","last_became_active":"2023-10-09T09:12:00.112495+0000","last_became_peered":"2023-10-09T09:12:00.112495+0000","last_unstale":"2023-10-09T09:17:05.561648+0000","last_undegraded":"2023-10-09T09:17:05.561648+0000","last_fullsized":"2023-10-09T09:17:05.561648+0000","mapping_epoch":248,"log_start":"255'5961","ondisk_log_start":"255'5961","created":248,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917528,"num_objects":15,"num_object_clones":0,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13539,"num_write_kb":433665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.359403+0000","last_change":"2023-10-09T09:09:44.439090+0000","last_active":"2023-10-09T09:16:28.359403+0000","last_peered":"2023-10-09T09:16:28.359403+0000","last_clean":"2023-10-09T09:16:28.359403+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:16:28.359403+0000","last_undegraded":"2023-10-09T09:16:28.359403+0000","last_fullsized":"2023-10-09T09:16:28.359403+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:56.679897+0000","last_change":"2023-10-09T09:14:48.319210+0000","last_active":"2023-10-09T09:16:56.679897+0000","last_peered":"2023-10-09T09:16:56.679897+0000","last_clean":"2023-10-09T09:16:56.679897+0000","last_became_active":"2023-10-09T09:14:48.318880+0000","last_became_peered":"2023-10-09T09:14:48.318880+0000","last_unstale":"2023-10-09T09:16:56.679897+0000","last_undegraded":"2023-10-09T09:16:56.679897+0000","last_fullsized":"2023-10-09T09:16:56.679897+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:12.472280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":297,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:02.303679+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:17:02.303679+0000","last_peered":"2023-10-09T09:17:02.303679+0000","last_clean":"2023-10-09T09:17:02.303679+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:17:02.303679+0000","last_undegraded":"2023-10-09T09:17:02.303679+0000","last_fullsized":"2023-10-09T09:17:02.303679+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"286'13216","reported_seq":22129,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.951889+0000","last_change":"2023-10-09T09:15:08.403520+0000","last_active":"2023-10-09T09:17:05.951889+0000","last_peered":"2023-10-09T09:17:05.951889+0000","last_clean":"2023-10-09T09:17:05.951889+0000","last_became_active":"2023-10-09T09:15:08.367545+0000","last_became_peered":"2023-10-09T09:15:08.367545+0000","last_unstale":"2023-10-09T09:17:05.951889+0000","last_undegraded":"2023-10-09T09:17:05.951889+0000","last_fullsized":"2023-10-09T09:17:05.951889+0000","mapping_epoch":268,"log_start":"258'6813","ondisk_log_start":"258'6813","created":248,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13216,"num_write_kb":422912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4420,"num_bytes_recovered":289669120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":53,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.275351+0000","last_change":"2023-10-09T09:14:19.749169+0000","last_active":"2023-10-09T09:16:35.275351+0000","last_peered":"2023-10-09T09:16:35.275351+0000","last_clean":"2023-10-09T09:16:35.275351+0000","last_became_active":"2023-10-09T09:14:19.740888+0000","last_became_peered":"2023-10-09T09:14:19.740888+0000","last_unstale":"2023-10-09T09:16:35.275351+0000","last_undegraded":"2023-10-09T09:16:35.275351+0000","last_fullsized":"2023-10-09T09:16:35.275351+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:01.361061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:01.680749+0000","last_change":"2023-10-09T09:14:48.317228+0000","last_active":"2023-10-09T09:17:01.680749+0000","last_peered":"2023-10-09T09:17:01.680749+0000","last_clean":"2023-10-09T09:17:01.680749+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:17:01.680749+0000","last_undegraded":"2023-10-09T09:17:01.680749+0000","last_fullsized":"2023-10-09T09:17:01.680749+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.357333+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:16:28.357333+0000","last_peered":"2023-10-09T09:16:28.357333+0000","last_clean":"2023-10-09T09:16:28.357333+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:16:28.357333+0000","last_undegraded":"2023-10-09T09:16:28.357333+0000","last_fullsized":"2023-10-09T09:16:28.357333+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"285'13288","reported_seq":19334,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:05.515529+0000","last_change":"2023-10-09T09:15:48.533323+0000","last_active":"2023-10-09T09:17:05.515529+0000","last_peered":"2023-10-09T09:17:05.515529+0000","last_clean":"2023-10-09T09:17:05.515529+0000","last_became_active":"2023-10-09T09:15:48.501622+0000","last_became_peered":"2023-10-09T09:15:48.501622+0000","last_unstale":"2023-10-09T09:17:05.515529+0000","last_undegraded":"2023-10-09T09:17:05.515529+0000","last_fullsized":"2023-10-09T09:17:05.515529+0000","mapping_epoch":277,"log_start":"255'5783","ondisk_log_start":"255'5783","created":248,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:11:59.086303+0000","last_clean_scrub_stamp":"2023-10-09T09:11:59.086303+0000","objects_scrubbed":0,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13288,"num_write_kb":425792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2981,"num_bytes_recovered":195297280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":152,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.337815+0000","last_change":"2023-10-09T09:14:48.319849+0000","last_active":"2023-10-09T09:16:28.337815+0000","last_peered":"2023-10-09T09:16:28.337815+0000","last_clean":"2023-10-09T09:16:28.337815+0000","last_became_active":"2023-10-09T09:14:48.319372+0000","last_became_peered":"2023-10-09T09:14:48.319372+0000","last_unstale":"2023-10-09T09:16:28.337815+0000","last_undegraded":"2023-10-09T09:16:28.337815+0000","last_fullsized":"2023-10-09T09:16:28.337815+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":339,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.277853+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:16:35.277853+0000","last_peered":"2023-10-09T09:16:35.277853+0000","last_clean":"2023-10-09T09:16:35.277853+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:16:35.277853+0000","last_undegraded":"2023-10-09T09:16:35.277853+0000","last_fullsized":"2023-10-09T09:16:35.277853+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":166,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280082+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:16:35.280082+0000","last_peered":"2023-10-09T09:16:35.280082+0000","last_clean":"2023-10-09T09:16:35.280082+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:16:35.280082+0000","last_undegraded":"2023-10-09T09:16:35.280082+0000","last_fullsized":"2023-10-09T09:16:35.280082+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":307,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.272384+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:16:35.272384+0000","last_peered":"2023-10-09T09:16:35.272384+0000","last_clean":"2023-10-09T09:16:35.272384+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:16:35.272384+0000","last_undegraded":"2023-10-09T09:16:35.272384+0000","last_fullsized":"2023-10-09T09:16:35.272384+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":32,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.268458+0000","last_change":"2023-10-09T09:14:48.318983+0000","last_active":"2023-10-09T09:16:35.268458+0000","last_peered":"2023-10-09T09:16:35.268458+0000","last_clean":"2023-10-09T09:16:35.268458+0000","last_became_active":"2023-10-09T09:14:48.317990+0000","last_became_peered":"2023-10-09T09:14:48.317990+0000","last_unstale":"2023-10-09T09:16:35.268458+0000","last_undegraded":"2023-10-09T09:16:35.268458+0000","last_fullsized":"2023-10-09T09:16:35.268458+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:43.629682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":90,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.349511+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:16:28.349511+0000","last_peered":"2023-10-09T09:16:28.349511+0000","last_clean":"2023-10-09T09:16:28.349511+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:16:28.349511+0000","last_undegraded":"2023-10-09T09:16:28.349511+0000","last_fullsized":"2023-10-09T09:16:28.349511+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":166,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280855+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:16:35.280855+0000","last_peered":"2023-10-09T09:16:35.280855+0000","last_clean":"2023-10-09T09:16:35.280855+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:16:35.280855+0000","last_undegraded":"2023-10-09T09:16:35.280855+0000","last_fullsized":"2023-10-09T09:16:35.280855+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":77,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.349150+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:16:28.349150+0000","last_peered":"2023-10-09T09:16:28.349150+0000","last_clean":"2023-10-09T09:16:28.349150+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:16:28.349150+0000","last_undegraded":"2023-10-09T09:16:28.349150+0000","last_fullsized":"2023-10-09T09:16:28.349150+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":148,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.278919+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:16:35.278919+0000","last_peered":"2023-10-09T09:16:35.278919+0000","last_clean":"2023-10-09T09:16:35.278919+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:16:35.278919+0000","last_undegraded":"2023-10-09T09:16:35.278919+0000","last_fullsized":"2023-10-09T09:16:35.278919+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":239,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T09:16:28.337621+0000","last_change":"2023-10-09T09:14:48.312443+0000","last_active":"2023-10-09T09:16:28.337621+0000","last_peered":"2023-10-09T09:16:28.337621+0000","last_clean":"2023-10-09T09:16:28.337621+0000","last_became_active":"2023-10-09T09:14:48.312053+0000","last_became_peered":"2023-10-09T09:14:48.312053+0000","last_unstale":"2023-10-09T09:16:28.337621+0000","last_undegraded":"2023-10-09T09:16:28.337621+0000","last_fullsized":"2023-10-09T09:16:28.337621+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":229,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:52.283221+0000","last_change":"2023-10-09T09:14:19.744537+0000","last_active":"2023-10-09T09:16:52.283221+0000","last_peered":"2023-10-09T09:16:52.283221+0000","last_clean":"2023-10-09T09:16:52.283221+0000","last_became_active":"2023-10-09T09:14:19.744078+0000","last_became_peered":"2023-10-09T09:14:19.744078+0000","last_unstale":"2023-10-09T09:16:52.283221+0000","last_undegraded":"2023-10-09T09:16:52.283221+0000","last_fullsized":"2023-10-09T09:16:52.283221+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:49.733990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":382,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:17:02.303748+0000","last_change":"2023-10-09T09:14:19.780202+0000","last_active":"2023-10-09T09:17:02.303748+0000","last_peered":"2023-10-09T09:17:02.303748+0000","last_clean":"2023-10-09T09:17:02.303748+0000","last_became_active":"2023-10-09T09:14:19.758344+0000","last_became_peered":"2023-10-09T09:14:19.758344+0000","last_unstale":"2023-10-09T09:17:02.303748+0000","last_undegraded":"2023-10-09T09:17:02.303748+0000","last_fullsized":"2023-10-09T09:17:02.303748+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:36.052871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":175,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T09:16:35.280443+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:16:35.280443+0000","last_peered":"2023-10-09T09:16:35.280443+0000","last_clean":"2023-10-09T09:16:35.280443+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:16:35.280443+0000","last_undegraded":"2023-10-09T09:16:35.280443+0000","last_fullsized":"2023-10-09T09:16:35.280443+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":11599896,"num_objects":178,"num_object_clones":0,"num_object_copies":534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7637,"num_objects_misplaced":3435,"num_objects_unfound":0,"num_objects_dirty":178,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":214264,"num_write_kb":6862081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27392,"num_bytes_recovered":1745354752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1455427584,"data_stored":1455423512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123792,"ondisk_log_size":123792,"up":48,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":19206,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430759,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":508860,"kb_used_data":199284,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93236292,"statfs":{"total":95995035648,"available":95473963008,"internally_reserved":0,"allocated":204066816,"data_stored":203486169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.675,"15min":2.657},"min":{"1min":0.27800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.492,"5min":6.7990000000000004,"15min":119.498},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.4079999999999999,"15min":2.5489999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":2.5720000000000001,"5min":5.5430000000000001,"15min":119.532},"last":4.2450000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":1.766,"15min":1.1479999999999999},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":21.800999999999998,"5min":21.800999999999998,"15min":21.800999999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.391,"15min":0.94999999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":21.734000000000002,"5min":21.734000000000002,"15min":21.734000000000002},"last":4.2169999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":1.9490000000000001,"15min":2.3290000000000002},"min":{"1min":0.53100000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.9430000000000001,"5min":45.345999999999997,"15min":126.072},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.8979999999999999,"5min":2.077,"15min":2.395},"min":{"1min":0.42099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.6210000000000004,"5min":46.228000000000002,"15min":120.729},"last":0.53700000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":2.415,"15min":3.4460000000000002},"min":{"1min":0.375,"5min":0.316,"15min":0.316},"max":{"1min":3.6200000000000001,"5min":62.060000000000002,"15min":191.30799999999999},"last":4.1900000000000004},{"interface":"front","average":{"1min":1.552,"5min":2.738,"15min":3.5350000000000001},"min":{"1min":0.41699999999999998,"5min":0.34999999999999998,"15min":0.318},"max":{"1min":5.6299999999999999,"5min":63.268999999999998,"15min":191.47900000000001},"last":0.51200000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.927,"15min":1.948},"min":{"1min":0.39900000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.0809999999999995,"5min":11.371,"15min":138.34299999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.76,"5min":1.8999999999999999,"15min":1.958},"min":{"1min":0.35999999999999999,"5min":0.314,"15min":0.314},"max":{"1min":8.0969999999999995,"5min":8.9879999999999995,"15min":137.04900000000001},"last":0.90200000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:17:05 2023","interfaces":[{"interface":"back","average":{"1min":3.8199999999999998,"5min":2.0009999999999999,"15min":1.3220000000000001},"min":{"1min":0.53500000000000003,"5min":0.34599999999999997,"15min":0.28199999999999997},"max":{"1min":15.616,"5min":15.616,"15min":15.616},"last":4.1470000000000002},{"interface":"front","average":{"1min":3.0510000000000002,"5min":2.331,"15min":1.5800000000000001},"min":{"1min":0.46600000000000003,"5min":0.36599999999999999,"15min":0.34499999999999997},"max":{"1min":11.818,"5min":18.501000000000001,"15min":18.501000000000001},"last":1.1200000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":1.8799999999999999,"15min":1.419},"min":{"1min":0.40300000000000002,"5min":0.39300000000000002,"15min":0.35599999999999998},"max":{"1min":6.8109999999999999,"5min":21.145,"15min":21.145},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.8319999999999999,"5min":2.323,"15min":1.573},"min":{"1min":0.54900000000000004,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":10.268000000000001,"5min":19.068999999999999,"15min":19.068999999999999},"last":0.75}]}]},{"osd":6,"up_from":42,"seq":180388626992,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925756,"kb_used_data":22388,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92819396,"statfs":{"total":95995035648,"available":95047061504,"internally_reserved":0,"allocated":22925312,"data_stored":22345081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:16:48 2023","interfaces":[{"interface":"back","average":{"1min":7.0910000000000002,"5min":5.4249999999999998,"15min":2.7869999999999999},"min":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":120.812,"5min":120.812,"15min":120.812},"last":0.46000000000000002},{"interface":"front","average":{"1min":6.3719999999999999,"5min":5.3070000000000004,"15min":2.8039999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.24099999999999999},"max":{"1min":120.857,"5min":120.857,"15min":120.857},"last":0.53900000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:16:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0529999999999999,"5min":1.7969999999999999,"15min":1.647},"min":{"1min":0.32000000000000001,"5min":0.316,"15min":0.314},"max":{"1min":10.657999999999999,"5min":12.369,"15min":109.453},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.889,"5min":1.589,"15min":1.5960000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.079999999999998,"5min":18.079999999999998,"15min":111.47499999999999},"last":0.49299999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.7210000000000001,"15min":1.5920000000000001},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.0730000000000004,"5min":10.590999999999999,"15min":18.756},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.7729999999999999,"15min":1.6140000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30199999999999999},"max":{"1min":5.2670000000000003,"5min":18.170000000000002,"15min":18.841999999999999},"last":0.59299999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:16:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.881,"15min":1.5760000000000001},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.630000000000001,"5min":12.35,"15min":16.977},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.506,"15min":1.6850000000000001},"min":{"1min":0.33700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2430000000000003,"5min":12.628,"15min":20.573},"last":0.50900000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":2.5019999999999998,"15min":1.7649999999999999},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":15.818,"5min":29.585999999999999,"15min":32.280999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.3380000000000001,"5min":2.4009999999999998,"15min":1.734},"min":{"1min":0.44,"5min":0.38300000000000001,"15min":0.317},"max":{"1min":7.2949999999999999,"5min":20.318000000000001,"15min":34.689},"last":3.4420000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":2.2970000000000002,"15min":1.6899999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.30199999999999999},"max":{"1min":5.3860000000000001,"5min":18.103000000000002,"15min":18.103000000000002},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.7270000000000001,"5min":2.1339999999999999,"15min":1.5680000000000001},"min":{"1min":0.35899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.3029999999999999,"5min":11.567,"15min":13.609},"last":3.395}]},{"osd":7,"last update":"Mon Oct 9 09:16:48 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":2.4300000000000002,"15min":1.7130000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.2439999999999998,"5min":29.530000000000001,"15min":29.530000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.9950000000000001,"15min":1.478},"min":{"1min":0.441,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":11.477,"5min":11.477,"15min":13.523999999999999},"last":3.4239999999999999}]}]},{"osd":1,"up_from":83,"seq":356482285940,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":451332,"kb_used_data":3964,"kb_used_omap":9,"kb_used_meta":447350,"kb_avail":93293820,"statfs":{"total":95995035648,"available":95532871680,"internally_reserved":0,"allocated":4059136,"data_stored":3012374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":458087401},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.96899999999999997,"15min":1.8640000000000001},"min":{"1min":0.29599999999999999,"5min":0.223,"15min":0.189},"max":{"1min":3.7160000000000002,"5min":15.298,"15min":94.915999999999997},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.3240000000000001,"15min":2.0619999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.21199999999999999},"max":{"1min":2.0329999999999999,"5min":19.899000000000001,"15min":98.227999999999994},"last":0.54400000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.80000000000000004,"15min":1.5449999999999999},"min":{"1min":0.28100000000000003,"5min":0.27800000000000002,"15min":0.23799999999999999},"max":{"1min":3.7559999999999998,"5min":5.1289999999999996,"15min":46.354999999999997},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.60099999999999998,"15min":1.3620000000000001},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.252},"max":{"1min":1.5880000000000001,"5min":1.829,"15min":46.241999999999997},"last":0.621}]},{"osd":3,"last update":"Mon Oct 9 09:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.80300000000000005,"15min":1.9119999999999999},"min":{"1min":0.36199999999999999,"5min":0.35299999999999998,"15min":0.26800000000000002},"max":{"1min":3.5830000000000002,"5min":3.5830000000000002,"15min":67.807000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.78000000000000003,"15min":1.788},"min":{"1min":0.379,"5min":0.308,"15min":0.22600000000000001},"max":{"1min":3.3759999999999999,"5min":3.3759999999999999,"15min":68.097999999999999},"last":0.59299999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:16:14 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.756,"15min":1.2789999999999999},"min":{"1min":0.52400000000000002,"5min":0.376,"15min":0.30099999999999999},"max":{"1min":8.0600000000000005,"5min":8.0600000000000005,"15min":8.0600000000000005},"last":3.6219999999999999},{"interface":"front","average":{"1min":2.5089999999999999,"5min":1.522,"15min":1.163},"min":{"1min":0.42599999999999999,"5min":0.36899999999999999,"15min":0.33100000000000002},"max":{"1min":10.523999999999999,"5min":10.523999999999999,"15min":10.523999999999999},"last":1.1859999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:16:14 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.3919999999999999,"15min":1.1279999999999999},"min":{"1min":0.376,"5min":0.34699999999999998,"15min":0.33800000000000002},"max":{"1min":5.6749999999999998,"5min":6.1929999999999996,"15min":7.6600000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.3,"15min":1.1120000000000001},"min":{"1min":0.45900000000000002,"5min":0.41699999999999998,"15min":0.35299999999999998},"max":{"1min":5.6449999999999996,"5min":6.2080000000000002,"15min":7.6399999999999997},"last":0.93100000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:16:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.6699999999999999,"15min":1.343},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.33200000000000002},"max":{"1min":4.016,"5min":7.9480000000000004,"15min":7.9480000000000004},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.4870000000000001,"5min":1.754,"15min":1.3460000000000001},"min":{"1min":0.45800000000000002,"5min":0.42499999999999999,"15min":0.39000000000000001},"max":{"1min":4.0439999999999996,"5min":8.9079999999999995,"15min":8.9079999999999995},"last":0.79100000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:16:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.488,"15min":1.1910000000000001},"min":{"1min":0.48399999999999999,"5min":0.41799999999999998,"15min":0.38500000000000001},"max":{"1min":2.8900000000000001,"5min":10.326000000000001,"15min":10.326000000000001},"last":3.3010000000000002},{"interface":"front","average":{"1min":1.575,"5min":1.5049999999999999,"15min":1.246},"min":{"1min":0.55200000000000005,"5min":0.46800000000000003,"15min":0.38900000000000001},"max":{"1min":4.9080000000000004,"5min":6.8330000000000002,"15min":7.6379999999999999},"last":4.6269999999999998}]}]},{"osd":0,"up_from":78,"seq":335007449466,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":875732,"kb_used_data":3660,"kb_used_omap":28,"kb_used_meta":872035,"kb_avail":92869420,"statfs":{"total":95995035648,"available":95098286080,"internally_reserved":0,"allocated":3747840,"data_stored":2686467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":892964688},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.878,"15min":1.081},"min":{"1min":0.25700000000000001,"5min":0.193,"15min":0.193},"max":{"1min":3.1539999999999999,"5min":9.7620000000000005,"15min":9.7620000000000005},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.96599999999999997,"15min":1.022},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.22800000000000001},"max":{"1min":4.3280000000000003,"5min":6.6040000000000001,"15min":10.102},"last":0.40500000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":1.9570000000000001,"15min":2.3900000000000001},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.23400000000000001},"max":{"1min":4.0199999999999996,"5min":74.200999999999993,"15min":74.200999999999993},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.81399999999999995,"5min":2.149,"15min":3.2770000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.24399999999999999},"max":{"1min":4.2110000000000003,"5min":73.983999999999995,"15min":219.00700000000001},"last":0.51600000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:16:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.4239999999999999,"15min":2.6920000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2480000000000002,"5min":20.364000000000001,"15min":219.041},"last":2.984},{"interface":"front","average":{"1min":1.248,"5min":1.423,"15min":2.1970000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28899999999999998},"max":{"1min":6.8079999999999998,"5min":18.895,"15min":137.94999999999999},"last":2.77}]},{"osd":4,"last update":"Mon Oct 9 09:17:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9379999999999999,"5min":2.3980000000000001,"15min":1.7529999999999999},"min":{"1min":0.45400000000000001,"5min":0.34000000000000002,"15min":0.318},"max":{"1min":6.4199999999999999,"5min":14.163,"15min":14.163},"last":4.9669999999999996},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.4550000000000001,"15min":1.901},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.29099999999999998},"max":{"1min":11.346,"5min":11.346,"15min":11.346},"last":5.4249999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:17:01 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.0529999999999999,"15min":1.7090000000000001},"min":{"1min":0.40200000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.6739999999999995,"5min":9.9770000000000003,"15min":9.9770000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.7709999999999999,"5min":1.677,"15min":1.6439999999999999},"min":{"1min":0.41599999999999998,"5min":0.40699999999999997,"15min":0.35299999999999998},"max":{"1min":6.0960000000000001,"5min":9.5909999999999993,"15min":16.364000000000001},"last":4.7590000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:17:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.994,"15min":2.3799999999999999},"min":{"1min":0.53800000000000003,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":11.314,"5min":11.314,"15min":98.864000000000004},"last":3.9390000000000001},{"interface":"front","average":{"1min":1.883,"5min":2.222,"15min":2.468},"min":{"1min":0.44,"5min":0.44,"15min":0.38300000000000001},"max":{"1min":12.561999999999999,"5min":12.561999999999999,"15min":98.878},"last":3.9540000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:17:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.895,"15min":1.6659999999999999},"min":{"1min":0.47599999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":8.3949999999999996,"5min":9.6780000000000008,"15min":10.157999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.2759999999999998,"5min":2.1819999999999999,"15min":1.8520000000000001},"min":{"1min":0.45100000000000001,"5min":0.41199999999999998,"15min":0.40100000000000002},"max":{"1min":9.6859999999999999,"5min":10.734,"15min":10.734},"last":3.468}]}]},{"osd":2,"up_from":0,"seq":377957122413,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":93,"seq":399431958888,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860360,"kb_used_data":4224,"kb_used_omap":7,"kb_used_meta":856120,"kb_avail":92884792,"statfs":{"total":95995035648,"available":95114027008,"internally_reserved":0,"allocated":4325376,"data_stored":3273514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":876667496},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:16:08 2023","interfaces":[{"interface":"back","average":{"1min":3.8929999999999998,"5min":3.1080000000000001,"15min":2.7629999999999999},"min":{"1min":0.27700000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":46.433999999999997,"5min":130.45500000000001,"15min":233.22900000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":3.8109999999999999,"5min":3.2639999999999998,"15min":3.0859999999999999},"min":{"1min":0.32200000000000001,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":46.387,"5min":129.74799999999999,"15min":237.447},"last":0.33600000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:16:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.036,"15min":2.331},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":32.520000000000003,"5min":32.520000000000003,"15min":326.75700000000001},"last":0.254},{"interface":"front","average":{"1min":2.3380000000000001,"5min":1.0569999999999999,"15min":2.1899999999999999},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.22900000000000001},"max":{"1min":32.585999999999999,"5min":32.585999999999999,"15min":327.024},"last":0.28199999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:16:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":2.0550000000000002,"15min":2.1190000000000002},"min":{"1min":0.32400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.1899999999999999,"5min":129.67699999999999,"15min":129.67699999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.752,"5min":2.0830000000000002,"15min":2.0499999999999998},"min":{"1min":0.309,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.3799999999999999,"5min":130.40000000000001,"15min":130.40000000000001},"last":0.377}]},{"osd":4,"last update":"Mon Oct 9 09:16:08 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.46,"15min":1.3240000000000001},"min":{"1min":0.41299999999999998,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":4.4109999999999996,"5min":6.6790000000000003,"15min":8.9879999999999995},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.448,"5min":1.744,"15min":1.734},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.313},"max":{"1min":12.617000000000001,"5min":12.617000000000001,"15min":12.617000000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:16:08 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":1.522,"15min":1.655},"min":{"1min":0.40600000000000003,"5min":0.40500000000000003,"15min":0.38100000000000001},"max":{"1min":4.9180000000000001,"5min":7.819,"15min":16.132999999999999},"last":3.5920000000000001},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.653,"15min":1.6599999999999999},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.5210000000000008,"5min":9.5210000000000008,"15min":14.361000000000001},"last":0.40000000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:16:08 2023","interfaces":[{"interface":"back","average":{"1min":2.121,"5min":1.9279999999999999,"15min":1.891},"min":{"1min":0.37,"5min":0.37,"15min":0.308},"max":{"1min":8.4220000000000006,"5min":9.1159999999999997,"15min":12.105},"last":3.552},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.6910000000000001,"15min":1.76},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.38500000000000001},"max":{"1min":8.3510000000000009,"5min":9.0199999999999996,"15min":9.5090000000000003},"last":3.5419999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:16:08 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":1.3839999999999999,"15min":1.577},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.41499999999999998},"max":{"1min":8.8780000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":3.6909999999999998},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.389,"15min":1.4299999999999999},"min":{"1min":0.441,"5min":0.436,"15min":0.38300000000000001},"max":{"1min":8.8930000000000007,"5min":8.8930000000000007,"15min":9.1560000000000006},"last":3.7120000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019458,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927292,"kb_used_data":37300,"kb_used_omap":0,"kb_used_meta":889984,"kb_avail":92817860,"statfs":{"total":95995035648,"available":95045488640,"internally_reserved":0,"allocated":38195200,"data_stored":37614228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911343616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:16:39 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":3.1509999999999998,"15min":2.7229999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":9.4359999999999999,"5min":116.59999999999999,"15min":141.07400000000001},"last":3.0950000000000002},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.7250000000000001,"15min":2.6269999999999998},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":7.4199999999999999,"5min":116.843,"15min":141.15600000000001},"last":2.5099999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:16:11 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.54,"15min":1.0529999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":6.9009999999999998,"5min":13.853999999999999,"15min":13.853999999999999},"last":2.3010000000000002},{"interface":"front","average":{"1min":1.927,"5min":1.601,"15min":1.1459999999999999},"min":{"1min":0.34699999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":6.6020000000000003,"5min":11.391999999999999,"15min":11.391999999999999},"last":3.133}]},{"osd":2,"last update":"Mon Oct 9 09:16:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.952,"15min":2.456},"min":{"1min":0.40100000000000002,"5min":0.32800000000000001,"15min":0.29299999999999998},"max":{"1min":9.4819999999999993,"5min":116.7,"15min":142.816},"last":3.1970000000000001},{"interface":"front","average":{"1min":1.329,"5min":2.7890000000000001,"15min":2.423},"min":{"1min":0.36499999999999999,"5min":0.316,"15min":0.315},"max":{"1min":7.4989999999999997,"5min":116.63800000000001,"15min":143.81800000000001},"last":2.4220000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:16:11 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":2.4990000000000001,"15min":4.2240000000000002},"min":{"1min":0.34799999999999998,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":6.9409999999999998,"5min":82.219999999999999,"15min":165.995},"last":2.3580000000000001},{"interface":"front","average":{"1min":1.931,"5min":2.6160000000000001,"15min":4.5899999999999999},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.314},"max":{"1min":7.4619999999999997,"5min":81.757999999999996,"15min":165.53200000000001},"last":2.5369999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:16:26 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":2.4079999999999999,"15min":1.4930000000000001},"min":{"1min":0.36299999999999999,"5min":0.30299999999999999,"15min":0.30099999999999999},"max":{"1min":5.2249999999999996,"5min":13.907,"15min":13.907},"last":3.1629999999999998},{"interface":"front","average":{"1min":2.25,"5min":1.9359999999999999,"15min":1.3260000000000001},"min":{"1min":0.46899999999999997,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":7.3280000000000003,"5min":11.042999999999999,"15min":11.042999999999999},"last":0.438}]},{"osd":6,"last update":"Mon Oct 9 09:17:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":2.5289999999999999,"15min":1.762},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":18.103999999999999,"5min":18.103999999999999,"15min":74.700999999999993},"last":2.2669999999999999},{"interface":"front","average":{"1min":4.0439999999999996,"5min":2.738,"15min":1.9099999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.33000000000000002},"max":{"1min":11.958,"5min":13.444000000000001,"15min":73.572999999999993},"last":3.0550000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:16:39 2023","interfaces":[{"interface":"back","average":{"1min":3.093,"5min":2.4119999999999999,"15min":1.5229999999999999},"min":{"1min":0.38200000000000001,"5min":0.376,"15min":0.32800000000000001},"max":{"1min":11.196999999999999,"5min":11.196999999999999,"15min":11.196999999999999},"last":2.573},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.2090000000000001,"15min":1.476},"min":{"1min":0.44700000000000001,"5min":0.376,"15min":0.36199999999999999},"max":{"1min":11.090999999999999,"5min":13.916,"15min":13.916},"last":2.3319999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823226,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1040808,"kb_used_data":131296,"kb_used_omap":0,"kb_used_meta":909504,"kb_avail":92704344,"statfs":{"total":95995035648,"available":94929248256,"internally_reserved":0,"allocated":134447104,"data_stored":133882332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931332096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.804,"5min":3.806,"15min":3.2269999999999999},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":10.089,"5min":128.96199999999999,"15min":156.97800000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.51,"5min":3.4860000000000002,"15min":3.2559999999999998},"min":{"1min":0.315,"5min":0.252,"15min":0.252},"max":{"1min":5.2859999999999996,"5min":128.93700000000001,"15min":157.45699999999999},"last":3.782}]},{"osd":1,"last update":"Mon Oct 9 09:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.877,"15min":1.395},"min":{"1min":0.33900000000000002,"5min":0.312,"15min":0.307},"max":{"1min":5.625,"5min":11.105,"15min":36.963999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.671,"15min":1.325},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.1269999999999998,"5min":10.959,"15min":36.158000000000001},"last":4.0149999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":3.577,"15min":2.6899999999999999},"min":{"1min":0.36599999999999999,"5min":0.29299999999999998,"15min":0.27700000000000002},"max":{"1min":10.163,"5min":128.77199999999999,"15min":156.69499999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.7629999999999999,"5min":3.4159999999999999,"15min":2.532},"min":{"1min":0.38900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4000000000000004,"5min":128.72999999999999,"15min":156.87799999999999},"last":3.8479999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:17:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":1.8680000000000001,"15min":3.226},"min":{"1min":0.33000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.427,"5min":11.427,"15min":161.678},"last":11.427},{"interface":"front","average":{"1min":1.502,"5min":1.913,"15min":3.3599999999999999},"min":{"1min":0.35299999999999998,"5min":0.31,"15min":0.31},"max":{"1min":7.9870000000000001,"5min":11.019,"15min":159.43899999999999},"last":3.9820000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.7400000000000002,"15min":1.7070000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.5359999999999996,"5min":13.904,"15min":13.904},"last":0.46899999999999997},{"interface":"front","average":{"1min":4.0579999999999998,"5min":2.6549999999999998,"15min":1.6060000000000001},"min":{"1min":0.36899999999999999,"5min":0.34899999999999998,"15min":0.33500000000000002},"max":{"1min":12.081,"5min":22.228000000000002,"15min":22.228000000000002},"last":11.478}]},{"osd":6,"last update":"Mon Oct 9 09:17:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":2.238,"15min":1.631},"min":{"1min":0.35999999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.016,"5min":11.125,"15min":16.052},"last":4.0490000000000004},{"interface":"front","average":{"1min":2.6890000000000001,"5min":2.3969999999999998,"15min":1.6970000000000001},"min":{"1min":0.42499999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.0279999999999996,"5min":11.863,"15min":11.863},"last":3.742}]},{"osd":7,"last update":"Mon Oct 9 09:17:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.2170000000000001,"15min":1.4730000000000001},"min":{"1min":0.36799999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.9699999999999998,"5min":12.159000000000001,"15min":12.159000000000001},"last":3.8180000000000001},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.2519999999999998,"15min":1.534},"min":{"1min":0.441,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":9.9499999999999993,"5min":11.090999999999999,"15min":11.090999999999999},"last":7.5279999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1113784320,"data_stored":1113784320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":23724032,"data_stored":23724032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":118489088,"data_stored":118489088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":7016448,"data_stored":7012376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":192413696,"data_stored":192413696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:17:13.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-09T09:17:13.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:13.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:13.834 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:13.857 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:13.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:13.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:13.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:13.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:14.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:14.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:14.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:14.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:14.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:14.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:14.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:14.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:14.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:14.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:14.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:14.508 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:14.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:14.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:14.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:14.644 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:14.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:14.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:14.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:14.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:14.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:14.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:14.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:14.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:14.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:14.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:15.047 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:15.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:15.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:15.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:15.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:15.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:15.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:15.317 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:15.344 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:15.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:15.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:17:15.448 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:15.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:15.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:15.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:15 smithi016 bash[75179]: cluster 2023-10-09T09:17:14.155527+0000 mgr.y (mgr.25421) 1200 : cluster 0 pgmap v1307: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 7295/72 objects degraded (10131.944%); 3418/72 objects misplaced (4747.222%); 1.2 MiB/s, 165 objects/s recovering 2023-10-09T09:17:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:15 smithi016 bash[75179]: cluster 2023-10-09T09:17:14.772060+0000 mon.a (mon.0) 1604 : cluster 3 Health check update: Degraded data redundancy: 7295/72 objects degraded (10131.944%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:15 smithi016 bash[77892]: cluster 2023-10-09T09:17:14.155527+0000 mgr.y (mgr.25421) 1200 : cluster 0 pgmap v1307: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 7295/72 objects degraded (10131.944%); 3418/72 objects misplaced (4747.222%); 1.2 MiB/s, 165 objects/s recovering 2023-10-09T09:17:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:15 smithi016 bash[77892]: cluster 2023-10-09T09:17:14.772060+0000 mon.a (mon.0) 1604 : cluster 3 Health check update: Degraded data redundancy: 7295/72 objects degraded (10131.944%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:15.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:15.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:15.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:15.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:15.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:17:15.710 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:15.722 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:15 smithi094 bash[59677]: cluster 2023-10-09T09:17:14.155527+0000 mgr.y (mgr.25421) 1200 : cluster 0 pgmap v1307: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 7295/72 objects degraded (10131.944%); 3418/72 objects misplaced (4747.222%); 1.2 MiB/s, 165 objects/s recovering 2023-10-09T09:17:15.722 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:15 smithi094 bash[59677]: cluster 2023-10-09T09:17:14.772060+0000 mon.a (mon.0) 1604 : cluster 3 Health check update: Degraded data redundancy: 7295/72 objects degraded (10131.944%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:15.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:15.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:15.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:15.858 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:17:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:17:15] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.33.4" 2023-10-09T09:17:15.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:15.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:15.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:15.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:16.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:16.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:16.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:16.248 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:16.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:16.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:16.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:16.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:16.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:16.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:16.658 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:16.689 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:16.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:16.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:16.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:16.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:16.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:16.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:16.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:17.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:17.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:17.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:17.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:17.227 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:17.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:17.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:17.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:17.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:17.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:17.478 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:17 smithi094 bash[59677]: cluster 2023-10-09T09:17:16.156520+0000 mgr.y (mgr.25421) 1201 : cluster 0 pgmap v1308: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 93 op/s; 7295/72 objects degraded (10131.944%); 3418/72 objects misplaced (4747.222%); 105 KiB/s, 143 objects/s recovering 2023-10-09T09:17:17.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:17.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:17.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:17.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:17.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:17.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:17.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:17.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:17 smithi016 bash[77892]: cluster 2023-10-09T09:17:16.156520+0000 mgr.y (mgr.25421) 1201 : cluster 0 pgmap v1308: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 93 op/s; 7295/72 objects degraded (10131.944%); 3418/72 objects misplaced (4747.222%); 105 KiB/s, 143 objects/s recovering 2023-10-09T09:17:17.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:17 smithi016 bash[75179]: cluster 2023-10-09T09:17:16.156520+0000 mgr.y (mgr.25421) 1201 : cluster 0 pgmap v1308: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 93 op/s; 7295/72 objects degraded (10131.944%); 3418/72 objects misplaced (4747.222%); 105 KiB/s, 143 objects/s recovering 2023-10-09T09:17:17.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:17.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:17.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:17.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:17.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:18.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:18.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:18.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:18.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:18.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:18.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:18.300 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:18.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:18.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:18.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:18.452 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:17:18.531 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:17:18.537 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:18.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:18 smithi094 bash[59677]: cluster 2023-10-09T09:17:17.437453+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:18 smithi094 bash[59677]: cluster 2023-10-09T09:17:17.438296+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-09T09:17:18.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:18 smithi094 bash[59677]: cluster 2023-10-09T09:17:17.446643+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:18 smithi094 bash[59677]: cluster 2023-10-09T09:17:17.449114+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.557 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:18.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:18.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:18.672 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[75179]: cluster 2023-10-09T09:17:17.437453+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.673 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[75179]: cluster 2023-10-09T09:17:17.438296+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-09T09:17:18.673 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[75179]: cluster 2023-10-09T09:17:17.446643+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.673 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[75179]: cluster 2023-10-09T09:17:17.449114+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.673 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[77892]: cluster 2023-10-09T09:17:17.437453+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.673 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[77892]: cluster 2023-10-09T09:17:17.438296+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-09T09:17:18.673 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[77892]: cluster 2023-10-09T09:17:17.446643+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.673 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:18 smithi016 bash[77892]: cluster 2023-10-09T09:17:17.449114+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T09:17:18.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:18.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:18.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:18.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:18.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:18.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:18.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:19.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:19.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:19.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:19.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:19.205 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:19.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:19.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:19.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:19.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:19.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:19.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:19.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:19 smithi094 bash[59677]: cluster 2023-10-09T09:17:18.157270+0000 mgr.y (mgr.25421) 1202 : cluster 0 pgmap v1310: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 17 op/s; 6352/72 objects degraded (8822.222%); 3418/72 objects misplaced (4747.222%); 109 KiB/s, 194 objects/s recovering 2023-10-09T09:17:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:19 smithi094 bash[59677]: cluster 2023-10-09T09:17:18.445406+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:19 smithi094 bash[59677]: cluster 2023-10-09T09:17:18.446137+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:19 smithi094 bash[59677]: cluster 2023-10-09T09:17:18.448058+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-09T09:17:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:19 smithi094 bash[59677]: cluster 2023-10-09T09:17:18.453646+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:19 smithi094 bash[59677]: audit 2023-10-09T09:17:19.252378+0000 mon.a (mon.0) 1607 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:19.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:19.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:19.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:19.549 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:17:19.550 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:17:19.620 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:19.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:19.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:19.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:19.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:19.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:19.872 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:19.892 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[77892]: cluster 2023-10-09T09:17:18.157270+0000 mgr.y (mgr.25421) 1202 : cluster 0 pgmap v1310: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 17 op/s; 6352/72 objects degraded (8822.222%); 3418/72 objects misplaced (4747.222%); 109 KiB/s, 194 objects/s recovering 2023-10-09T09:17:19.892 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[77892]: cluster 2023-10-09T09:17:18.445406+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[77892]: cluster 2023-10-09T09:17:18.446137+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[77892]: cluster 2023-10-09T09:17:18.448058+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-09T09:17:19.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[77892]: cluster 2023-10-09T09:17:18.453646+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[77892]: audit 2023-10-09T09:17:19.252378+0000 mon.a (mon.0) 1607 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:19.893 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[75179]: cluster 2023-10-09T09:17:18.157270+0000 mgr.y (mgr.25421) 1202 : cluster 0 pgmap v1310: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 17 op/s; 6352/72 objects degraded (8822.222%); 3418/72 objects misplaced (4747.222%); 109 KiB/s, 194 objects/s recovering 2023-10-09T09:17:19.893 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[75179]: cluster 2023-10-09T09:17:18.445406+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[75179]: cluster 2023-10-09T09:17:18.446137+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[75179]: cluster 2023-10-09T09:17:18.448058+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-09T09:17:19.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[75179]: cluster 2023-10-09T09:17:18.453646+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T09:17:19.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:19 smithi016 bash[75179]: audit 2023-10-09T09:17:19.252378+0000 mon.a (mon.0) 1607 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:19.896 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:19.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:19.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:19.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:20.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:20.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:20.129 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:20.157 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:20.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:20.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:20.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:20.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:20.387 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:20.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:20.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:20.523 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:20 smithi016 bash[77892]: cluster 2023-10-09T09:17:19.773441+0000 mon.a (mon.0) 1608 : cluster 3 Health check update: Degraded data redundancy: 6352/72 objects degraded (8822.222%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:20.523 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:20 smithi016 bash[75179]: cluster 2023-10-09T09:17:19.773441+0000 mon.a (mon.0) 1608 : cluster 3 Health check update: Degraded data redundancy: 6352/72 objects degraded (8822.222%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:20.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:20.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:20.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:20.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:20.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:20 smithi094 bash[59677]: cluster 2023-10-09T09:17:19.773441+0000 mon.a (mon.0) 1608 : cluster 3 Health check update: Degraded data redundancy: 6352/72 objects degraded (8822.222%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:17:20.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:20.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:20.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:20.922 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:20.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:20.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:21.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:21.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:21.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:21.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:21.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:21.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:21.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:21.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:21.549 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:21.578 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:21.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:21.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:21 smithi016 bash[75179]: cluster 2023-10-09T09:17:20.158056+0000 mgr.y (mgr.25421) 1203 : cluster 0 pgmap v1312: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 6352/72 objects degraded (8822.222%); 3418/72 objects misplaced (4747.222%); 0 B/s, 240 objects/s recovering 2023-10-09T09:17:21.695 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:21 smithi016 bash[77892]: cluster 2023-10-09T09:17:20.158056+0000 mgr.y (mgr.25421) 1203 : cluster 0 pgmap v1312: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 6352/72 objects degraded (8822.222%); 3418/72 objects misplaced (4747.222%); 0 B/s, 240 objects/s recovering 2023-10-09T09:17:21.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:21.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:21 smithi094 bash[59677]: cluster 2023-10-09T09:17:20.158056+0000 mgr.y (mgr.25421) 1203 : cluster 0 pgmap v1312: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 6352/72 objects degraded (8822.222%); 3418/72 objects misplaced (4747.222%); 0 B/s, 240 objects/s recovering 2023-10-09T09:17:21.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:21.843 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:21.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:21.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:21.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:21.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:21.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:22.077 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:22.104 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:22.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:22.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:22.234 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:22.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:22.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:22.336 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:22.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:22.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:22.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:22.493 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:22.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:22.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:22.596 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:22.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:22.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:22.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:22.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:22.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:22.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:22.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:22.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:22.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:23.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:23.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:23.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:23.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:23.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:23.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:23.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:23.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:23.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:23.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:23.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:23 smithi016 bash[75179]: cluster 2023-10-09T09:17:22.158860+0000 mgr.y (mgr.25421) 1204 : cluster 0 pgmap v1313: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 86 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 140 objects/s recovering 2023-10-09T09:17:23.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:23 smithi016 bash[77892]: cluster 2023-10-09T09:17:22.158860+0000 mgr.y (mgr.25421) 1204 : cluster 0 pgmap v1313: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 86 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 140 objects/s recovering 2023-10-09T09:17:23.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:23.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:23.662 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:23.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:23 smithi094 bash[59677]: cluster 2023-10-09T09:17:22.158860+0000 mgr.y (mgr.25421) 1204 : cluster 0 pgmap v1313: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 86 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 140 objects/s recovering 2023-10-09T09:17:23.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:23.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:23.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:23.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:23.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:23.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:24.028 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:24.056 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:24.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:24.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:24.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:24.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:24.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:24.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:24.447 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:24.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:24.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:24.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:24.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:24.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:24.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:24.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:24.829 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:24.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:24.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:24.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:24.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:25.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:25.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:25.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:25.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:25.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:25.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:25.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:25.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:25.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:25.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:25.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:25.472 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:25 smithi016 bash[75179]: cluster 2023-10-09T09:17:24.159786+0000 mgr.y (mgr.25421) 1205 : cluster 0 pgmap v1314: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 140 objects/s recovering 2023-10-09T09:17:25.472 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:25 smithi016 bash[75179]: cluster 2023-10-09T09:17:24.774674+0000 mon.a (mon.0) 1609 : cluster 3 Health check update: Degraded data redundancy: 4990/72 objects degraded (6930.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:17:25.472 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:25 smithi016 bash[77892]: cluster 2023-10-09T09:17:24.159786+0000 mgr.y (mgr.25421) 1205 : cluster 0 pgmap v1314: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 140 objects/s recovering 2023-10-09T09:17:25.472 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:25 smithi016 bash[77892]: cluster 2023-10-09T09:17:24.774674+0000 mon.a (mon.0) 1609 : cluster 3 Health check update: Degraded data redundancy: 4990/72 objects degraded (6930.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:17:25.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:25.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:25.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:25.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:25.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:25.716 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:25 smithi094 bash[59677]: cluster 2023-10-09T09:17:24.159786+0000 mgr.y (mgr.25421) 1205 : cluster 0 pgmap v1314: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 140 objects/s recovering 2023-10-09T09:17:25.716 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:25 smithi094 bash[59677]: cluster 2023-10-09T09:17:24.774674+0000 mon.a (mon.0) 1609 : cluster 3 Health check update: Degraded data redundancy: 4990/72 objects degraded (6930.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:17:25.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:25.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:25.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:25.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:25.972 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:25.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:25.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:25.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:25.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:17:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:17:25] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.33.4" 2023-10-09T09:17:26.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:26.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:26.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:26.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:26.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:26.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:26.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:26.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:26.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:26.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:26.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:26.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:26.506 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:26.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:26.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:26.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:26.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:26.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:26.760 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:26.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:26.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:26.863 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:26.883 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:26.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:26.986 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:27.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:27.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:27.136 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:27.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:27.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:27.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:27.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:27.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:27.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:27.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:27.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:27.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:27 smithi016 bash[75179]: cluster 2023-10-09T09:17:26.160960+0000 mgr.y (mgr.25421) 1206 : cluster 0 pgmap v1315: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 53 objects/s recovering 2023-10-09T09:17:27.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:27 smithi016 bash[77892]: cluster 2023-10-09T09:17:26.160960+0000 mgr.y (mgr.25421) 1206 : cluster 0 pgmap v1315: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 53 objects/s recovering 2023-10-09T09:17:27.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:27.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:27.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:27.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:27.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:27.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:27 smithi094 bash[59677]: cluster 2023-10-09T09:17:26.160960+0000 mgr.y (mgr.25421) 1206 : cluster 0 pgmap v1315: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 53 objects/s recovering 2023-10-09T09:17:27.753 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:27.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:27.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:27.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:27.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:27.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:28.020 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:28.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:28.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:28.046 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:17:28.046 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:17:28.146 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:28.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:28.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:28.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:28.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:28.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:28.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:28.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:28.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:28.542 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:28.566 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:28.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:28.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:28.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:28.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:28.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:28.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:28.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:28.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:29.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:29.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:29.187 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:29.212 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:29.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:29.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:29.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:29.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:29.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:29.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:29.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:29.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:29.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:29.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:29.722 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:29 smithi016 bash[75179]: cluster 2023-10-09T09:17:28.162043+0000 mgr.y (mgr.25421) 1207 : cluster 0 pgmap v1316: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 46 objects/s recovering 2023-10-09T09:17:29.722 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:29 smithi016 bash[77892]: cluster 2023-10-09T09:17:28.162043+0000 mgr.y (mgr.25421) 1207 : cluster 0 pgmap v1316: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 46 objects/s recovering 2023-10-09T09:17:29.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:29.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:29.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:29 smithi094 bash[59677]: cluster 2023-10-09T09:17:28.162043+0000 mgr.y (mgr.25421) 1207 : cluster 0 pgmap v1316: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 46 objects/s recovering 2023-10-09T09:17:29.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:29.861 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:29.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:17:29.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:29.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:29.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:29.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:30.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:30.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:30.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:30.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:30.225 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:30.256 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:30.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:30.272 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:17:30.295 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:17:30.316 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:17:30.341 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:17:30.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:30.371 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:17:30.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:30.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:30.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:30.403 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:17:30.431 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:17:30.457 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:17:30.460 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.460 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.460 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.460 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.460 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.461 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.461 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.461 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:17:30.494 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:30.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T09:17:30.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:30.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:30.624 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:30.655 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:30.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:30.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:30.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:30.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:30.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:30.796 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:17:30.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:30.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:30.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:30.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[75179]: cluster 2023-10-09T09:17:30.162690+0000 mgr.y (mgr.25421) 1208 : cluster 0 pgmap v1317: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 39 objects/s recovering 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[75179]: audit 2023-10-09T09:17:30.245836+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.16:0/3770225184' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[75179]: audit 2023-10-09T09:17:30.246165+0000 mon.a (mon.0) 1610 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[75179]: audit 2023-10-09T09:17:30.248384+0000 mon.a (mon.0) 1611 : audit 0 from='client.? 172.21.15.16:0/712694652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[75179]: audit 2023-10-09T09:17:30.500278+0000 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.16:0/519737240' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[77892]: cluster 2023-10-09T09:17:30.162690+0000 mgr.y (mgr.25421) 1208 : cluster 0 pgmap v1317: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 39 objects/s recovering 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[77892]: audit 2023-10-09T09:17:30.245836+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.16:0/3770225184' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[77892]: audit 2023-10-09T09:17:30.246165+0000 mon.a (mon.0) 1610 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[77892]: audit 2023-10-09T09:17:30.248384+0000 mon.a (mon.0) 1611 : audit 0 from='client.? 172.21.15.16:0/712694652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:31.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:30 smithi016 bash[77892]: audit 2023-10-09T09:17:30.500278+0000 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.16:0/519737240' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:17:31.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:31.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:31.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:31.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:31.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:30 smithi094 bash[59677]: cluster 2023-10-09T09:17:30.162690+0000 mgr.y (mgr.25421) 1208 : cluster 0 pgmap v1317: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 39 objects/s recovering 2023-10-09T09:17:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:30 smithi094 bash[59677]: audit 2023-10-09T09:17:30.245836+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.16:0/3770225184' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:17:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:30 smithi094 bash[59677]: audit 2023-10-09T09:17:30.246165+0000 mon.a (mon.0) 1610 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:17:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:30 smithi094 bash[59677]: audit 2023-10-09T09:17:30.248384+0000 mon.a (mon.0) 1611 : audit 0 from='client.? 172.21.15.16:0/712694652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:30 smithi094 bash[59677]: audit 2023-10-09T09:17:30.500278+0000 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.16:0/519737240' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:17:31.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:31.311 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:31.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:31.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:31.542 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:31.572 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:31.676 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:31.700 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:31.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:31.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:31.816 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:31 smithi094 bash[59677]: cluster 2023-10-09T09:17:30.779836+0000 mon.a (mon.0) 1613 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:17:31.816 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:31 smithi094 bash[59677]: cluster 2023-10-09T09:17:30.788232+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T09:17:31.816 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:31 smithi094 bash[59677]: cluster 2023-10-09T09:17:30.788721+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T09:17:31.817 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:31 smithi094 bash[59677]: audit 2023-10-09T09:17:30.790494+0000 mon.a (mon.0) 1614 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:17:31.817 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:31 smithi094 bash[59677]: cluster 2023-10-09T09:17:30.790588+0000 mon.a (mon.0) 1615 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-09T09:17:31.829 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:31.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:31.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[77892]: cluster 2023-10-09T09:17:30.779836+0000 mon.a (mon.0) 1613 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:17:31.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[77892]: cluster 2023-10-09T09:17:30.788232+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T09:17:31.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[77892]: cluster 2023-10-09T09:17:30.788721+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T09:17:31.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[77892]: audit 2023-10-09T09:17:30.790494+0000 mon.a (mon.0) 1614 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:17:31.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[77892]: cluster 2023-10-09T09:17:30.790588+0000 mon.a (mon.0) 1615 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-09T09:17:31.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[75179]: cluster 2023-10-09T09:17:30.779836+0000 mon.a (mon.0) 1613 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:17:31.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[75179]: cluster 2023-10-09T09:17:30.788232+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T09:17:31.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[75179]: cluster 2023-10-09T09:17:30.788721+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T09:17:31.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[75179]: audit 2023-10-09T09:17:30.790494+0000 mon.a (mon.0) 1614 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:17:31.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:31 smithi016 bash[75179]: cluster 2023-10-09T09:17:30.790588+0000 mon.a (mon.0) 1615 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-09T09:17:31.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:31.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:32.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:32.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:32.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:32.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:32.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:32.340 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:32.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:32.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:32.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:32.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:32.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:32.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:32.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:32.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:32.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:32.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:32.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:32.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:32.973 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:32 smithi016 bash[75179]: cluster 2023-10-09T09:17:32.163357+0000 mgr.y (mgr.25421) 1209 : cluster 0 pgmap v1319: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 58 objects/s recovering 2023-10-09T09:17:32.973 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:32 smithi016 bash[77892]: cluster 2023-10-09T09:17:32.163357+0000 mgr.y (mgr.25421) 1209 : cluster 0 pgmap v1319: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 58 objects/s recovering 2023-10-09T09:17:32.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:32.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T09:17:33.049 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T09:17:33.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T09:17:33.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:33.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:33.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:33.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:32 smithi094 bash[59677]: cluster 2023-10-09T09:17:32.163357+0000 mgr.y (mgr.25421) 1209 : cluster 0 pgmap v1319: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 58 objects/s recovering 2023-10-09T09:17:33.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:33.296 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:33.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:33.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:33.424 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:33.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:33.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:33.526 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:33.556 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:33.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:33.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:33.660 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:33.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:33.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:33.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:33.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:33.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:33.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:33.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:33.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:33.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:34.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:34.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:34.281 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:34.309 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:34.412 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:34.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:34.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:34.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:34.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:34.682 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:34.702 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:17:34.710 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:34.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:34.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:34.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:34.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:34.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:34.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:34.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:34.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:34.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:17:35.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:17:35.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:35.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:35.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:35.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:35.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:35.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:35.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:35.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:35.346 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:35 smithi016 bash[75179]: cluster 2023-10-09T09:17:34.164236+0000 mgr.y (mgr.25421) 1210 : cluster 0 pgmap v1320: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 58 objects/s recovering 2023-10-09T09:17:35.346 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:35 smithi016 bash[75179]: audit 2023-10-09T09:17:34.253054+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:35.347 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:35 smithi016 bash[77892]: cluster 2023-10-09T09:17:34.164236+0000 mgr.y (mgr.25421) 1210 : cluster 0 pgmap v1320: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 58 objects/s recovering 2023-10-09T09:17:35.347 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:35 smithi016 bash[77892]: audit 2023-10-09T09:17:34.253054+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:35.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:35.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:35.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:35.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:35.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:35 smithi094 bash[59677]: cluster 2023-10-09T09:17:34.164236+0000 mgr.y (mgr.25421) 1210 : cluster 0 pgmap v1320: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 58 objects/s recovering 2023-10-09T09:17:35.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:35 smithi094 bash[59677]: audit 2023-10-09T09:17:34.253054+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:35.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:35.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:35.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:35.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:35.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:35.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:35.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:35.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:35.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:35.874 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:17:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:17:35] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T09:17:35.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:35.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:35.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:36.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:36.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:36.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:36.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:36.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:36.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:36.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:36.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:36.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:36.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:36.967 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:36.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:37.070 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:37.089 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:37.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:37.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:37.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:37.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:37.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:37.328 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:37.355 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:37.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:37.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:37.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:37.485 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:37 smithi094 bash[59677]: cluster 2023-10-09T09:17:36.165365+0000 mgr.y (mgr.25421) 1211 : cluster 0 pgmap v1321: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:37.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:37.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:37 smithi016 bash[75179]: cluster 2023-10-09T09:17:36.165365+0000 mgr.y (mgr.25421) 1211 : cluster 0 pgmap v1321: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:37.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:37 smithi016 bash[77892]: cluster 2023-10-09T09:17:36.165365+0000 mgr.y (mgr.25421) 1211 : cluster 0 pgmap v1321: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:37.586 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:37.611 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:37.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:37.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:37.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:37.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:37.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:37.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:37.879 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:17:37.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:37.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:37.986 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:38.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:38.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:38.050 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:17:38.050 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:17:38.050 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T09:17:38.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T09:17:38.117 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:38.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:38.246 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:38.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:38.380 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:38.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:38.508 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:38.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:38.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:38.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:38.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:38.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:38.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:38.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:38.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:38.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:38.895 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:17:38.897 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:17:38.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:38.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:38.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:39.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:39.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:39.156 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:39.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:39.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:39.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:39.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:39.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:39.428 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:39 smithi016 bash[75179]: cluster 2023-10-09T09:17:38.166006+0000 mgr.y (mgr.25421) 1212 : cluster 0 pgmap v1322: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:39.429 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:39 smithi016 bash[77892]: cluster 2023-10-09T09:17:38.166006+0000 mgr.y (mgr.25421) 1212 : cluster 0 pgmap v1322: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:39.443 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:39 smithi094 bash[59677]: cluster 2023-10-09T09:17:38.166006+0000 mgr.y (mgr.25421) 1212 : cluster 0 pgmap v1322: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:39.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:39.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:39.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:39.571 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:39.675 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:39.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:39.810 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:39.836 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:39.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:39.963 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:39.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:40.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:40.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:40.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:40.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:40.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:40.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:40.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:40.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:40.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:40.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:40.461 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:40.490 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:40.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:40.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:40.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:40.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:40.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:40.726 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:40.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:40.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:40.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:40.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:40.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:40.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:40.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:40.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:40 smithi094 bash[59677]: cluster 2023-10-09T09:17:40.166902+0000 mgr.y (mgr.25421) 1213 : cluster 0 pgmap v1323: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:41.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:41.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:41.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:41.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:41.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:41.237 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:41.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:41.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:41.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:40 smithi016 bash[75179]: cluster 2023-10-09T09:17:40.166902+0000 mgr.y (mgr.25421) 1213 : cluster 0 pgmap v1323: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:41.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:40 smithi016 bash[77892]: cluster 2023-10-09T09:17:40.166902+0000 mgr.y (mgr.25421) 1213 : cluster 0 pgmap v1323: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 57 objects/s recovering 2023-10-09T09:17:41.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:41.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:41.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:41.529 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:41.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:41.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:41.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:41.695 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:17:41.695 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:17:41.695 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:17:41.695 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:17:41.696 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:17:41.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:41.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:41.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:41.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:41.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:41.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:41.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:41.941 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:17:42.030 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:42.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:42.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:42.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:42.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:42.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:42.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:42.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:42.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:42.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:42.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:42.444 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:42.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:42.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:42.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:42.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:43.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:43.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:43.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:43.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:43.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:43.618 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:43 smithi016 bash[77892]: cluster 2023-10-09T09:17:42.168200+0000 mgr.y (mgr.25421) 1214 : cluster 0 pgmap v1324: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:43.618 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:43 smithi016 bash[75179]: cluster 2023-10-09T09:17:42.168200+0000 mgr.y (mgr.25421) 1214 : cluster 0 pgmap v1324: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:43.619 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:43 smithi094 bash[59677]: cluster 2023-10-09T09:17:42.168200+0000 mgr.y (mgr.25421) 1214 : cluster 0 pgmap v1324: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:43.620 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:17:43.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:43.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:43.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:43.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:43.778 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:43.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:43.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:43.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:43.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:44.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:44.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:44.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:44.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:44.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:44.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:44.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:44.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:44.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:44.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:44.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:44.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:44.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:44.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:44.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:44.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:44.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:44.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:44.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:45.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:45.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:45.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:45.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:45.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:45.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:45.459 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:45.487 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:45.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:45 smithi094 bash[59677]: cluster 2023-10-09T09:17:44.169000+0000 mgr.y (mgr.25421) 1215 : cluster 0 pgmap v1325: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 48 objects/s recovering 2023-10-09T09:17:45.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:45.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:45.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:45 smithi016 bash[75179]: cluster 2023-10-09T09:17:44.169000+0000 mgr.y (mgr.25421) 1215 : cluster 0 pgmap v1325: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 48 objects/s recovering 2023-10-09T09:17:45.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:45 smithi016 bash[77892]: cluster 2023-10-09T09:17:44.169000+0000 mgr.y (mgr.25421) 1215 : cluster 0 pgmap v1325: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4990/72 objects degraded (6930.556%); 3418/72 objects misplaced (4747.222%); 0 B/s, 48 objects/s recovering 2023-10-09T09:17:45.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:45.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:45.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:45.748 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:45.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:45.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:45.851 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:45.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:45.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:45.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:45.894 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:17:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:17:45] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T09:17:45.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:46.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:46.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:46.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:46.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:46.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:46.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:46.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:46.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:46 smithi016 bash[75179]: cluster 2023-10-09T09:17:46.227037+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: Degraded data redundancy: 4700/72 objects degraded (6527.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:17:46.415 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:46 smithi016 bash[77892]: cluster 2023-10-09T09:17:46.227037+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: Degraded data redundancy: 4700/72 objects degraded (6527.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:17:46.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:46.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:46.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:46.646 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:46.656 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:46 smithi094 bash[59677]: cluster 2023-10-09T09:17:46.227037+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: Degraded data redundancy: 4700/72 objects degraded (6527.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:17:46.673 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:46.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:46.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:46.803 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:46.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:46.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:46.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:46.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:47.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:47.068 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:47.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:47.172 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:47.198 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:47.301 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:47.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:47 smithi094 bash[59677]: cluster 2023-10-09T09:17:46.169842+0000 mgr.y (mgr.25421) 1216 : cluster 0 pgmap v1326: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 74 objects/s recovering 2023-10-09T09:17:47.329 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:47.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:47.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:47.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:47.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:47.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:47.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:47 smithi016 bash[75179]: cluster 2023-10-09T09:17:46.169842+0000 mgr.y (mgr.25421) 1216 : cluster 0 pgmap v1326: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 74 objects/s recovering 2023-10-09T09:17:47.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:47 smithi016 bash[77892]: cluster 2023-10-09T09:17:46.169842+0000 mgr.y (mgr.25421) 1216 : cluster 0 pgmap v1326: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 74 objects/s recovering 2023-10-09T09:17:47.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:47.683 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:47.709 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:47.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:47.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:47.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:47.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:47.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:47.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:48.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:48.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:48.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:48.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:48.241 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:48.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:48.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:48.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:48.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:48.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:48.390 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:17:48.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:48.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:48.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:48.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:48.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:48.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:48.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:48.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:48.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:48.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:48.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:48.868 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:48.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:48.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:48.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:49.002 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:49.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:49.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:49.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:49.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:49.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:49.282 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:49.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:49.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:49 smithi016 bash[77892]: cluster 2023-10-09T09:17:48.170881+0000 mgr.y (mgr.25421) 1217 : cluster 0 pgmap v1327: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:49.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:49 smithi016 bash[77892]: audit 2023-10-09T09:17:49.252784+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:49.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:49.414 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:49.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:49 smithi094 bash[59677]: cluster 2023-10-09T09:17:48.170881+0000 mgr.y (mgr.25421) 1217 : cluster 0 pgmap v1327: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:49.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:49 smithi094 bash[59677]: audit 2023-10-09T09:17:49.252784+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:49.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:49.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:49.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:49 smithi016 bash[75179]: cluster 2023-10-09T09:17:48.170881+0000 mgr.y (mgr.25421) 1217 : cluster 0 pgmap v1327: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:49.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:49 smithi016 bash[75179]: audit 2023-10-09T09:17:49.252784+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:49.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:49.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:49.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:49.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:49.801 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:49.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:49.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:49.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:50.037 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:50.060 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:50.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:50.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:50.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:50.287 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:50.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:50.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:50.446 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:50.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:50.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:50.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:50.709 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:50.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:50.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:50.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:50 smithi016 bash[75179]: cluster 2023-10-09T09:17:50.171655+0000 mgr.y (mgr.25421) 1218 : cluster 0 pgmap v1328: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:50.820 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:50 smithi016 bash[77892]: cluster 2023-10-09T09:17:50.171655+0000 mgr.y (mgr.25421) 1218 : cluster 0 pgmap v1328: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:50.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:50.936 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:50.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:50 smithi094 bash[59677]: cluster 2023-10-09T09:17:50.171655+0000 mgr.y (mgr.25421) 1218 : cluster 0 pgmap v1328: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 4700/72 objects degraded (6527.778%); 3418/72 objects misplaced (4747.222%); 0 B/s, 51 objects/s recovering 2023-10-09T09:17:50.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:50.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:50.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:51.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:51.083 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:51.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:51.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.209 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:51.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.339 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:51.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.468 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:51.572 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:51.594 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:51.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:51.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.728 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:51.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:51.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:51.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:51.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:51.976 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:51.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:52.059 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_4' removed 2023-10-09T09:17:52.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:52.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:52.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:52.157 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:52 smithi016 bash[75179]: audit 2023-10-09T09:17:51.997276+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.16:0/1172958767' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:17:52.157 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:52 smithi016 bash[75179]: audit 2023-10-09T09:17:51.997744+0000 mon.a (mon.0) 1619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:17:52.157 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:52 smithi016 bash[77892]: audit 2023-10-09T09:17:51.997276+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.16:0/1172958767' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:17:52.157 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:52 smithi016 bash[77892]: audit 2023-10-09T09:17:51.997744+0000 mon.a (mon.0) 1619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:17:52.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:52.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:52.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:52 smithi094 bash[59677]: audit 2023-10-09T09:17:51.997276+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.16:0/1172958767' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:17:52.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:52 smithi094 bash[59677]: audit 2023-10-09T09:17:51.997744+0000 mon.a (mon.0) 1619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:17:52.325 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:52.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:52.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:52.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:52.489 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:52.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:52.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:52.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:52.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:52.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:52.870 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:52.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:52.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:52.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:52.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:53.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:53.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:53.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:53.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:53.230 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:53.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:53.358 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:53 smithi094 bash[59677]: cluster 2023-10-09T09:17:52.051366+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.370 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:53 smithi094 bash[59677]: cluster 2023-10-09T09:17:52.051900+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.370 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:53 smithi094 bash[59677]: audit 2023-10-09T09:17:52.052400+0000 mon.a (mon.0) 1620 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:17:53.370 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:53 smithi094 bash[59677]: cluster 2023-10-09T09:17:52.052479+0000 mon.a (mon.0) 1621 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-09T09:17:53.370 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:53 smithi094 bash[59677]: cluster 2023-10-09T09:17:52.172145+0000 mgr.y (mgr.25421) 1219 : cluster 0 pgmap v1330: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:53.370 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:53 smithi094 bash[59677]: cluster 2023-10-09T09:17:52.700808+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:53.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:53.434 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[77892]: cluster 2023-10-09T09:17:52.051366+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.434 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[77892]: cluster 2023-10-09T09:17:52.051900+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[77892]: audit 2023-10-09T09:17:52.052400+0000 mon.a (mon.0) 1620 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[77892]: cluster 2023-10-09T09:17:52.052479+0000 mon.a (mon.0) 1621 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[77892]: cluster 2023-10-09T09:17:52.172145+0000 mgr.y (mgr.25421) 1219 : cluster 0 pgmap v1330: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[77892]: cluster 2023-10-09T09:17:52.700808+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[75179]: cluster 2023-10-09T09:17:52.051366+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[75179]: cluster 2023-10-09T09:17:52.051900+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[75179]: audit 2023-10-09T09:17:52.052400+0000 mon.a (mon.0) 1620 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[75179]: cluster 2023-10-09T09:17:52.052479+0000 mon.a (mon.0) 1621 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-09T09:17:53.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[75179]: cluster 2023-10-09T09:17:52.172145+0000 mgr.y (mgr.25421) 1219 : cluster 0 pgmap v1330: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:53.436 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:53 smithi016 bash[75179]: cluster 2023-10-09T09:17:52.700808+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T09:17:53.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:53.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:53.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:53.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:53.624 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:53.649 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:53.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:53.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:53.729 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:17:53.751 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:17:53.752 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:53.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:53.777 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:17:53.805 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:17:53.829 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:17:53.855 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:17:53.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.877 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:17:53.899 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:17:53.899 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:53.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:53.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:17:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:17:54.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:54.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:54.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:54.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:54.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:54.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:54 smithi094 bash[59677]: cluster 2023-10-09T09:17:53.058298+0000 mon.a (mon.0) 1622 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4700/72 objects degraded (6527.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T09:17:54.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:54 smithi094 bash[59677]: audit 2023-10-09T09:17:53.704604+0000 mon.a (mon.0) 1623 : audit 0 from='client.? 172.21.15.16:0/1989114250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:54.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:54.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:54.397 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:54.428 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:54.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:54.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:54 smithi016 bash[75179]: cluster 2023-10-09T09:17:53.058298+0000 mon.a (mon.0) 1622 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4700/72 objects degraded (6527.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T09:17:54.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:54 smithi016 bash[75179]: audit 2023-10-09T09:17:53.704604+0000 mon.a (mon.0) 1623 : audit 0 from='client.? 172.21.15.16:0/1989114250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:54.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:54 smithi016 bash[77892]: cluster 2023-10-09T09:17:53.058298+0000 mon.a (mon.0) 1622 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4700/72 objects degraded (6527.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T09:17:54.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:54 smithi016 bash[77892]: audit 2023-10-09T09:17:53.704604+0000 mon.a (mon.0) 1623 : audit 0 from='client.? 172.21.15.16:0/1989114250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:17:54.531 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:54.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:54.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:54.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:54.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:54.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:54.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:54.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:54.930 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:54.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:54.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:54.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:55.064 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:55.088 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:55.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:55.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:55.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:55.203 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:55 smithi094 bash[59677]: cluster 2023-10-09T09:17:54.172703+0000 mgr.y (mgr.25421) 1220 : cluster 0 pgmap v1331: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:55.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:55.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:55.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:55.266 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:17:55.267 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:17:55.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:55.340 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:55 smithi016 bash[75179]: cluster 2023-10-09T09:17:54.172703+0000 mgr.y (mgr.25421) 1220 : cluster 0 pgmap v1331: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:55.340 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:55 smithi016 bash[77892]: cluster 2023-10-09T09:17:54.172703+0000 mgr.y (mgr.25421) 1220 : cluster 0 pgmap v1331: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:55.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:55.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:55.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:55.462 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:55.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:55.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:55.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:55.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:55.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:55.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:55.680 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:17:55.681 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:17:55.681 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T09:17:55.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T09:17:55.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:55.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:17:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:55.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:17:55.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:55.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:55.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:55.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:55.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:17:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:17:55] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-10-09T09:17:56.020 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:56.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:56.123 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:56.157 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:56.174 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:17:56.207 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:17:56.260 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:56.293 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:56.397 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:56.409 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:56 smithi094 bash[59677]: audit 2023-10-09T09:17:56.143639+0000 mon.a (mon.0) 1624 : audit 1 from='client.? 172.21.15.16:0/2433276846' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:56.409 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:56 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.194350+0000 mon.a (mon.0) 1625 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:17:56.409 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:56 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.194401+0000 mon.a (mon.0) 1626 : cluster 1 Cluster is now healthy 2023-10-09T09:17:56.428 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:56.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:56 smithi016 bash[75179]: audit 2023-10-09T09:17:56.143639+0000 mon.a (mon.0) 1624 : audit 1 from='client.? 172.21.15.16:0/2433276846' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:56.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:56 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.194350+0000 mon.a (mon.0) 1625 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:17:56.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:56 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.194401+0000 mon.a (mon.0) 1626 : cluster 1 Cluster is now healthy 2023-10-09T09:17:56.488 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:56 smithi016 bash[77892]: audit 2023-10-09T09:17:56.143639+0000 mon.a (mon.0) 1624 : audit 1 from='client.? 172.21.15.16:0/2433276846' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:17:56.488 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:56 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.194350+0000 mon.a (mon.0) 1625 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:17:56.488 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:56 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.194401+0000 mon.a (mon.0) 1626 : cluster 1 Cluster is now healthy 2023-10-09T09:17:56.530 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:56.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:56.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:56.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:56.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:56.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:56.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:56.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:56.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:56.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:56.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:56.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:57.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:57.076 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:57.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:57.180 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:57.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:57.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:57.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:57.245 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.2. 2023-10-09T09:17:57.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:57.341 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:57.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:57.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:57.395 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: debug 2023-10-09T09:17:57.190+0000 7fa046a4d700 -1 mon.a@0(leader).osd e291 definitely_dead 0 2023-10-09T09:17:57.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:57.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:57.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:57.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:57 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.173495+0000 mgr.y (mgr.25421) 1221 : cluster 0 pgmap v1332: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:57 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.197671+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:57 smithi094 bash[59677]: audit 2023-10-09T09:17:56.199022+0000 mon.a (mon.0) 1627 : audit 1 from='client.? 172.21.15.16:0/2433276846' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:17:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:57 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.199087+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-09T09:17:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:57 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.440662+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:57 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.453941+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:57 smithi094 bash[59677]: audit 2023-10-09T09:17:57.193587+0000 mon.a (mon.0) 1629 : audit 0 from='client.? 172.21.15.16:0/2267679588' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:17:57.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:57.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:57.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:57.708 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:57.744 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:57.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:57.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:57.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.173495+0000 mgr.y (mgr.25421) 1221 : cluster 0 pgmap v1332: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:57.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.197671+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: audit 2023-10-09T09:17:56.199022+0000 mon.a (mon.0) 1627 : audit 1 from='client.? 172.21.15.16:0/2433276846' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.199087+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.440662+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.453941+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[75179]: audit 2023-10-09T09:17:57.193587+0000 mon.a (mon.0) 1629 : audit 0 from='client.? 172.21.15.16:0/2267679588' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.173495+0000 mgr.y (mgr.25421) 1221 : cluster 0 pgmap v1332: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.197671+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[77892]: audit 2023-10-09T09:17:56.199022+0000 mon.a (mon.0) 1627 : audit 1 from='client.? 172.21.15.16:0/2433276846' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:17:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.199087+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-09T09:17:57.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.440662+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.453941+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:57.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:57 smithi016 bash[77892]: audit 2023-10-09T09:17:57.193587+0000 mon.a (mon.0) 1629 : audit 0 from='client.? 172.21.15.16:0/2267679588' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:17:57.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:57.877 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:17:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:57.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:17:57.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:58.012 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:58.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:58.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:58.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:58.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:58.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:58.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:58.316 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[77892]: cluster 2023-10-09T09:17:56.453848+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:58.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[77892]: cluster 2023-10-09T09:17:57.229113+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[77892]: audit 2023-10-09T09:17:57.230151+0000 mon.a (mon.0) 1630 : audit 1 from='client.? 172.21.15.16:0/2267679588' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:17:58.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[77892]: cluster 2023-10-09T09:17:57.230223+0000 mon.a (mon.0) 1631 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-10-09T09:17:58.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[77892]: cluster 2023-10-09T09:17:57.233266+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.317 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[77892]: cluster 2023-10-09T09:17:57.235368+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.317 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[75179]: cluster 2023-10-09T09:17:56.453848+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:58.318 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[75179]: cluster 2023-10-09T09:17:57.229113+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.318 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[75179]: audit 2023-10-09T09:17:57.230151+0000 mon.a (mon.0) 1630 : audit 1 from='client.? 172.21.15.16:0/2267679588' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:17:58.318 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[75179]: cluster 2023-10-09T09:17:57.230223+0000 mon.a (mon.0) 1631 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-10-09T09:17:58.318 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[75179]: cluster 2023-10-09T09:17:57.233266+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.318 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:58 smithi016 bash[75179]: cluster 2023-10-09T09:17:57.235368+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:58.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:58 smithi094 bash[59677]: cluster 2023-10-09T09:17:56.453848+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T09:17:58.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:58 smithi094 bash[59677]: cluster 2023-10-09T09:17:57.229113+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:58 smithi094 bash[59677]: audit 2023-10-09T09:17:57.230151+0000 mon.a (mon.0) 1630 : audit 1 from='client.? 172.21.15.16:0/2267679588' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:17:58.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:58 smithi094 bash[59677]: cluster 2023-10-09T09:17:57.230223+0000 mon.a (mon.0) 1631 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-10-09T09:17:58.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:58 smithi094 bash[59677]: cluster 2023-10-09T09:17:57.233266+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.387 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:58 smithi094 bash[59677]: cluster 2023-10-09T09:17:57.235368+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T09:17:58.404 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:58.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:58.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:58.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:58.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:58.644 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:58.675 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:58.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:58.809 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:17:58.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:17:58.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:58.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:58.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:17:58.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:58.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:59.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:59.072 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:17:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:59.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:17:59.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:59.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:17:59.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:59.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:17:59.256 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T09:17:59.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:59.317 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.173989+0000 mgr.y (mgr.25421) 1222 : cluster 0 pgmap v1335: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:59.317 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.237237+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.317 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.237881+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.317 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.241665+0000 mon.a (mon.0) 1632 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-10-09T09:17:59.317 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.242199+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.318 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.242256+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.318 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.243241+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.318 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:17:59 smithi094 bash[59677]: cluster 2023-10-09T09:17:58.245344+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.337 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:59.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:59.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:59.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:59.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.173989+0000 mgr.y (mgr.25421) 1222 : cluster 0 pgmap v1335: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.237237+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.237881+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.241665+0000 mon.a (mon.0) 1632 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.242199+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.242256+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.243241+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[75179]: cluster 2023-10-09T09:17:58.245344+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.173989+0000 mgr.y (mgr.25421) 1222 : cluster 0 pgmap v1335: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:17:59.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.237237+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.237881+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.241665+0000 mon.a (mon.0) 1632 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-10-09T09:17:59.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.242199+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.242256+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.243241+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:17:59 smithi016 bash[77892]: cluster 2023-10-09T09:17:58.245344+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T09:17:59.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:59.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:59.576 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:59.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:17:59.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:59.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:17:59.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:59.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:17:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:59.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:17:59.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:59.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:17:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:17:59.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:17:59.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:00.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:00.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:00.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:00.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:00.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:00.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:00.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:18:00.392 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:00.420 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:00.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:00.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:00.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:00.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:00.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:00.793 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:00.822 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:00.926 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:00.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:00.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:00.995 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:18:01.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:01.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:01.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:01.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:01.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:01.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:01.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:01.345 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:01 smithi016 bash[75179]: cluster 2023-10-09T09:18:00.174658+0000 mgr.y (mgr.25421) 1223 : cluster 0 pgmap v1337: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:01.345 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:01 smithi016 bash[75179]: audit 2023-10-09T09:18:00.206132+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:01.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:01 smithi016 bash[77892]: cluster 2023-10-09T09:18:00.174658+0000 mgr.y (mgr.25421) 1223 : cluster 0 pgmap v1337: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:01.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:01 smithi016 bash[77892]: audit 2023-10-09T09:18:00.206132+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:01.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:01.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:01.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:01.480 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:01 smithi094 bash[59677]: cluster 2023-10-09T09:18:00.174658+0000 mgr.y (mgr.25421) 1223 : cluster 0 pgmap v1337: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:01.480 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:01 smithi094 bash[59677]: audit 2023-10-09T09:18:00.206132+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:01.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:01.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:01.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:01.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:01.627 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:01.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:01.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:01.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:01.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:01.771 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:18:01.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:01.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:01.874 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:01.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:01.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:02.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:02.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:02.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:02.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:02.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:02.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:02.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:02.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:02.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:02.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:02.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:02.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:03.027 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:03.130 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:03.158 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:03.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:03.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:03.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:03.416 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:03 smithi094 bash[59677]: cluster 2023-10-09T09:18:02.175648+0000 mgr.y (mgr.25421) 1224 : cluster 0 pgmap v1338: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:03.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:03.463 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:03 smithi016 bash[75179]: cluster 2023-10-09T09:18:02.175648+0000 mgr.y (mgr.25421) 1224 : cluster 0 pgmap v1338: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:03.463 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:03 smithi016 bash[77892]: cluster 2023-10-09T09:18:02.175648+0000 mgr.y (mgr.25421) 1224 : cluster 0 pgmap v1338: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:03.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:03.567 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:03.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:03.704 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:03.807 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:03.811 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:18:03.833 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:03.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:03.937 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:03.968 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:03.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:04.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:04.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:04.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:04.228 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:04.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:04.257 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:18:04.257 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:18:04.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:04.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:04.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.493 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:04.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:04.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:04.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:04.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:04.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:04.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:04.861 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:04.889 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:04.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:04.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:04.993 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:05.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:05.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:05.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:05.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:05.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:05.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:05.289 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:05.375 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:18:05.375 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:18:05.392 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:05.407 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:05 smithi016 bash[75179]: cluster 2023-10-09T09:18:04.176391+0000 mgr.y (mgr.25421) 1225 : cluster 0 pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:05.407 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:05 smithi016 bash[75179]: audit 2023-10-09T09:18:04.257320+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:05.407 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:05 smithi016 bash[75179]: audit 2023-10-09T09:18:04.259301+0000 mon.a (mon.0) 1635 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:05.419 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:05.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:05 smithi094 bash[59677]: cluster 2023-10-09T09:18:04.176391+0000 mgr.y (mgr.25421) 1225 : cluster 0 pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:05.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:05 smithi094 bash[59677]: audit 2023-10-09T09:18:04.257320+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:05.421 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:05 smithi094 bash[59677]: audit 2023-10-09T09:18:04.259301+0000 mon.a (mon.0) 1635 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:05.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:05.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:05.551 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:18:05.552 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:05.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:05.656 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:05.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:05 smithi016 bash[77892]: cluster 2023-10-09T09:18:04.176391+0000 mgr.y (mgr.25421) 1225 : cluster 0 pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:05.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:05 smithi016 bash[77892]: audit 2023-10-09T09:18:04.257320+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:05.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:05 smithi016 bash[77892]: audit 2023-10-09T09:18:04.259301+0000 mon.a (mon.0) 1635 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:05.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:05.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:05.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:05.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:05.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:05.935 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:18:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:18:05] "GET /metrics HTTP/1.1" 200 34049 "" "Prometheus/2.33.4" 2023-10-09T09:18:05.951 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:05.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:05.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:06.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:06.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:06.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:06.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:06.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:06.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:06.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:06.357 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:06.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:06.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:06.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:06.470 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:18:06.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:06.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:06.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:06.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:06.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:06.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:06.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:06.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:06.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:06.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:06.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:06.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:06.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:07.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:07.109 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:07.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:07.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:07.278 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:07.304 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:07 smithi016 bash[75179]: cluster 2023-10-09T09:18:06.177275+0000 mgr.y (mgr.25421) 1226 : cluster 0 pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 65 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:07.382 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:07.394 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:07 smithi094 bash[59677]: cluster 2023-10-09T09:18:06.177275+0000 mgr.y (mgr.25421) 1226 : cluster 0 pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 65 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:07.414 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:07.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:07.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:07.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:07.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:07 smithi016 bash[77892]: cluster 2023-10-09T09:18:06.177275+0000 mgr.y (mgr.25421) 1226 : cluster 0 pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 65 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:18:07.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:07.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:07.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:07.792 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:07.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:07.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:07.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:07.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:07.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:08.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:08.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:08.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:08.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:08.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:08.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:08.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:08.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:08.430 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:08 smithi016 bash[75179]: audit 2023-10-09T09:18:07.493187+0000 mon.a (mon.0) 1636 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:18:08.430 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:08 smithi016 bash[75179]: audit 2023-10-09T09:18:07.868681+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:08.430 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:08 smithi016 bash[75179]: audit 2023-10-09T09:18:07.875918+0000 mon.a (mon.0) 1638 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:08.430 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:08 smithi016 bash[77892]: audit 2023-10-09T09:18:07.493187+0000 mon.a (mon.0) 1636 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:18:08.430 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:08 smithi016 bash[77892]: audit 2023-10-09T09:18:07.868681+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:08.431 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:08 smithi016 bash[77892]: audit 2023-10-09T09:18:07.875918+0000 mon.a (mon.0) 1638 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:08.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:08.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:08.563 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:08 smithi094 bash[59677]: audit 2023-10-09T09:18:07.493187+0000 mon.a (mon.0) 1636 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:18:08.563 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:08 smithi094 bash[59677]: audit 2023-10-09T09:18:07.868681+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:08.563 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:08 smithi094 bash[59677]: audit 2023-10-09T09:18:07.875918+0000 mon.a (mon.0) 1638 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:08.583 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:08.686 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:08.718 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:08.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:08.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:08.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:08.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:08.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:09.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:09.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:09.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:09.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:09.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:09.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:09.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:09.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:09.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:09 smithi016 bash[75179]: cluster 2023-10-09T09:18:08.177878+0000 mgr.y (mgr.25421) 1227 : cluster 0 pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 58 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T09:18:09.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:09 smithi016 bash[75179]: audit 2023-10-09T09:18:08.842953+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.16:0/1198735002' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:18:09.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:09 smithi016 bash[77892]: cluster 2023-10-09T09:18:08.177878+0000 mgr.y (mgr.25421) 1227 : cluster 0 pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 58 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T09:18:09.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:09 smithi016 bash[77892]: audit 2023-10-09T09:18:08.842953+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.16:0/1198735002' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:18:09.369 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_5' created 2023-10-09T09:18:09.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:09.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:09.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:09.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:09 smithi094 bash[59677]: cluster 2023-10-09T09:18:08.177878+0000 mgr.y (mgr.25421) 1227 : cluster 0 pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 58 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T09:18:09.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:09 smithi094 bash[59677]: audit 2023-10-09T09:18:08.842953+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.16:0/1198735002' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:18:09.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:09.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:09.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:09.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:09.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:09.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:09.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:09.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:09.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:09.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:09.865 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:09.899 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:09.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:09.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:10.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:10.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:10.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:10.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:10.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:10.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:10.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:10.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:10.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:10.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:10.423 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:10.526 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:10.539 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: audit 2023-10-09T09:18:08.843474+0000 mon.a (mon.0) 1639 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:18:10.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: cluster 2023-10-09T09:18:09.355685+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: cluster 2023-10-09T09:18:09.358584+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: cluster 2023-10-09T09:18:09.359373+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: audit 2023-10-09T09:18:09.359595+0000 mon.a (mon.0) 1640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T09:18:10.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: cluster 2023-10-09T09:18:09.359875+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-10-09T09:18:10.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: cluster 2023-10-09T09:18:09.365524+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:10 smithi094 bash[59677]: audit 2023-10-09T09:18:10.222494+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:10.558 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:10.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:10.661 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:10.691 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:10.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: audit 2023-10-09T09:18:08.843474+0000 mon.a (mon.0) 1639 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:18:10.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: cluster 2023-10-09T09:18:09.355685+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: cluster 2023-10-09T09:18:09.358584+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: cluster 2023-10-09T09:18:09.359373+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: audit 2023-10-09T09:18:09.359595+0000 mon.a (mon.0) 1640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: cluster 2023-10-09T09:18:09.359875+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: cluster 2023-10-09T09:18:09.365524+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[77892]: audit 2023-10-09T09:18:10.222494+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: audit 2023-10-09T09:18:08.843474+0000 mon.a (mon.0) 1639 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: cluster 2023-10-09T09:18:09.355685+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: cluster 2023-10-09T09:18:09.358584+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: cluster 2023-10-09T09:18:09.359373+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: audit 2023-10-09T09:18:09.359595+0000 mon.a (mon.0) 1640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T09:18:10.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: cluster 2023-10-09T09:18:09.359875+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-10-09T09:18:10.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: cluster 2023-10-09T09:18:09.365524+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T09:18:10.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:10 smithi016 bash[75179]: audit 2023-10-09T09:18:10.222494+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:10.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:10.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:10.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:10.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:10.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:10.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:10.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:11.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:11.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:11.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:11.190 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:11.218 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:11.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:11.252 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:18:11.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:11.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:11.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:11.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:11.456 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:11.478 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:11.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:11.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:11.581 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:11.589 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:11 smithi094 bash[59677]: cluster 2023-10-09T09:18:10.178531+0000 mgr.y (mgr.25421) 1228 : cluster 0 pgmap v1343: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 58 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T09:18:11.589 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:11 smithi094 bash[59677]: cluster 2023-10-09T09:18:10.359700+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-10-09T09:18:11.589 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:11 smithi094 bash[59677]: cluster 2023-10-09T09:18:10.361084+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.589 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:11 smithi094 bash[59677]: cluster 2023-10-09T09:18:10.361569+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.589 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:11 smithi094 bash[59677]: cluster 2023-10-09T09:18:10.361898+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:11.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:11.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:11.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:11.729 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:11.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[75179]: cluster 2023-10-09T09:18:10.178531+0000 mgr.y (mgr.25421) 1228 : cluster 0 pgmap v1343: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 58 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T09:18:11.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[75179]: cluster 2023-10-09T09:18:10.359700+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-10-09T09:18:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[75179]: cluster 2023-10-09T09:18:10.361084+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[75179]: cluster 2023-10-09T09:18:10.361569+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[75179]: cluster 2023-10-09T09:18:10.361898+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[77892]: cluster 2023-10-09T09:18:10.178531+0000 mgr.y (mgr.25421) 1228 : cluster 0 pgmap v1343: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 58 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T09:18:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[77892]: cluster 2023-10-09T09:18:10.359700+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-10-09T09:18:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[77892]: cluster 2023-10-09T09:18:10.361084+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[77892]: cluster 2023-10-09T09:18:10.361569+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:11 smithi016 bash[77892]: cluster 2023-10-09T09:18:10.361898+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:11.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:11.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:11.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:11.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:11.970 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:12.003 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:12.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:12.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:12.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:12.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:12.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T09:18:12.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:12.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:12.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:12.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:12.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:12.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:12.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:12.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:12.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:12.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[75179]: cluster 2023-10-09T09:18:10.557722+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:12.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[75179]: cluster 2023-10-09T09:18:11.359678+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T09:18:12.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[75179]: cluster 2023-10-09T09:18:11.360210+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T09:18:12.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[75179]: cluster 2023-10-09T09:18:11.361065+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-10-09T09:18:12.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[77892]: cluster 2023-10-09T09:18:10.557722+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:12.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[77892]: cluster 2023-10-09T09:18:11.359678+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T09:18:12.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[77892]: cluster 2023-10-09T09:18:11.360210+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T09:18:12.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:12 smithi016 bash[77892]: cluster 2023-10-09T09:18:11.361065+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-10-09T09:18:12.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:12 smithi094 bash[59677]: cluster 2023-10-09T09:18:10.557722+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T09:18:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:12 smithi094 bash[59677]: cluster 2023-10-09T09:18:11.359678+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T09:18:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:12 smithi094 bash[59677]: cluster 2023-10-09T09:18:11.360210+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T09:18:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:12 smithi094 bash[59677]: cluster 2023-10-09T09:18:11.361065+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-10-09T09:18:12.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:12.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:12.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:12.914 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:12.940 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:12.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:12.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:13.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:13.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:13.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:13.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:13.235 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:13.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:13.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:13.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:13.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:13.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:13.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:13.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:13.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:13 smithi094 bash[59677]: cluster 2023-10-09T09:18:12.179156+0000 mgr.y (mgr.25421) 1229 : cluster 0 pgmap v1346: 89 pgs: 5 creating+activating, 3 unknown, 81 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:13.497 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:13.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:13.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:13.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:13 smithi016 bash[75179]: cluster 2023-10-09T09:18:12.179156+0000 mgr.y (mgr.25421) 1229 : cluster 0 pgmap v1346: 89 pgs: 5 creating+activating, 3 unknown, 81 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:13.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:13 smithi016 bash[77892]: cluster 2023-10-09T09:18:12.179156+0000 mgr.y (mgr.25421) 1229 : cluster 0 pgmap v1346: 89 pgs: 5 creating+activating, 3 unknown, 81 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:13.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:13.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:13.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:13.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:13.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:13.765 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:13.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:13.868 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:13.894 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:13.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:13.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:13.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:14.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:14.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:14.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:14.264 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:14.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:14.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:14.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:14.407 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:18:14.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:14.436 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:18:14.463 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:18:14.487 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:18:14.516 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:18:14.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:14.540 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:18:14.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:14.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:14.562 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:18:14.589 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:18:14.591 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.591 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.591 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.591 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.591 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.591 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.592 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.592 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:18:14.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:14.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:14.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:14.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:14.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:14.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:14.910 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:14.940 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:14.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:15.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:15.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:15.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:15.174 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:15.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:14 smithi094 bash[59677]: audit 2023-10-09T09:18:13.963759+0000 mon.a (mon.0) 1645 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:15.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:14 smithi094 bash[59677]: audit 2023-10-09T09:18:13.970324+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:15.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:14 smithi094 bash[59677]: cluster 2023-10-09T09:18:14.179706+0000 mgr.y (mgr.25421) 1230 : cluster 0 pgmap v1347: 89 pgs: 5 creating+activating, 84 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:15.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:14 smithi094 bash[59677]: audit 2023-10-09T09:18:14.382387+0000 mon.a (mon.0) 1647 : audit 0 from='client.? 172.21.15.16:0/3266693544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:18:15.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:15.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[77892]: audit 2023-10-09T09:18:13.963759+0000 mon.a (mon.0) 1645 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:15.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[77892]: audit 2023-10-09T09:18:13.970324+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:15.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[77892]: cluster 2023-10-09T09:18:14.179706+0000 mgr.y (mgr.25421) 1230 : cluster 0 pgmap v1347: 89 pgs: 5 creating+activating, 84 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:15.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[77892]: audit 2023-10-09T09:18:14.382387+0000 mon.a (mon.0) 1647 : audit 0 from='client.? 172.21.15.16:0/3266693544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:18:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[75179]: audit 2023-10-09T09:18:13.963759+0000 mon.a (mon.0) 1645 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[75179]: audit 2023-10-09T09:18:13.970324+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[75179]: cluster 2023-10-09T09:18:14.179706+0000 mgr.y (mgr.25421) 1230 : cluster 0 pgmap v1347: 89 pgs: 5 creating+activating, 84 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:18:15.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:14 smithi016 bash[75179]: audit 2023-10-09T09:18:14.382387+0000 mon.a (mon.0) 1647 : audit 0 from='client.? 172.21.15.16:0/3266693544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:18:15.306 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:15.335 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:15.383 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:18:15.384 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:18:15.384 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:18:15.384 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:18:15.384 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:18:15.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:15.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:15.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:15.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:15.596 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:15.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:15.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:15.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:15.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:15.858 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:15.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:15.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:15.974 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:18:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:18:15] "GET /metrics HTTP/1.1" 200 34049 "" "Prometheus/2.33.4" 2023-10-09T09:18:15.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:15.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:16.093 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:16.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:16.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:16.224 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:16.255 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:16.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:16.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:16.310 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:18:16.359 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:16.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:16.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:16.495 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:16.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:16.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:16.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:16.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:16.652 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:16.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:16.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:16.754 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:18:16.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:16.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:16.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:16.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:16.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:16.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:16.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:17.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:17.049 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:17.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:17.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:17.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:17.249 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:18:17.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:17.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:17.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:17.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:17.433 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:17 smithi094 bash[59677]: cluster 2023-10-09T09:18:16.180573+0000 mgr.y (mgr.25421) 1231 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:17.433 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:17 smithi094 bash[59677]: audit 2023-10-09T09:18:16.629462+0000 mon.a (mon.0) 1648 : audit 1 from='client.? 172.21.15.16:0/341947887' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:18:17.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:17.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:17 smithi016 bash[75179]: cluster 2023-10-09T09:18:16.180573+0000 mgr.y (mgr.25421) 1231 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:17.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:17 smithi016 bash[75179]: audit 2023-10-09T09:18:16.629462+0000 mon.a (mon.0) 1648 : audit 1 from='client.? 172.21.15.16:0/341947887' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:18:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:17 smithi016 bash[77892]: cluster 2023-10-09T09:18:16.180573+0000 mgr.y (mgr.25421) 1231 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:17.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:17 smithi016 bash[77892]: audit 2023-10-09T09:18:16.629462+0000 mon.a (mon.0) 1648 : audit 1 from='client.? 172.21.15.16:0/341947887' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:18:17.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:17.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:17.685 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:17.717 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:17.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:17.821 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:17.851 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:17.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:17.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:17.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:17.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:18.086 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:18.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:18.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:18.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:18.224 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:18.251 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:18.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:18.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:18.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:18:18.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:18.368 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:18 smithi094 bash[59677]: cluster 2023-10-09T09:18:17.233945+0000 mon.a (mon.0) 1649 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:18:18.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:18 smithi094 bash[59677]: cluster 2023-10-09T09:18:17.242239+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:18.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:18 smithi094 bash[59677]: audit 2023-10-09T09:18:17.244186+0000 mon.a (mon.0) 1650 : audit 1 from='client.? 172.21.15.16:0/341947887' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:18:18.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:18 smithi094 bash[59677]: cluster 2023-10-09T09:18:17.244255+0000 mon.a (mon.0) 1651 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-10-09T09:18:18.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:18 smithi094 bash[59677]: audit 2023-10-09T09:18:18.042908+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:18.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:18 smithi094 bash[59677]: audit 2023-10-09T09:18:18.048725+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:18.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:18.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:18.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[75179]: cluster 2023-10-09T09:18:17.233945+0000 mon.a (mon.0) 1649 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:18:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[75179]: cluster 2023-10-09T09:18:17.242239+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[75179]: audit 2023-10-09T09:18:17.244186+0000 mon.a (mon.0) 1650 : audit 1 from='client.? 172.21.15.16:0/341947887' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:18:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[75179]: cluster 2023-10-09T09:18:17.244255+0000 mon.a (mon.0) 1651 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-10-09T09:18:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[75179]: audit 2023-10-09T09:18:18.042908+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:18.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[75179]: audit 2023-10-09T09:18:18.048725+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:18.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[77892]: cluster 2023-10-09T09:18:17.233945+0000 mon.a (mon.0) 1649 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:18:18.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[77892]: cluster 2023-10-09T09:18:17.242239+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:18.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[77892]: audit 2023-10-09T09:18:17.244186+0000 mon.a (mon.0) 1650 : audit 1 from='client.? 172.21.15.16:0/341947887' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:18:18.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[77892]: cluster 2023-10-09T09:18:17.244255+0000 mon.a (mon.0) 1651 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-10-09T09:18:18.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[77892]: audit 2023-10-09T09:18:18.042908+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:18.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:18 smithi016 bash[77892]: audit 2023-10-09T09:18:18.048725+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:18.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:18.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:18.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:18.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:18.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:18.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:18.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:18.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:18.880 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:18.910 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:18.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:18.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:19.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:19.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:19.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:19.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:19.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:19.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:19.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:19.300 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:19 smithi016 bash[75179]: cluster 2023-10-09T09:18:18.181322+0000 mgr.y (mgr.25421) 1232 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:19.300 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:19 smithi016 bash[75179]: cluster 2023-10-09T09:18:18.891300+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:19.300 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:19 smithi016 bash[77892]: cluster 2023-10-09T09:18:18.181322+0000 mgr.y (mgr.25421) 1232 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:19.300 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:19 smithi016 bash[77892]: cluster 2023-10-09T09:18:18.891300+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:19.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:19.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:19 smithi094 bash[59677]: cluster 2023-10-09T09:18:18.181322+0000 mgr.y (mgr.25421) 1232 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:19.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:19 smithi094 bash[59677]: cluster 2023-10-09T09:18:18.891300+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:19.532 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:19.564 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:19.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:19.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:19.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:19.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:19.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:19.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:19.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:19.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:20.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:20.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:20.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:20.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:20.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:20.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:20.331 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:20.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:20.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:20.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:20.493 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:20.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[77892]: cluster 2023-10-09T09:18:19.106410+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:20.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[77892]: audit 2023-10-09T09:18:19.258738+0000 mon.a (mon.0) 1654 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[77892]: audit 2023-10-09T09:18:19.260092+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:20.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[77892]: cluster 2023-10-09T09:18:19.273563+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:20.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[77892]: audit 2023-10-09T09:18:19.393469+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[77892]: audit 2023-10-09T09:18:19.400270+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.494 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[77892]: audit 2023-10-09T09:18:20.236192+0000 mon.a (mon.0) 1658 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[75179]: cluster 2023-10-09T09:18:19.106410+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:20.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[75179]: audit 2023-10-09T09:18:19.258738+0000 mon.a (mon.0) 1654 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[75179]: audit 2023-10-09T09:18:19.260092+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:20.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[75179]: cluster 2023-10-09T09:18:19.273563+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:20.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[75179]: audit 2023-10-09T09:18:19.393469+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[75179]: audit 2023-10-09T09:18:19.400270+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:20 smithi016 bash[75179]: audit 2023-10-09T09:18:20.236192+0000 mon.a (mon.0) 1658 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:20.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:20.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:20.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:20 smithi094 bash[59677]: cluster 2023-10-09T09:18:19.106410+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:20.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:20 smithi094 bash[59677]: audit 2023-10-09T09:18:19.258738+0000 mon.a (mon.0) 1654 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:20 smithi094 bash[59677]: audit 2023-10-09T09:18:19.260092+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:20.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:20 smithi094 bash[59677]: cluster 2023-10-09T09:18:19.273563+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T09:18:20.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:20 smithi094 bash[59677]: audit 2023-10-09T09:18:19.393469+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:20 smithi094 bash[59677]: audit 2023-10-09T09:18:19.400270+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:20 smithi094 bash[59677]: audit 2023-10-09T09:18:20.236192+0000 mon.a (mon.0) 1658 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:20.628 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:20.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:20.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:20.731 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:20.761 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:20.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:20.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:20.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:20.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:20.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:20.997 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:21.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:21.086 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:18:21.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:21.155 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:21.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:21.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:21.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:21.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:21.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:21.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:21 smithi094 bash[59677]: cluster 2023-10-09T09:18:20.182055+0000 mgr.y (mgr.25421) 1233 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:21.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:21.527 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:21 smithi016 bash[75179]: cluster 2023-10-09T09:18:20.182055+0000 mgr.y (mgr.25421) 1233 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:21.527 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:21 smithi016 bash[77892]: cluster 2023-10-09T09:18:20.182055+0000 mgr.y (mgr.25421) 1233 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:21.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:21.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:21.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:18:21.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:21.677 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:21.781 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:21.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:21.912 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:21.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:21.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:22.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:22.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:22.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:22.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:22.249 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:22.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:22.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:22.352 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:22.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:22.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:22.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:22.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:22.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:22.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:22.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:23.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:23.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:23.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:23.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:23.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:23.274 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T09:18:23.275 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:23 smithi094 bash[59677]: cluster 2023-10-09T09:18:22.183213+0000 mgr.y (mgr.25421) 1234 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:23.275 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:23 smithi094 bash[59677]: audit 2023-10-09T09:18:22.834525+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.16:0/252012451' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:18:23.275 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:23 smithi094 bash[59677]: audit 2023-10-09T09:18:22.835224+0000 mon.a (mon.0) 1659 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:18:23.292 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:23.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:23.396 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:23.424 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:23.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:23 smithi016 bash[75179]: cluster 2023-10-09T09:18:22.183213+0000 mgr.y (mgr.25421) 1234 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:23.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:23 smithi016 bash[75179]: audit 2023-10-09T09:18:22.834525+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.16:0/252012451' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:18:23.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:23 smithi016 bash[75179]: audit 2023-10-09T09:18:22.835224+0000 mon.a (mon.0) 1659 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:18:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:23 smithi016 bash[77892]: cluster 2023-10-09T09:18:22.183213+0000 mgr.y (mgr.25421) 1234 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:23 smithi016 bash[77892]: audit 2023-10-09T09:18:22.834525+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.16:0/252012451' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:18:23.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:23 smithi016 bash[77892]: audit 2023-10-09T09:18:22.835224+0000 mon.a (mon.0) 1659 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:18:23.528 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:23.553 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:23.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:23.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:23.789 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:23.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:23.921 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:23.950 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:23.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:24.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:24.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:24.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:24.185 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:24.210 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:24.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:24.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:24.343 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:24.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:24.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:24.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:24 smithi094 bash[59677]: cluster 2023-10-09T09:18:23.263438+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:24.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:24 smithi094 bash[59677]: audit 2023-10-09T09:18:23.267381+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:18:24.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:24 smithi094 bash[59677]: cluster 2023-10-09T09:18:23.267519+0000 mon.a (mon.0) 1661 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-10-09T09:18:24.460 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:24 smithi094 bash[59677]: cluster 2023-10-09T09:18:23.459246+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:24.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[75179]: cluster 2023-10-09T09:18:23.263438+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:24.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[75179]: audit 2023-10-09T09:18:23.267381+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:18:24.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[75179]: cluster 2023-10-09T09:18:23.267519+0000 mon.a (mon.0) 1661 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-10-09T09:18:24.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[75179]: cluster 2023-10-09T09:18:23.459246+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:24.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[77892]: cluster 2023-10-09T09:18:23.263438+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:24.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[77892]: audit 2023-10-09T09:18:23.267381+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:18:24.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[77892]: cluster 2023-10-09T09:18:23.267519+0000 mon.a (mon.0) 1661 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-10-09T09:18:24.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:24 smithi016 bash[77892]: cluster 2023-10-09T09:18:23.459246+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:24.474 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:24.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:24.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:24.578 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:24.607 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:24.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:24.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:24.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:24.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:24.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:24.864 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:18:24.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:24.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:24.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:24.985 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:25.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:25.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:25.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:25.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:25.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:25.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:25.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:25.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:25.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:25.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:25.414 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:18:25.414 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167500 9.6 9:608998e5:::benchmark_data_smithi016_172771_object70914:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.670445+0000", 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 470.73836511799999, 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7207328950000003, 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.415 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167500 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670445+0000", 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670445+0000", 2023-10-09T09:18:25.416 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670443+0000", 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4409999999999993e-06 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670451+0000", 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4917e-05 2023-10-09T09:18:25.417 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670466+0000", 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0020000000000001e-05 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670476+0000", 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9809000000000003e-05 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.418 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670536+0000", 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0988e-05 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op must wait for map", 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.670557+0000", 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.010199121 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.419 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.680756+0000", 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0258000000000001e-05 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.680767+0000", 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6815948920000001 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.362361+0000", 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0017832639999999999 2023-10-09T09:18:25.420 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.364145+0000", 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.025215772000000001 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.389361+0000", 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0018175439999999999 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.421 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.391178+0000", 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7207328950000003 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167525 9.6 9:6180dd57:::benchmark_data_smithi016_172771_object70939:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.762870+0000", 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 470.64594044500001, 2023-10-09T09:18:25.422 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6350496899999998, 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167525 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.423 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762870+0000", 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762870+0000", 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762868+0000", 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0620000000000003e-06 2023-10-09T09:18:25.424 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762875+0000", 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9110000000000002e-05 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762895+0000", 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0680000000000002e-06 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.425 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762901+0000", 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2203999999999997e-05 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762953+0000", 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2585000000000001e-05 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.426 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.762965+0000", 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6013517049999999 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.364317+0000", 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019394639999999999 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.366257+0000", 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0031352530000000002 2023-10-09T09:18:25.427 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.369392+0000", 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.019547374999999999 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.388939+0000", 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028240269999999998 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.428 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.391763+0000", 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00052667899999999995 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.392290+0000", 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000939492 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.393229+0000", 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0046900860000000004 2023-10-09T09:18:25.429 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.397920+0000", 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6350496899999998 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167530 9.6 9:667f2bd8:::benchmark_data_smithi016_172771_object70944:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:18:25.430 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.796984+0000", 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 470.61182581000003, 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6069060149999999, 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167530 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.431 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796984+0000", 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796984+0000", 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.432 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796983+0000", 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.1619999999999997e-06 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.796992+0000", 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7893999999999998e-05 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797010+0000", 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8969999999999997e-06 2023-10-09T09:18:25.433 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797019+0000", 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8380999999999995e-05 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797087+0000", 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3692e-05 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.434 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.797101+0000", 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5762976860000002 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.373398+0000", 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0023449650000000001 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.375743+0000", 2023-10-09T09:18:25.435 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029730749999999999 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.378716+0000", 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.010364816000000001 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.389081+0000", 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0094196139999999998 2023-10-09T09:18:25.436 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.398501+0000", 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00068523300000000004 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.399186+0000", 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0011877840000000001 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.437 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.400374+0000", 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0035166569999999999 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.403890+0000", 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6069060149999999 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.438 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167537 9.6 9:62deb5e7:::benchmark_data_smithi016_172771_object70951:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.810885+0000", 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 470.59792503, 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7884914350000001, 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167537 2023-10-09T09:18:25.439 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810885+0000", 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810885+0000", 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.440 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810884+0000", 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4e-06 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810889+0000", 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7440000000000002e-06 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.441 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810899+0000", 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.369e-06 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810903+0000", 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.2126999999999997e-05 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810966+0000", 2023-10-09T09:18:25.442 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0293e-05 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.810986+0000", 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.6004881640000002 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.411474+0000", 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0018830400000000001 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.443 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.413357+0000", 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.012155733 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.425513+0000", 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019261440000000001 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.444 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.427439+0000", 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.156210243 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.583649+0000", 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001402569 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.585052+0000", 2023-10-09T09:18:25.445 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028961970000000001 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.587948+0000", 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0031777770000000001 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.591126+0000", 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0041007320000000002 2023-10-09T09:18:25.446 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595226+0000", 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014585499999999999 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595372+0000", 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00072834699999999996 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.447 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.596101+0000", 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0032759899999999999 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.599377+0000", 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.7884914350000001 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.448 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167551 9.17 9.309d0bf7 (undecoded) ondisk+write+known_if_redirected e239)", 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.856594+0000", 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 470.55221599800001, 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.82954309000000004, 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167551 2023-10-09T09:18:25.449 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856594+0000", 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856594+0000", 2023-10-09T09:18:25.450 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856592+0000", 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7029999999999995e-06 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856601+0000", 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2045000000000001e-05 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.451 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856613+0000", 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3899999999999998e-06 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856619+0000", 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.0243e-05 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.452 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856669+0000", 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.6370000000000005e-06 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.856678+0000", 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.82943719800000004 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.686115+0000", 2023-10-09T09:18:25.453 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.213e-05 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.686137+0000", 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.82954309000000004 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.454 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "MOSDPGPush(9.e 240/230 [PushOp(9:7d10c4b0:::benchmark_data_smithi016_172771_object72506:head, version: 219'4540, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:7d10c4b0:::benchmark_data_smithi016_172771_object72506:head@219'4540, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.105023+0000", 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 469.30378727099998, 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.377055428, 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "started", 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105023+0000", 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.455 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105023+0000", 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105022+0000", 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.2552000000000003e-05 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.456 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105054+0000", 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7813999999999999e-05 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105072+0000", 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0685e-05 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.105083+0000", 2023-10-09T09:18:25.457 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.373906643 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.478990+0000", 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1800000000000001e-05 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.479001+0000", 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0030770110000000002 2023-10-09T09:18:25.458 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.482078+0000", 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.377055428 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167572 9.16 9:68db5b2c:::benchmark_data_smithi016_172771_object70986:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.733117+0000", 2023-10-09T09:18:25.459 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 468.67569295599998, 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87027597099999998, 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167572 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733117+0000", 2023-10-09T09:18:25.460 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733117+0000", 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733116+0000", 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.0509999999999997e-06 2023-10-09T09:18:25.461 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733124+0000", 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1333999999999999e-05 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733145+0000", 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.1690000000000003e-06 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.462 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733152+0000", 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.69478544200000003 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.427938+0000", 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020526250000000002 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.463 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.429990+0000", 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.022311549 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.452302+0000", 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020267430000000001 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.454329+0000", 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.133338768 2023-10-09T09:18:25.464 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.587667+0000", 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001300692 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.588968+0000", 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002805227 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.465 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.591773+0000", 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0034174190000000001 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595191+0000", 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00044434500000000002 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.595635+0000", 2023-10-09T09:18:25.466 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0039520429999999997 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.599587+0000", 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00095617100000000002 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.600543+0000", 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002850109 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.467 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.603393+0000", 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87027597099999998 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167573 9.6 9:6683712f:::benchmark_data_smithi016_172771_object70987:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 468.67524685400002, 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87949885500000002, 2023-10-09T09:18:25.468 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167573 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.469 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733563+0000", 2023-10-09T09:18:25.470 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4440000000000001e-06 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733567+0000", 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.079e-05 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733578+0000", 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3959999999999997e-06 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.471 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.733583+0000", 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.69850582100000003 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.432089+0000", 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001875011 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.472 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.433964+0000", 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.018259147999999999 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.452223+0000", 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0020477120000000001 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.454271+0000", 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.14543932900000001 2023-10-09T09:18:25.473 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.599710+0000", 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00133927 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.601050+0000", 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0024509449999999999 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.474 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.603500+0000", 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.005160548 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.608661+0000", 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00046506899999999998 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.609126+0000", 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014417600000000001 2023-10-09T09:18:25.475 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.609270+0000", 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00076132999999999995 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.610032+0000", 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0030306650000000001 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.476 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.613062+0000", 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87949885500000002 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167610 9.6 9:65930b06:::benchmark_data_smithi016_172771_object71024:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.786339+0000", 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 468.62247164899998, 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.85008152999999997, 2023-10-09T09:18:25.477 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167610 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786339+0000", 2023-10-09T09:18:25.478 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786339+0000", 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786337+0000", 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7220000000000005e-06 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.479 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786345+0000", 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5143000000000001e-05 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786360+0000", 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8770000000000004e-06 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.480 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.786367+0000", 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.66811855799999997 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.454486+0000", 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019376689999999999 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.456424+0000", 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0099610730000000008 2023-10-09T09:18:25.481 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.466385+0000", 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001688169 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.468073+0000", 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.148861836 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.482 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.616935+0000", 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001150933 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.618086+0000", 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029704890000000002 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.621056+0000", 2023-10-09T09:18:25.483 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.010123839000000001 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.631180+0000", 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00043493299999999998 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.631615+0000", 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00053206800000000002 2023-10-09T09:18:25.484 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.632147+0000", 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00090994600000000002 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.633057+0000", 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033633209999999998 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.485 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.636420+0000", 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.85008152999999997 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167613 9.6 9:63f11c86:::benchmark_data_smithi016_172771_object71027:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.788607+0000", 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 468.62020329299997, 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87494654599999999, 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.486 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167613 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788607+0000", 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.487 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788607+0000", 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788605+0000", 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5630000000000003e-06 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788613+0000", 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2583e-05 2023-10-09T09:18:25.488 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788626+0000", 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4509999999999997e-06 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.788632+0000", 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.66800163499999998 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.489 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.456634+0000", 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00184935 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.458483+0000", 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024125830000000001 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.482609+0000", 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0021895199999999999 2023-10-09T09:18:25.490 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.484798+0000", 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15935247699999999 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.644151+0000", 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001645175 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.491 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.645796+0000", 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033096229999999998 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.649106+0000", 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0041119559999999999 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.653218+0000", 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00056759400000000004 2023-10-09T09:18:25.492 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.653785+0000", 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.005000666 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.658786+0000", 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000948802 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.493 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.659735+0000", 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003818905 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.663554+0000", 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87494654599999999 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.494 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167615 9.16 9:6b4dc5a4:::benchmark_data_smithi016_172771_object71029:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.792429+0000", 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 468.61638153799998, 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87891387899999995, 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167615 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.495 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792429+0000", 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792429+0000", 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.496 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792427+0000", 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.8710000000000002e-06 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792433+0000", 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0434e-05 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.497 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792444+0000", 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9355e-05 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.792463+0000", 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.69274825900000003 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.485211+0000", 2023-10-09T09:18:25.498 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001540334 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.486752+0000", 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00884186 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.495593+0000", 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001833157 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.499 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.497427+0000", 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.16112549900000001 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.658552+0000", 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0013227250000000001 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.500 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.659875+0000", 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0032222000000000002 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.663097+0000", 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028207079999999999 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.665918+0000", 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00069836799999999999 2023-10-09T09:18:25.501 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.666616+0000", 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00021289 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.666829+0000", 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00096081200000000002 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.502 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.667790+0000", 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0035527670000000001 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.671343+0000", 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87891387899999995 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.503 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167626 9.16 9:68673687:::benchmark_data_smithi016_172771_object71040:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.806080+0000", 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 468.60273055300001, 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90435950899999995, 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167626 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.504 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806080+0000", 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806080+0000", 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.505 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806078+0000", 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.888e-06 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806085+0000", 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.5667999999999999e-05 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.506 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806111+0000", 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4659999999999996e-06 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.806117+0000", 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.680842265 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.486960+0000", 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001570407 2023-10-09T09:18:25.507 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.488530+0000", 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.016700744 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.505231+0000", 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00155108 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.508 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.506782+0000", 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.18548661699999999 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.692268+0000", 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0017039589999999999 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.693972+0000", 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003072249 2023-10-09T09:18:25.509 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.697045+0000", 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0080081349999999996 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705053+0000", 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00072625800000000003 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.510 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705779+0000", 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000194952 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705974+0000", 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00094332200000000002 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.511 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.706917+0000", 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0035220020000000002 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.710439+0000", 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.90435950899999995 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.512 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167631 9.6 9:61adcc4d:::benchmark_data_smithi016_172771_object71045:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:36.812915+0000", 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 468.59589478999999, 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.89219442299999996, 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167631 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.513 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812915+0000", 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812915+0000", 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.514 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812913+0000", 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8959999999999992e-06 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812922+0000", 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.9108000000000001e-05 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812941+0000", 2023-10-09T09:18:25.515 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8959999999999998e-06 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:36.812951+0000", 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.67575335400000003 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.488704+0000", 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001590089 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.516 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for missing object", 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.490295+0000", 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.017790391999999999 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.508085+0000", 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0019484459999999999 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for rw locks", 2023-10-09T09:18:25.517 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.510033+0000", 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.17243972599999999 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.682473+0000", 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001713743 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.684187+0000", 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033628519999999999 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.518 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.687550+0000", 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0047191000000000004 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.692269+0000", 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0082264759999999999 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:25.519 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.700495+0000", 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019154999999999999 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.700687+0000", 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00095907800000000004 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.701646+0000", 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0034639990000000002 2023-10-09T09:18:25.520 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:37.705110+0000", 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.89219442299999996 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162039 10.9 10.77a07ed9 (undecoded) ondisk+write+known_if_redirected e269)", 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:22.400187+0000", 2023-10-09T09:18:25.521 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 183.00862335900001, 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0076362329999999, 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162039 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.522 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.400187+0000", 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.400187+0000", 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999909 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.400178+0000", 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5024999999999999e-05 2023-10-09T09:18:25.523 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.400193+0000", 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0764000000000001e-05 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.400204+0000", 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4069999999999998e-06 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.524 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.400210+0000", 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.034987422999999997 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.435197+0000", 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.6540000000000005e-06 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.435206+0000", 2023-10-09T09:18:25.525 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.97258757399999995 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.407794+0000", 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.9533e-05 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.407823+0000", 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0076362329999999 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.526 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162044 10.9 10.47e3cbf9 (undecoded) ondisk+write+known_if_redirected e269)", 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:22.410800+0000", 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 182.99801015700001, 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99719575699999996, 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.527 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162044 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.410800+0000", 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.528 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.410800+0000", 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.410798+0000", 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.2e-06 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.410807+0000", 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2048999999999999e-05 2023-10-09T09:18:25.529 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.410819+0000", 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.579e-06 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.410826+0000", 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.025009750000000001 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.530 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.435836+0000", 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1759999999999999e-05 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.435847+0000", 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.97211008700000001 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.407957+0000", 2023-10-09T09:18:25.531 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.8404e-05 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.407996+0000", 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.99719575699999996 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162103 10.9 10.dc8aeef9 (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T09:18:25.532 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:22.529222+0000", 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 182.879588063, 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87882453299999996, 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162103 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.533 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529222+0000", 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529222+0000", 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.534 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529220+0000", 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0183000000000001e-05 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529230+0000", 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0984000000000001e-05 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529251+0000", 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.6969999999999999e-06 2023-10-09T09:18:25.535 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529260+0000", 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.7797000000000001e-05 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529338+0000", 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3402999999999999e-05 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.536 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.529371+0000", 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87866223700000001 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408033+0000", 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3305999999999999e-05 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408047+0000", 2023-10-09T09:18:25.537 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.87882453299999996 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162109 10.9 10.3b2b8119 (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:22.539261+0000", 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 182.86954900800001, 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.86880992400000001, 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.538 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162109 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539261+0000", 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.539 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539261+0000", 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539260+0000", 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.5540000000000003e-06 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539266+0000", 2023-10-09T09:18:25.540 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.7839999999999998e-06 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539276+0000", 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.5230000000000003e-06 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539282+0000", 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.6137999999999996e-05 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.541 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539348+0000", 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1246e-05 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.539359+0000", 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.86870477300000004 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408064+0000", 2023-10-09T09:18:25.542 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.3989999999999998e-06 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408071+0000", 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.86880992400000001 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162121 10.9 10.2ac453c9 (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T09:18:25.543 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:22.557421+0000", 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 182.851389033, 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.850675399, 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162121 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.544 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557421+0000", 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557421+0000", 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557420+0000", 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.9959999999999993e-06 2023-10-09T09:18:25.545 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557428+0000", 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.6276000000000001e-05 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557444+0000", 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.1170000000000004e-06 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.546 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557452+0000", 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0547000000000001e-05 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557523+0000", 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1915e-05 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.547 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.557535+0000", 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.85055122800000005 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408086+0000", 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0859999999999999e-05 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408097+0000", 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.850675399 2023-10-09T09:18:25.548 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162141 10.9 10.7d2dd4f9 (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:22.605283+0000", 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 182.80352686000001, 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.80285380799999995, 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.549 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162141 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605283+0000", 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.550 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605283+0000", 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605281+0000", 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0560000000000001e-05 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605292+0000", 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.0129e-05 2023-10-09T09:18:25.551 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605312+0000", 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0027000000000001e-05 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605322+0000", 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.0517000000000002e-05 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.552 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605402+0000", 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4866e-05 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.605417+0000", 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.80270596999999999 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408123+0000", 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3987e-05 2023-10-09T09:18:25.553 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408137+0000", 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.80285380799999995 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162142 10.9 10.f600a009 (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:22.607184+0000", 2023-10-09T09:18:25.554 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 182.80162657700001, 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.80099814700000005, 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162142 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:25.555 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607184+0000", 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607184+0000", 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607182+0000", 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8659999999999995e-06 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.556 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607191+0000", 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8870000000000001e-05 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607210+0000", 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7139999999999999e-06 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:25.557 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607218+0000", 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.2168000000000005e-05 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607281+0000", 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3908e-05 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:22.607295+0000", 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.80087262699999995 2023-10-09T09:18:25.558 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408167+0000", 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.472e-05 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408182+0000", 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.80099814700000005 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.559 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.560 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:25.560 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:25.560 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:18:25.560 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T09:18:25.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.561 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T09:18:25.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:18:25.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:25 smithi016 bash[75179]: cluster 2023-10-09T09:18:23.459201+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:25.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:25 smithi016 bash[75179]: cluster 2023-10-09T09:18:23.459625+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:25.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:25 smithi016 bash[75179]: cluster 2023-10-09T09:18:24.183992+0000 mgr.y (mgr.25421) 1235 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 582 KiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:25.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:25 smithi016 bash[77892]: cluster 2023-10-09T09:18:23.459201+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:25.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:25 smithi016 bash[77892]: cluster 2023-10-09T09:18:23.459625+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:25.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:25 smithi016 bash[77892]: cluster 2023-10-09T09:18:24.183992+0000 mgr.y (mgr.25421) 1235 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 582 KiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:25.564 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:18:25.564 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:18:25.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:25.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:25.606 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-10-09T09:18:25.606 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:18:25.606 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_218422 2023-10-09T09:18:25.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:18:25.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:18:25.672 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:25 smithi094 bash[59677]: cluster 2023-10-09T09:18:23.459201+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:25.672 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:25 smithi094 bash[59677]: cluster 2023-10-09T09:18:23.459625+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T09:18:25.673 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:25 smithi094 bash[59677]: cluster 2023-10-09T09:18:24.183992+0000 mgr.y (mgr.25421) 1235 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 582 KiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T09:18:25.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:25.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:25.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:25.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.855 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:18:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:18:25] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.33.4" 2023-10-09T09:18:25.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:25.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:25.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:25.982 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:26.009 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:26.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:26.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:26.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:26.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:26.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:26.273 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:26.360 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:18:26.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:26.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:26.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:26.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:26.564 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:26.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2796 2780 173.741 173.75 0.0029068 0.00572339 2023-10-09T09:18:26.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:26.690 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:26.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:26.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:26.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:26.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:26.925 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:26.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:26.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:26.989 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:18:27.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:27.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:27.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:27.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:27.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:27.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:27.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:27.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:27.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:27.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:27.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:27.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:27.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:27 smithi016 bash[75179]: cluster 2023-10-09T09:18:26.184578+0000 mgr.y (mgr.25421) 1236 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 5.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 587 KiB/s wr, 9 op/s 2023-10-09T09:18:27.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:27 smithi016 bash[77892]: cluster 2023-10-09T09:18:26.184578+0000 mgr.y (mgr.25421) 1236 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 5.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 587 KiB/s wr, 9 op/s 2023-10-09T09:18:27.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:27.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:27.589 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:27.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:27 smithi094 bash[59677]: cluster 2023-10-09T09:18:26.184578+0000 mgr.y (mgr.25421) 1236 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 5.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 587 KiB/s wr, 9 op/s 2023-10-09T09:18:27.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5488 5472 170.988 168.25 0.00223903 0.00583155 2023-10-09T09:18:27.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:27.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:27.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:27.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:27.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:27.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:27.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:27.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:27.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:27.990 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:28.021 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:28.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:28.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:28.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:28.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:28.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:28.281 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:28.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:28.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:28.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:28.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:28.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:28.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7717 7701 160.425 139.312 0.00870696 0.00622373 2023-10-09T09:18:28.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:28.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:28.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:28.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:28.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:28.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:28.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:29.071 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:29.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:29.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:29.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:29.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:29.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:29.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:29.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:29.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:29.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:29.464 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:29 smithi094 bash[59677]: cluster 2023-10-09T09:18:28.185397+0000 mgr.y (mgr.25421) 1237 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 104 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s 2023-10-09T09:18:29.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:29.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:29.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:29 smithi016 bash[75179]: cluster 2023-10-09T09:18:28.185397+0000 mgr.y (mgr.25421) 1237 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 104 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s 2023-10-09T09:18:29.555 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:29 smithi016 bash[77892]: cluster 2023-10-09T09:18:28.185397+0000 mgr.y (mgr.25421) 1237 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 104 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s 2023-10-09T09:18:29.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:29.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9604 9588 149.794 117.938 0.00612074 0.00666651 2023-10-09T09:18:29.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:29.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:29.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:29.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:29.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:29.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:29.853 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:29.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:29.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:29.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:29.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:30.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:30.042 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:18:30.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:30.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:30.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:30.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:30.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:30.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:30.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:30.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:30.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:30.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:30.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 11827 11811 147.62 138.938 0.00651617 0.00676447 2023-10-09T09:18:30.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:30.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:30.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:30.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:30.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:30.941 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:30.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:31.045 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.076 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:30 smithi094 bash[59677]: cluster 2023-10-09T09:18:30.185899+0000 mgr.y (mgr.25421) 1238 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 133 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s wr, 211 op/s 2023-10-09T09:18:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:31.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:31.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:30 smithi016 bash[75179]: cluster 2023-10-09T09:18:30.185899+0000 mgr.y (mgr.25421) 1238 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 133 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s wr, 211 op/s 2023-10-09T09:18:31.187 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:30 smithi016 bash[77892]: cluster 2023-10-09T09:18:30.185899+0000 mgr.y (mgr.25421) 1238 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 133 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s wr, 211 op/s 2023-10-09T09:18:31.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:31.222 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:18:31.308 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:31.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:31.432 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:31.458 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:31.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:31.561 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:31.607 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 14056 14040 146.233 139.312 0.00678155 0.00682742 2023-10-09T09:18:31.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:31.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:31.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.979 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:31.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:31.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:32.083 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:32.109 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:32.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:32.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:32.240 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:32.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:32.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:32.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:32.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:32.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:32.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:32.500 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:32.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:32.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:32.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:32.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 16268 16252 145.091 138.25 0.00936851 0.00688497 2023-10-09T09:18:32.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:32.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:32.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:32.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:32.758 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:32.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:32.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:32.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:32.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:32.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:32.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:32.990 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:33.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:33.122 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:33.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:33.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:33.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:33.273 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:33.376 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:33.386 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:33 smithi094 bash[59677]: cluster 2023-10-09T09:18:32.187418+0000 mgr.y (mgr.25421) 1239 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 718 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:18:33.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:33.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:33.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:33 smithi016 bash[75179]: cluster 2023-10-09T09:18:32.187418+0000 mgr.y (mgr.25421) 1239 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 718 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:18:33.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:33 smithi016 bash[77892]: cluster 2023-10-09T09:18:32.187418+0000 mgr.y (mgr.25421) 1239 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 718 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:18:33.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:33.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:33.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:33.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 18473 18457 144.18 137.812 0.0025161 0.00692953 2023-10-09T09:18:33.641 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:33.665 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:33.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:33.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:33.794 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:33.898 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:33.928 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:33.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:33.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:34.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:34.058 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:34.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:34.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:34.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:34.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:34.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:34.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:34.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:34.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:34.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:34.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:34.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 20146 20130 139.775 104.562 0.00195419 0.0070361 2023-10-09T09:18:34.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:34.719 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:34.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:34.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:34.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:34.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:34.988 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:34.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:34.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:35.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:35.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:35.218 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:35.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:35.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:35 smithi016 bash[77892]: cluster 2023-10-09T09:18:34.188032+0000 mgr.y (mgr.25421) 1240 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 845 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:18:35.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:35 smithi016 bash[77892]: audit 2023-10-09T09:18:34.254311+0000 mon.a (mon.0) 1662 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:35.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:35.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:35.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:35.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:35 smithi094 bash[59677]: cluster 2023-10-09T09:18:34.188032+0000 mgr.y (mgr.25421) 1240 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 845 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:18:35.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:35 smithi094 bash[59677]: audit 2023-10-09T09:18:34.254311+0000 mon.a (mon.0) 1662 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:35.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:35.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:35.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:35.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:35 smithi016 bash[75179]: cluster 2023-10-09T09:18:34.188032+0000 mgr.y (mgr.25421) 1240 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 845 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:18:35.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:35 smithi016 bash[75179]: audit 2023-10-09T09:18:34.254311+0000 mon.a (mon.0) 1662 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:35.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 21756 21740 135.859 100.625 0.00303894 0.00734643 2023-10-09T09:18:35.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:35.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:35.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:35.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:35.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:35.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:35.875 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:18:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:18:35] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.33.4" 2023-10-09T09:18:35.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:35.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:35.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:36.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:36.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:36.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:36.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:36.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:36.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:36.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:36.427 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:36.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:36.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:36.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:36.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 22873 22857 129.854 69.8125 0.00158509 0.00757014 2023-10-09T09:18:36.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:36.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:36.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:36.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:36.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:36.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:36.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:36.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:36.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:37.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:37.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:37.179 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:37.205 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:37.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:37.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:37.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:37.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:37.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:37.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:37.586 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:37 smithi016 bash[75179]: cluster 2023-10-09T09:18:36.188817+0000 mgr.y (mgr.25421) 1241 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T09:18:37.587 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:37 smithi016 bash[77892]: cluster 2023-10-09T09:18:36.188817+0000 mgr.y (mgr.25421) 1241 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T09:18:37.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:37.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 24664 24648 128.36 111.938 0.00807667 0.00778446 2023-10-09T09:18:37.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:37.716 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:37 smithi094 bash[59677]: cluster 2023-10-09T09:18:36.188817+0000 mgr.y (mgr.25421) 1241 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T09:18:37.731 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:37.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:37.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:37.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:37.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:37.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:37.988 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:37.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:37.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:38.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:38.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:38.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:38.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:38.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:38.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:38.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:38.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:38.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:38.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:38.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:38.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:38.507 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:38.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:38.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:38.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 26252 26236 126.12 99.25 0.00804549 0.00792209 2023-10-09T09:18:38.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:38.639 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:38.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:38.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:38.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:38.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:38.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:38.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:38.868 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26669 11.9 11:925adaba:::benchmark_data_smithi016_218422_object26668:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.860768+0000", 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0062445449999999998, 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0062823319999999998, 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.869 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26669 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.860768+0000", 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.870 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.860768+0000", 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.860766+0000", 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.5531999999999998e-05 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.860802+0000", 2023-10-09T09:18:38.871 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.375e-05 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.860815+0000", 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.9130000000000003e-06 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.860823+0000", 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4418000000000001e-05 2023-10-09T09:18:38.872 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.860868+0000", 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015111200000000001 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.861019+0000", 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014909100000000001 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.873 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.861168+0000", 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0055572140000000004 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866725+0000", 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0059575230000000002 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.874 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26671 11.9 11:933c28e8:::benchmark_data_smithi016_218422_object26670:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.862442+0000", 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0045705809999999998, 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0046844419999999996, 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26671 2023-10-09T09:18:38.875 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862442+0000", 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862442+0000", 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.876 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862441+0000", 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.8188000000000001e-05 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862479+0000", 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1124999999999999e-05 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.877 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862490+0000", 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8290000000000002e-06 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862497+0000", 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.7024999999999997e-05 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.878 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862554+0000", 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015379400000000001 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862707+0000", 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00029455099999999999 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.863002+0000", 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0037318160000000002 2023-10-09T09:18:38.879 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866734+0000", 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0042922519999999999 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26672 11.9 11:90aad9f6:::benchmark_data_smithi016_218422_object26671:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.862509+0000", 2023-10-09T09:18:38.880 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0045036199999999998, 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0046983499999999996, 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26672 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.881 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862509+0000", 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862509+0000", 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862508+0000", 2023-10-09T09:18:38.882 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.5803999999999997e-05 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862554+0000", 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.2940000000000006e-06 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862562+0000", 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2089999999999996e-06 2023-10-09T09:18:38.883 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862567+0000", 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0028245950000000001 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.865392+0000", 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000128501 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.884 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.865520+0000", 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015157699999999999 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.865672+0000", 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0031632079999999998 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.885 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26673 11.1 11:87ff02c1:::benchmark_data_smithi016_218422_object26672:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.862578+0000", 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0044336849999999997, 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0047115530000000003, 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26673 2023-10-09T09:18:38.886 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862578+0000", 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862578+0000", 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:18:38.887 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862578+0000", 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3325999999999998e-05 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862621+0000", 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8809999999999996e-06 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.888 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862629+0000", 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1529999999999996e-06 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862634+0000", 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0012383260000000001 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.863872+0000", 2023-10-09T09:18:38.889 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019660000000000001 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.864069+0000", 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000149562 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.864219+0000", 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0016401390000000001 2023-10-09T09:18:38.890 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26674 11.5 11:a98b0e8f:::benchmark_data_smithi016_218422_object26673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.862639+0000", 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0043727970000000003, 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0047013310000000004, 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.891 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26674 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862639+0000", 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.892 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862639+0000", 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862639+0000", 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.7021999999999999e-05 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.893 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862676+0000", 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.2880000000000004e-06 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862683+0000", 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4840000000000001e-06 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862690+0000", 2023-10-09T09:18:38.894 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.8319999999999999e-05 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862728+0000", 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.5902000000000005e-05 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862814+0000", 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000157942 2023-10-09T09:18:38.895 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862972+0000", 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00340793 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866380+0000", 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000231444 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.896 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866611+0000", 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0039717249999999997 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26677 11.a 11:51fac765:::benchmark_data_smithi016_218422_object26676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.862698+0000", 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0043140449999999999, 2023-10-09T09:18:38.897 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0046986770000000001, 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26677 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862698+0000", 2023-10-09T09:18:38.898 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862698+0000", 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862697+0000", 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.5337999999999999e-05 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.899 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862733+0000", 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5329999999999997e-06 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862740+0000", 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.8600000000000001e-06 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.900 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862745+0000", 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000109728 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.862855+0000", 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014912200000000001 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.863004+0000", 2023-10-09T09:18:38.901 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000167257 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.863171+0000", 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0027513720000000002 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.865923+0000", 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0013786270000000001 2023-10-09T09:18:38.902 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867301+0000", 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0046031869999999999 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26679 11.5 11:a47af204:::benchmark_data_smithi016_218422_object26678:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.903 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.863967+0000", 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.0030455059999999999, 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0034873119999999998, 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26679 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.904 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.863967+0000", 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.863967+0000", 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.863965+0000", 2023-10-09T09:18:38.905 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.9227000000000007e-05 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.864035+0000", 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.3119999999999999e-05 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.864058+0000", 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.5949999999999999e-06 2023-10-09T09:18:38.906 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.864066+0000", 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00077331000000000003 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.864840+0000", 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8801999999999994e-05 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.907 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.864939+0000", 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000106069 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.865045+0000", 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0015740490000000001 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.908 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866619+0000", 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00029264699999999999 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866911+0000", 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0029445999999999999 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.909 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26682 11.1 11.e485e1b1 (undecoded) ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.866770+0000", 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.00024200599999999999, 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00074085899999999996, 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26682 2023-10-09T09:18:38.910 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866770+0000", 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866770+0000", 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:38.911 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866769+0000", 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3578000000000002e-05 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866812+0000", 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3217000000000001e-05 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.912 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866826+0000", 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.1379999999999998e-06 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866831+0000", 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0541999999999997e-05 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.913 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26683 11.1 11.e7ac4421 (undecoded) ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.866838+0000", 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 0.00017460899999999999, 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00071642799999999997, 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "queued for pg", 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.914 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26683 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866838+0000", 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866838+0000", 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:18:38.915 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866837+0000", 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6655000000000001e-05 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866874+0000", 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.502e-06 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.916 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866879+0000", 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3689999999999998e-06 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.866882+0000", 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4886999999999999e-05 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.917 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:26684 11.3 11:cece95e0:::benchmark_data_smithi016_218422_object26683:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:18:38.867273+0000", 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 4294967295.9997387, 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00033218199999999999, 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:18:38.918 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 26684 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867273+0000", 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867273+0000", 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:18:38.919 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867272+0000", 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.2679000000000003e-05 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867315+0000", 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4044e-05 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.920 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867329+0000", 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1792e-05 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867341+0000", 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.5992000000000003e-05 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:18:38.921 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867387+0000", 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8710999999999997e-05 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867485+0000", 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00011235600000000001 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 1,7", 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:18:38.867598+0000", 2023-10-09T09:18:38.922 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000324209 2023-10-09T09:18:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:18:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:18:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: ], 2023-10-09T09:18:38.923 INFO:teuthology.orchestra.run.smithi094.stdout: "num_ops": 10 2023-10-09T09:18:38.923 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:18:38.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:38.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:38.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:38.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:39.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:39.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:39.178 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:39.200 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:39.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:39.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:39.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:39.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:39.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:39.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:39 smithi094 bash[59677]: cluster 2023-10-09T09:18:38.189631+0000 mgr.y (mgr.25421) 1242 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T09:18:39.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:39.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:39 smithi016 bash[75179]: cluster 2023-10-09T09:18:38.189631+0000 mgr.y (mgr.25421) 1242 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T09:18:39.571 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:39 smithi016 bash[77892]: cluster 2023-10-09T09:18:38.189631+0000 mgr.y (mgr.25421) 1242 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T09:18:39.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:39.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:39.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 27880 27864 124.378 101.75 0.00844721 0.00803447 2023-10-09T09:18:39.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:39.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:39.820 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:39.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:39.956 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:39.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:39.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:39.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:40.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:40.113 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:40.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:40.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:40.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:40.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:40.349 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:18:40.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:40.433 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:40.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:40.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:40.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:40.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:40.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:40.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:40.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 29417 29401 122.49 96.0625 0.00770092 0.00815761 2023-10-09T09:18:40.666 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:40.694 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:40.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:40.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:40.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:40.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:40.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:40.933 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:40.942 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:40 smithi094 bash[59677]: cluster 2023-10-09T09:18:40.190124+0000 mgr.y (mgr.25421) 1243 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T09:18:40.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:40.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:40.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:41.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:41.075 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:40 smithi016 bash[75179]: cluster 2023-10-09T09:18:40.190124+0000 mgr.y (mgr.25421) 1243 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T09:18:41.075 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:40 smithi016 bash[77892]: cluster 2023-10-09T09:18:40.190124+0000 mgr.y (mgr.25421) 1243 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T09:18:41.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:41.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:41.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:41.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:41.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:41.314 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:41.341 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:41.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:41.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:41.444 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:41.473 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:41.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:41.579 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:41.608 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:41.608 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 30609 30593 119.49 74.5 0.0205949 0.00836073 2023-10-09T09:18:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:41.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:41.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:41.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:41.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:41.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:41.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:41.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:41.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:41.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:41.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:42.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:42.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:42.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:42.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:42.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:42.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:42.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:42.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 32142 32126 118.097 95.8125 0.00723181 0.00846186 2023-10-09T09:18:42.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:42.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:42.790 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:42.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:42.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:42.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:42.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:42.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:43.057 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:43.084 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:43.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:43.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:43.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:43.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:43.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:43.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:43.322 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:43 smithi094 bash[59677]: cluster 2023-10-09T09:18:42.190800+0000 mgr.y (mgr.25421) 1244 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-10-09T09:18:43.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:43.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:43.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:43.455 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:43 smithi016 bash[75179]: cluster 2023-10-09T09:18:42.190800+0000 mgr.y (mgr.25421) 1244 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-10-09T09:18:43.456 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:43 smithi016 bash[77892]: cluster 2023-10-09T09:18:42.190800+0000 mgr.y (mgr.25421) 1244 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-10-09T09:18:43.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:43.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:43.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:43.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:43.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:43.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:43.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 33683 33667 116.886 96.3125 0.00595726 0.00854899 2023-10-09T09:18:43.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:43.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:43.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:43.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:43.833 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:43.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:43.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:43.971 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:44.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:44.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:44.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:44.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:44.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:44.253 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:44.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:44.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:44.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:44.422 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:44.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:44.552 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:44.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 34824 34808 114.488 71.3125 0.00705313 0.00872859 2023-10-09T09:18:44.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:44.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:44.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:44.781 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:44.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:44.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:44.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:44.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:44.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:45.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:45.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:45.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:45.184 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:18:45.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:45.230 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:45.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:45.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:45.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:45.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:45.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:45.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:45 smithi094 bash[59677]: cluster 2023-10-09T09:18:44.191327+0000 mgr.y (mgr.25421) 1245 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T09:18:45.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:45.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:45 smithi016 bash[75179]: cluster 2023-10-09T09:18:44.191327+0000 mgr.y (mgr.25421) 1245 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T09:18:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:45.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:45.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:45 smithi016 bash[77892]: cluster 2023-10-09T09:18:44.191327+0000 mgr.y (mgr.25421) 1245 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T09:18:45.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:18:45.608850+0000 min lat: 0.00130401 max lat: 0.241195 avg lat: 0.00883566 2023-10-09T09:18:45.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:18:45.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 36144 36128 112.888 82.5 0.0674185 0.00883566 2023-10-09T09:18:45.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:45.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:45.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:45.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:45.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:45.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:45.781 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:18:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:18:45] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.33.4" 2023-10-09T09:18:45.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:45.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:45.883 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:45.911 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:45.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:46.014 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:46.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:46.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:46.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:46.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:46.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:46.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:46.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:46.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:46.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:46.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:46.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:46.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 37286 37270 110.911 71.375 0.0131421 0.00901022 2023-10-09T09:18:46.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:46.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:46.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:46.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:46.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:46.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:46.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:46.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:46.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:47.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:47.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:47.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:47.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:47.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:47.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:47.322 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:47.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:47.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:47.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:47.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:47.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:47 smithi016 bash[75179]: cluster 2023-10-09T09:18:46.192128+0000 mgr.y (mgr.25421) 1246 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:18:47.459 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:47 smithi016 bash[77892]: cluster 2023-10-09T09:18:46.192128+0000 mgr.y (mgr.25421) 1246 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:18:47.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:47.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:47.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:47.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 38769 38753 110.082 92.6875 0.0174659 0.00907718 2023-10-09T09:18:47.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:47.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:47.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:47.730 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:47.739 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:47 smithi094 bash[59677]: cluster 2023-10-09T09:18:46.192128+0000 mgr.y (mgr.25421) 1246 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:18:47.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:47.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:47.858 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:47.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:47.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:47.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:48.014 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:18:48.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:48.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:48.059 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:18:48.102 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:18:48.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:48.147 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:48.147 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:18:48.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:48.190 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:18:48.229 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:18:48.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:48.263 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:48 smithi094 bash[59677]: audit 2023-10-09T09:18:47.965084+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.16:0/1988368081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:18:48.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:18:48.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:48.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:48.319 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.326 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:18:48.382 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:48.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:48.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:48.528 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:48 smithi016 bash[75179]: audit 2023-10-09T09:18:47.965084+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.16:0/1988368081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:18:48.528 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:48 smithi016 bash[77892]: audit 2023-10-09T09:18:47.965084+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.16:0/1988368081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:18:48.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:48.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 40105 40089 108.926 83.5 0.00986863 0.00917535 2023-10-09T09:18:48.650 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:48.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:48.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:48.770 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:48.804 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:48.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:48.939 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:48.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:49.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:49.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:49.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:49.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:49.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:49.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:49.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:49.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:49.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:49.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:49 smithi094 bash[59677]: cluster 2023-10-09T09:18:48.192735+0000 mgr.y (mgr.25421) 1247 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:18:49.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:49 smithi094 bash[59677]: audit 2023-10-09T09:18:49.254079+0000 mon.a (mon.0) 1663 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:49.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:49.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:49.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:49.543 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:49 smithi016 bash[75179]: cluster 2023-10-09T09:18:48.192735+0000 mgr.y (mgr.25421) 1247 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:18:49.543 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:49 smithi016 bash[75179]: audit 2023-10-09T09:18:49.254079+0000 mon.a (mon.0) 1663 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:49.544 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:49 smithi016 bash[77892]: cluster 2023-10-09T09:18:48.192735+0000 mgr.y (mgr.25421) 1247 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:18:49.544 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:49 smithi016 bash[77892]: audit 2023-10-09T09:18:49.254079+0000 mon.a (mon.0) 1663 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:49.555 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:49.584 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:49.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:49.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 40932 40916 106.541 51.6875 0.00741202 0.00937664 2023-10-09T09:18:49.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:49.717 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:49.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:49.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:49.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:49.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:49.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:49.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:50.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:50.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:50.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:50.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:50.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:50.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:50.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:50.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:50.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:50.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:50.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:50.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:50.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:50.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:50.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 42258 42242 105.594 82.875 0.0181742 0.00946502 2023-10-09T09:18:50.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:50.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:50.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:50.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:50.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:50.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:50.891 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:50.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:50.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:50.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:51.020 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:51.048 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:51.153 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:51.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:50 smithi094 bash[59677]: cluster 2023-10-09T09:18:50.193240+0000 mgr.y (mgr.25421) 1248 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:18:51.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:50 smithi094 bash[59677]: audit 2023-10-09T09:18:50.715570+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-10-09T09:18:51.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:50 smithi094 bash[59677]: audit 2023-10-09T09:18:50.716108+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:18:51.162 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:50 smithi094 bash[59677]: audit 2023-10-09T09:18:50.716836+0000 mon.a (mon.0) 1666 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T09:18:51.178 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:51.225 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:18:51.225 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:18:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[75179]: cluster 2023-10-09T09:18:50.193240+0000 mgr.y (mgr.25421) 1248 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:18:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[75179]: audit 2023-10-09T09:18:50.715570+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-10-09T09:18:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[75179]: audit 2023-10-09T09:18:50.716108+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:18:51.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[75179]: audit 2023-10-09T09:18:50.716836+0000 mon.a (mon.0) 1666 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T09:18:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[77892]: cluster 2023-10-09T09:18:50.193240+0000 mgr.y (mgr.25421) 1248 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:18:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[77892]: audit 2023-10-09T09:18:50.715570+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-10-09T09:18:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[77892]: audit 2023-10-09T09:18:50.716108+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:18:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:50 smithi016 bash[77892]: audit 2023-10-09T09:18:50.716836+0000 mon.a (mon.0) 1666 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T09:18:51.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:51.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:51.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:51.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:51.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:51.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:51.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:51.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:51.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:51.609 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 43579 43563 104.708 82.5625 0.00816526 0.00954413 2023-10-09T09:18:51.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:51.708 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:51.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:51.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:51.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:51.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:51.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:51.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:51.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:51.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[75179]: cluster 2023-10-09T09:18:50.820360+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T09:18:51.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[75179]: cluster 2023-10-09T09:18:50.820952+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T09:18:51.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[75179]: audit 2023-10-09T09:18:50.828014+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-10-09T09:18:51.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[75179]: audit 2023-10-09T09:18:50.828639+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[75179]: audit 2023-10-09T09:18:50.828864+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[75179]: cluster 2023-10-09T09:18:50.829094+0000 mon.a (mon.0) 1670 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[77892]: cluster 2023-10-09T09:18:50.820360+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[77892]: cluster 2023-10-09T09:18:50.820952+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[77892]: audit 2023-10-09T09:18:50.828014+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[77892]: audit 2023-10-09T09:18:50.828639+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[77892]: audit 2023-10-09T09:18:50.828864+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T09:18:52.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:51 smithi016 bash[77892]: cluster 2023-10-09T09:18:50.829094+0000 mon.a (mon.0) 1670 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2023-10-09T09:18:52.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:52.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:52.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:52.114 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:51 smithi094 bash[59677]: cluster 2023-10-09T09:18:50.820360+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T09:18:52.114 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:51 smithi094 bash[59677]: cluster 2023-10-09T09:18:50.820952+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T09:18:52.114 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:51 smithi094 bash[59677]: audit 2023-10-09T09:18:50.828014+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-10-09T09:18:52.114 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:51 smithi094 bash[59677]: audit 2023-10-09T09:18:50.828639+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:18:52.114 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:51 smithi094 bash[59677]: audit 2023-10-09T09:18:50.828864+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T09:18:52.115 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:51 smithi094 bash[59677]: cluster 2023-10-09T09:18:50.829094+0000 mon.a (mon.0) 1670 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2023-10-09T09:18:52.191 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:52.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:52.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:52.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:52.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:52.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:52.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:52.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:52.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:52.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:52.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:52.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 44601 44585 103.194 63.875 0.00651483 0.00968361 2023-10-09T09:18:52.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:52.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:52.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:52.641 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:18:52.720 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:52.752 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:52.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:52.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[77892]: cluster 2023-10-09T09:18:51.840972+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:52.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[77892]: cluster 2023-10-09T09:18:51.845124+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:52.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[77892]: cluster 2023-10-09T09:18:51.848074+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2023-10-09T09:18:52.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[77892]: cluster 2023-10-09T09:18:51.853233+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:52.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[77892]: cluster 2023-10-09T09:18:52.193688+0000 mgr.y (mgr.25421) 1249 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:18:52.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[77892]: audit 2023-10-09T09:18:52.324997+0000 mon.a (mon.0) 1672 : audit 1 from='client.? 172.21.15.16:0/2963397657' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:18:52.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:52.857 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:18:52.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:52.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:52 smithi094 bash[59677]: cluster 2023-10-09T09:18:51.840972+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:52 smithi094 bash[59677]: cluster 2023-10-09T09:18:51.845124+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:52 smithi094 bash[59677]: cluster 2023-10-09T09:18:51.848074+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2023-10-09T09:18:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:52 smithi094 bash[59677]: cluster 2023-10-09T09:18:51.853233+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:52 smithi094 bash[59677]: cluster 2023-10-09T09:18:52.193688+0000 mgr.y (mgr.25421) 1249 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:18:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:52 smithi094 bash[59677]: audit 2023-10-09T09:18:52.324997+0000 mon.a (mon.0) 1672 : audit 1 from='client.? 172.21.15.16:0/2963397657' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:18:52.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:53.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:53.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.141 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[75179]: cluster 2023-10-09T09:18:51.840972+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:53.141 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[75179]: cluster 2023-10-09T09:18:51.845124+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:53.141 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[75179]: cluster 2023-10-09T09:18:51.848074+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2023-10-09T09:18:53.141 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[75179]: cluster 2023-10-09T09:18:51.853233+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T09:18:53.141 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[75179]: cluster 2023-10-09T09:18:52.193688+0000 mgr.y (mgr.25421) 1249 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:18:53.141 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:52 smithi016 bash[75179]: audit 2023-10-09T09:18:52.324997+0000 mon.a (mon.0) 1672 : audit 1 from='client.? 172.21.15.16:0/2963397657' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:18:53.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:53.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:53.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:53.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:53.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:53.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:53.610 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 45968 45952 102.56 85.4375 0.0125377 0.00974471 2023-10-09T09:18:53.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:53.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:53.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:53.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:53.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:53.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:53.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:53.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:54.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:54.087 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[75179]: cluster 2023-10-09T09:18:52.838714+0000 mon.a (mon.0) 1673 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:18:54.087 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[75179]: cluster 2023-10-09T09:18:52.847657+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[75179]: audit 2023-10-09T09:18:52.851252+0000 mon.a (mon.0) 1674 : audit 1 from='client.? 172.21.15.16:0/2963397657' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:18:54.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[75179]: cluster 2023-10-09T09:18:52.851704+0000 mon.a (mon.0) 1675 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2023-10-09T09:18:54.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[75179]: cluster 2023-10-09T09:18:52.859499+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[75179]: cluster 2023-10-09T09:18:52.864140+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[77892]: cluster 2023-10-09T09:18:52.838714+0000 mon.a (mon.0) 1673 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:18:54.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[77892]: cluster 2023-10-09T09:18:52.847657+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[77892]: audit 2023-10-09T09:18:52.851252+0000 mon.a (mon.0) 1674 : audit 1 from='client.? 172.21.15.16:0/2963397657' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:18:54.089 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[77892]: cluster 2023-10-09T09:18:52.851704+0000 mon.a (mon.0) 1675 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2023-10-09T09:18:54.089 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[77892]: cluster 2023-10-09T09:18:52.859499+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.089 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:53 smithi016 bash[77892]: cluster 2023-10-09T09:18:52.864140+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:54.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:54.219 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:54.228 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:53 smithi094 bash[59677]: cluster 2023-10-09T09:18:52.838714+0000 mon.a (mon.0) 1673 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:18:54.228 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:53 smithi094 bash[59677]: cluster 2023-10-09T09:18:52.847657+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.228 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:53 smithi094 bash[59677]: audit 2023-10-09T09:18:52.851252+0000 mon.a (mon.0) 1674 : audit 1 from='client.? 172.21.15.16:0/2963397657' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:18:54.228 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:53 smithi094 bash[59677]: cluster 2023-10-09T09:18:52.851704+0000 mon.a (mon.0) 1675 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2023-10-09T09:18:54.229 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:53 smithi094 bash[59677]: cluster 2023-10-09T09:18:52.859499+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.229 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:53 smithi094 bash[59677]: cluster 2023-10-09T09:18:52.864140+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T09:18:54.247 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:54.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:54.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:54.351 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:54.378 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:54.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:54.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:54.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:54.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:54.610 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 47185 47169 101.646 76.0625 0.00726133 0.00983123 2023-10-09T09:18:54.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:54.648 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:54.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:54.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:54.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:54.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:54.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:54.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:54.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:54.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:54.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:54.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:54 smithi094 bash[59677]: cluster 2023-10-09T09:18:54.194204+0000 mgr.y (mgr.25421) 1250 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:18:55.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:55.023 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:54 smithi016 bash[77892]: cluster 2023-10-09T09:18:54.194204+0000 mgr.y (mgr.25421) 1250 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:18:55.023 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:54 smithi016 bash[75179]: cluster 2023-10-09T09:18:54.194204+0000 mgr.y (mgr.25421) 1250 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:18:55.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:55.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:55.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:55.185 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:55.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:55.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:55.321 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:55.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:55.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:55.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:55.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:55.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:55.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:55.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:55.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:55.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:55.610 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 48358 48342 100.701 73.3125 0.0197825 0.0099241 2023-10-09T09:18:55.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:55.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:55.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:55.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:55.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:55.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:55.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:55.961 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:55.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:55.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:18:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:18:55] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T09:18:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:56.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:56.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:18:56.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:56.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:56.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:56.368 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:56.395 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:18:56.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:18:56.412 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:18:56.425 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:18:56.499 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:56.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:56.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:56.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 49404 49388 99.5613 65.375 0.00310621 0.0100013 2023-10-09T09:18:56.626 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:56.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:18:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:18:56.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:56.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:56.894 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:56.915 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:56.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:57.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:57.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:57.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:57.165 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:57.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:57.270 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:57.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:57.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:57 smithi016 bash[75179]: cluster 2023-10-09T09:18:56.194831+0000 mgr.y (mgr.25421) 1251 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T09:18:57.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:57 smithi016 bash[75179]: audit 2023-10-09T09:18:56.398473+0000 mgr.y (mgr.25421) 1252 : audit 0 from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:18:57.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:57 smithi016 bash[77892]: cluster 2023-10-09T09:18:56.194831+0000 mgr.y (mgr.25421) 1251 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T09:18:57.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:57 smithi016 bash[77892]: audit 2023-10-09T09:18:56.398473+0000 mgr.y (mgr.25421) 1252 : audit 0 from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:18:57.401 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:57.409 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:57 smithi094 bash[59677]: cluster 2023-10-09T09:18:56.194831+0000 mgr.y (mgr.25421) 1251 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T09:18:57.409 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:57 smithi094 bash[59677]: audit 2023-10-09T09:18:56.398473+0000 mgr.y (mgr.25421) 1252 : audit 0 from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:18:57.431 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:57.468 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:18:57.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:57.562 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:57.610 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 50624 50608 98.8325 76.25 0.0155834 0.0101115 2023-10-09T09:18:57.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:57.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:57.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:57.812 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:57.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:57.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:57.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:57.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:58.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:58.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:18:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:58.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:18:58.223 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:58.254 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:18:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:58.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:18:58.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:58.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:58.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:58.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:58.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:58.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:58.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:58.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:58.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 51850 51834 98.1594 76.625 0.019968 0.0101814 2023-10-09T09:18:58.617 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:58.641 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:58.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:58.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:58.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:58.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:58.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:58.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:58.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:18:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:58.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:18:59.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:59.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:59.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:59.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:59.113 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1373,"stamp":"2023-10-09T09:18:56.194410+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3035633568,"num_objects":46335,"num_object_clones":0,"num_object_copies":139005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46335,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":46571,"num_write_kb":2966349,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":2971634,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46505,"ondisk_log_size":46505,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16163996,"kb_used_data":9513308,"kb_used_omap":126,"kb_used_meta":6650497,"kb_avail":733797220,"statfs":{"total":767960285184,"available":751408353280,"internally_reserved":0,"allocated":9741627392,"data_stored":9734850126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6810109563},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":601620480,"num_objects":9180,"num_object_clones":0,"num_object_copies":27540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9180,"num_write_kb":587520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001481"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":190,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869090+0000","last_change":"2023-10-09T09:17:58.260195+0000","last_active":"2023-10-09T09:18:51.869090+0000","last_peered":"2023-10-09T09:18:51.869090+0000","last_clean":"2023-10-09T09:18:51.869090+0000","last_became_active":"2023-10-09T09:17:58.259718+0000","last_became_peered":"2023-10-09T09:17:58.259718+0000","last_unstale":"2023-10-09T09:18:51.869090+0000","last_undegraded":"2023-10-09T09:18:51.869090+0000","last_fullsized":"2023-10-09T09:18:51.869090+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.266128+0000","last_change":"2023-10-09T09:14:48.316760+0000","last_active":"2023-10-09T09:18:23.266128+0000","last_peered":"2023-10-09T09:18:23.266128+0000","last_clean":"2023-10-09T09:18:23.266128+0000","last_became_active":"2023-10-09T09:14:48.316432+0000","last_became_peered":"2023-10-09T09:14:48.316432+0000","last_unstale":"2023-10-09T09:18:23.266128+0000","last_undegraded":"2023-10-09T09:18:23.266128+0000","last_fullsized":"2023-10-09T09:18:23.266128+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:41.630374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.265281+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:18:23.265281+0000","last_peered":"2023-10-09T09:18:23.265281+0000","last_clean":"2023-10-09T09:18:23.265281+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:18:23.265281+0000","last_undegraded":"2023-10-09T09:18:23.265281+0000","last_fullsized":"2023-10-09T09:18:23.265281+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":336,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.849556+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:18:50.849556+0000","last_peered":"2023-10-09T09:18:50.849556+0000","last_clean":"2023-10-09T09:18:50.849556+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:18:50.849556+0000","last_undegraded":"2023-10-09T09:18:50.849556+0000","last_fullsized":"2023-10-09T09:18:50.849556+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":280,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.850926+0000","last_change":"2023-10-09T09:14:48.312713+0000","last_active":"2023-10-09T09:18:51.850926+0000","last_peered":"2023-10-09T09:18:51.850926+0000","last_clean":"2023-10-09T09:18:51.850926+0000","last_became_active":"2023-10-09T09:14:48.312429+0000","last_became_peered":"2023-10-09T09:14:48.312429+0000","last_unstale":"2023-10-09T09:18:51.850926+0000","last_undegraded":"2023-10-09T09:18:51.850926+0000","last_fullsized":"2023-10-09T09:18:51.850926+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:56.755105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.860617+0000","last_change":"2023-10-09T09:18:51.858702+0000","last_active":"2023-10-09T09:18:51.860617+0000","last_peered":"2023-10-09T09:18:51.860617+0000","last_clean":"2023-10-09T09:18:51.860617+0000","last_became_active":"2023-10-09T09:18:51.858359+0000","last_became_peered":"2023-10-09T09:18:51.858359+0000","last_unstale":"2023-10-09T09:18:51.860617+0000","last_undegraded":"2023-10-09T09:18:51.860617+0000","last_fullsized":"2023-10-09T09:18:51.860617+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":288,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.848739+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:18:50.848739+0000","last_peered":"2023-10-09T09:18:50.848739+0000","last_clean":"2023-10-09T09:18:50.848739+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:18:50.848739+0000","last_undegraded":"2023-10-09T09:18:50.848739+0000","last_fullsized":"2023-10-09T09:18:50.848739+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.265182+0000","last_change":"2023-10-09T09:14:48.306040+0000","last_active":"2023-10-09T09:18:23.265182+0000","last_peered":"2023-10-09T09:18:23.265182+0000","last_clean":"2023-10-09T09:18:23.265182+0000","last_became_active":"2023-10-09T09:14:48.304137+0000","last_became_peered":"2023-10-09T09:14:48.304137+0000","last_unstale":"2023-10-09T09:18:23.265182+0000","last_undegraded":"2023-10-09T09:18:23.265182+0000","last_fullsized":"2023-10-09T09:18:23.265182+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:47.446761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854439+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:18:51.854439+0000","last_peered":"2023-10-09T09:18:51.854439+0000","last_clean":"2023-10-09T09:18:51.854439+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:18:51.854439+0000","last_undegraded":"2023-10-09T09:18:51.854439+0000","last_fullsized":"2023-10-09T09:18:51.854439+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":264,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.855235+0000","last_change":"2023-10-09T09:13:48.190068+0000","last_active":"2023-10-09T09:18:51.855235+0000","last_peered":"2023-10-09T09:18:51.855235+0000","last_clean":"2023-10-09T09:18:51.855235+0000","last_became_active":"2023-10-09T09:13:48.189740+0000","last_became_peered":"2023-10-09T09:13:48.189740+0000","last_unstale":"2023-10-09T09:18:51.855235+0000","last_undegraded":"2023-10-09T09:18:51.855235+0000","last_fullsized":"2023-10-09T09:18:51.855235+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":87,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.860039+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:18:51.860039+0000","last_peered":"2023-10-09T09:18:51.860039+0000","last_clean":"2023-10-09T09:18:51.860039+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:18:51.860039+0000","last_undegraded":"2023-10-09T09:18:51.860039+0000","last_fullsized":"2023-10-09T09:18:51.860039+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.265614+0000","last_change":"2023-10-09T09:17:58.252706+0000","last_active":"2023-10-09T09:18:23.265614+0000","last_peered":"2023-10-09T09:18:23.265614+0000","last_clean":"2023-10-09T09:18:23.265614+0000","last_became_active":"2023-10-09T09:17:58.252151+0000","last_became_peered":"2023-10-09T09:17:58.252151+0000","last_unstale":"2023-10-09T09:18:23.265614+0000","last_undegraded":"2023-10-09T09:18:23.265614+0000","last_fullsized":"2023-10-09T09:18:23.265614+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:35:57.970175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":202,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869588+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:18:51.869588+0000","last_peered":"2023-10-09T09:18:51.869588+0000","last_clean":"2023-10-09T09:18:51.869588+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:18:51.869588+0000","last_undegraded":"2023-10-09T09:18:51.869588+0000","last_fullsized":"2023-10-09T09:18:51.869588+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":190,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869217+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:18:51.869217+0000","last_peered":"2023-10-09T09:18:51.869217+0000","last_clean":"2023-10-09T09:18:51.869217+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:18:51.869217+0000","last_undegraded":"2023-10-09T09:18:51.869217+0000","last_fullsized":"2023-10-09T09:18:51.869217+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":117,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.859899+0000","last_change":"2023-10-09T09:17:58.350536+0000","last_active":"2023-10-09T09:18:51.859899+0000","last_peered":"2023-10-09T09:18:51.859899+0000","last_clean":"2023-10-09T09:18:51.859899+0000","last_became_active":"2023-10-09T09:17:58.259691+0000","last_became_peered":"2023-10-09T09:17:58.259691+0000","last_unstale":"2023-10-09T09:18:51.859899+0000","last_undegraded":"2023-10-09T09:18:51.859899+0000","last_fullsized":"2023-10-09T09:18:51.859899+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.266555+0000","last_change":"2023-10-09T09:14:48.319108+0000","last_active":"2023-10-09T09:18:23.266555+0000","last_peered":"2023-10-09T09:18:23.266555+0000","last_clean":"2023-10-09T09:18:23.266555+0000","last_became_active":"2023-10-09T09:14:48.318443+0000","last_became_peered":"2023-10-09T09:14:48.318443+0000","last_unstale":"2023-10-09T09:18:23.266555+0000","last_undegraded":"2023-10-09T09:18:23.266555+0000","last_fullsized":"2023-10-09T09:18:23.266555+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:31.939020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":316,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.842377+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:18:50.842377+0000","last_peered":"2023-10-09T09:18:50.842377+0000","last_clean":"2023-10-09T09:18:50.842377+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:18:50.842377+0000","last_undegraded":"2023-10-09T09:18:50.842377+0000","last_fullsized":"2023-10-09T09:18:50.842377+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":100,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.859923+0000","last_change":"2023-10-09T09:15:23.415336+0000","last_active":"2023-10-09T09:18:51.859923+0000","last_peered":"2023-10-09T09:18:51.859923+0000","last_clean":"2023-10-09T09:18:51.859923+0000","last_became_active":"2023-10-09T09:15:23.414900+0000","last_became_peered":"2023-10-09T09:15:23.414900+0000","last_unstale":"2023-10-09T09:18:51.859923+0000","last_undegraded":"2023-10-09T09:18:51.859923+0000","last_fullsized":"2023-10-09T09:18:51.859923+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":144,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869036+0000","last_change":"2023-10-09T09:13:48.205269+0000","last_active":"2023-10-09T09:18:51.869036+0000","last_peered":"2023-10-09T09:18:51.869036+0000","last_clean":"2023-10-09T09:18:51.869036+0000","last_became_active":"2023-10-09T09:13:48.204894+0000","last_became_peered":"2023-10-09T09:13:48.204894+0000","last_unstale":"2023-10-09T09:18:51.869036+0000","last_undegraded":"2023-10-09T09:18:51.869036+0000","last_fullsized":"2023-10-09T09:18:51.869036+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":288,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.849634+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:18:50.849634+0000","last_peered":"2023-10-09T09:18:50.849634+0000","last_clean":"2023-10-09T09:18:50.849634+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:18:50.849634+0000","last_undegraded":"2023-10-09T09:18:50.849634+0000","last_fullsized":"2023-10-09T09:18:50.849634+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":217,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854584+0000","last_change":"2023-10-09T09:15:23.420333+0000","last_active":"2023-10-09T09:18:51.854584+0000","last_peered":"2023-10-09T09:18:51.854584+0000","last_clean":"2023-10-09T09:18:51.854584+0000","last_became_active":"2023-10-09T09:15:23.412581+0000","last_became_peered":"2023-10-09T09:15:23.412581+0000","last_unstale":"2023-10-09T09:18:51.854584+0000","last_undegraded":"2023-10-09T09:18:51.854584+0000","last_fullsized":"2023-10-09T09:18:51.854584+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:52.856624+0000","last_change":"2023-10-09T09:18:51.977540+0000","last_active":"2023-10-09T09:18:52.856624+0000","last_peered":"2023-10-09T09:18:52.856624+0000","last_clean":"2023-10-09T09:18:52.856624+0000","last_became_active":"2023-10-09T09:18:51.977190+0000","last_became_peered":"2023-10-09T09:18:51.977190+0000","last_unstale":"2023-10-09T09:18:52.856624+0000","last_undegraded":"2023-10-09T09:18:52.856624+0000","last_fullsized":"2023-10-09T09:18:52.856624+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:20.658136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.7","version":"299'2721","reported_seq":2735,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.689075+0000","last_change":"2023-10-09T09:18:10.373025+0000","last_active":"2023-10-09T09:18:51.689075+0000","last_peered":"2023-10-09T09:18:51.689075+0000","last_clean":"2023-10-09T09:18:51.689075+0000","last_became_active":"2023-10-09T09:18:10.372667+0000","last_became_peered":"2023-10-09T09:18:10.372667+0000","last_unstale":"2023-10-09T09:18:51.689075+0000","last_undegraded":"2023-10-09T09:18:51.689075+0000","last_fullsized":"2023-10-09T09:18:51.689075+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2721,"log_dups_size":0,"ondisk_log_size":2721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:26.426396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178323456,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2721,"num_write_kb":174144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":98,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.860176+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:18:51.860176+0000","last_peered":"2023-10-09T09:18:51.860176+0000","last_clean":"2023-10-09T09:18:51.860176+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:18:51.860176+0000","last_undegraded":"2023-10-09T09:18:51.860176+0000","last_fullsized":"2023-10-09T09:18:51.860176+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.859716+0000","last_change":"2023-10-09T09:15:23.414649+0000","last_active":"2023-10-09T09:18:51.859716+0000","last_peered":"2023-10-09T09:18:51.859716+0000","last_clean":"2023-10-09T09:18:51.859716+0000","last_became_active":"2023-10-09T09:15:23.414272+0000","last_became_peered":"2023-10-09T09:15:23.414272+0000","last_unstale":"2023-10-09T09:18:51.859716+0000","last_undegraded":"2023-10-09T09:18:51.859716+0000","last_fullsized":"2023-10-09T09:18:51.859716+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"300'2760","reported_seq":2775,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:52.333072+0000","last_change":"2023-10-09T09:18:10.364624+0000","last_active":"2023-10-09T09:18:52.333072+0000","last_peered":"2023-10-09T09:18:52.333072+0000","last_clean":"2023-10-09T09:18:52.333072+0000","last_became_active":"2023-10-09T09:18:10.364280+0000","last_became_peered":"2023-10-09T09:18:10.364280+0000","last_unstale":"2023-10-09T09:18:52.333072+0000","last_undegraded":"2023-10-09T09:18:52.333072+0000","last_fullsized":"2023-10-09T09:18:52.333072+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2760,"log_dups_size":0,"ondisk_log_size":2760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:29.392502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180879360,"num_objects":2760,"num_object_clones":0,"num_object_copies":8280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2760,"num_write_kb":176640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":98,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.860301+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:18:51.860301+0000","last_peered":"2023-10-09T09:18:51.860301+0000","last_clean":"2023-10-09T09:18:51.860301+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:18:51.860301+0000","last_undegraded":"2023-10-09T09:18:51.860301+0000","last_fullsized":"2023-10-09T09:18:51.860301+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":190,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.868930+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:18:51.868930+0000","last_peered":"2023-10-09T09:18:51.868930+0000","last_clean":"2023-10-09T09:18:51.868930+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:18:51.868930+0000","last_undegraded":"2023-10-09T09:18:51.868930+0000","last_fullsized":"2023-10-09T09:18:51.868930+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"301'2975","reported_seq":2990,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:55.589015+0000","last_change":"2023-10-09T09:18:10.916026+0000","last_active":"2023-10-09T09:18:55.589015+0000","last_peered":"2023-10-09T09:18:55.589015+0000","last_clean":"2023-10-09T09:18:55.589015+0000","last_became_active":"2023-10-09T09:18:10.915063+0000","last_became_peered":"2023-10-09T09:18:10.915063+0000","last_unstale":"2023-10-09T09:18:55.589015+0000","last_undegraded":"2023-10-09T09:18:55.589015+0000","last_fullsized":"2023-10-09T09:18:55.589015+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2975,"log_dups_size":0,"ondisk_log_size":2975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194969600,"num_objects":2975,"num_object_clones":0,"num_object_copies":8925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2975,"num_write_kb":190400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":174,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.855749+0000","last_change":"2023-10-09T09:15:23.437420+0000","last_active":"2023-10-09T09:18:51.855749+0000","last_peered":"2023-10-09T09:18:51.855749+0000","last_clean":"2023-10-09T09:18:51.855749+0000","last_became_active":"2023-10-09T09:15:23.429358+0000","last_became_peered":"2023-10-09T09:15:23.429358+0000","last_unstale":"2023-10-09T09:18:51.855749+0000","last_undegraded":"2023-10-09T09:18:51.855749+0000","last_fullsized":"2023-10-09T09:18:51.855749+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":204,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.857233+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:18:51.857233+0000","last_peered":"2023-10-09T09:18:51.857233+0000","last_clean":"2023-10-09T09:18:51.857233+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:18:51.857233+0000","last_undegraded":"2023-10-09T09:18:51.857233+0000","last_fullsized":"2023-10-09T09:18:51.857233+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"301'2976","reported_seq":2991,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:55.494487+0000","last_change":"2023-10-09T09:18:10.375845+0000","last_active":"2023-10-09T09:18:55.494487+0000","last_peered":"2023-10-09T09:18:55.494487+0000","last_clean":"2023-10-09T09:18:55.494487+0000","last_became_active":"2023-10-09T09:18:10.375463+0000","last_became_peered":"2023-10-09T09:18:10.375463+0000","last_unstale":"2023-10-09T09:18:55.494487+0000","last_undegraded":"2023-10-09T09:18:55.494487+0000","last_fullsized":"2023-10-09T09:18:55.494487+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2976,"log_dups_size":0,"ondisk_log_size":2976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195035136,"num_objects":2976,"num_object_clones":0,"num_object_copies":8928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2976,"num_write_kb":190464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854085+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:18:51.854085+0000","last_peered":"2023-10-09T09:18:51.854085+0000","last_clean":"2023-10-09T09:18:51.854085+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:18:51.854085+0000","last_undegraded":"2023-10-09T09:18:51.854085+0000","last_fullsized":"2023-10-09T09:18:51.854085+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":316,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869617+0000","last_change":"2023-10-09T09:18:51.869617+0000","last_active":"2023-10-09T09:18:51.869617+0000","last_peered":"2023-10-09T09:18:51.869617+0000","last_clean":"2023-10-09T09:18:51.869617+0000","last_became_active":"2023-10-09T09:18:51.869212+0000","last_became_peered":"2023-10-09T09:18:51.869212+0000","last_unstale":"2023-10-09T09:18:51.869617+0000","last_undegraded":"2023-10-09T09:18:51.869617+0000","last_fullsized":"2023-10-09T09:18:51.869617+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:36.753958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"301'3022","reported_seq":3037,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:55.608436+0000","last_change":"2023-10-09T09:18:10.372418+0000","last_active":"2023-10-09T09:18:55.608436+0000","last_peered":"2023-10-09T09:18:55.608436+0000","last_clean":"2023-10-09T09:18:55.608436+0000","last_became_active":"2023-10-09T09:18:10.371749+0000","last_became_peered":"2023-10-09T09:18:10.371749+0000","last_unstale":"2023-10-09T09:18:55.608436+0000","last_undegraded":"2023-10-09T09:18:55.608436+0000","last_fullsized":"2023-10-09T09:18:55.608436+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":3022,"log_dups_size":0,"ondisk_log_size":3022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198049792,"num_objects":3022,"num_object_clones":0,"num_object_copies":9066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3022,"num_write_kb":193408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":110,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.842433+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:18:50.842433+0000","last_peered":"2023-10-09T09:18:50.842433+0000","last_clean":"2023-10-09T09:18:50.842433+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:18:50.842433+0000","last_undegraded":"2023-10-09T09:18:50.842433+0000","last_fullsized":"2023-10-09T09:18:50.842433+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869422+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:18:51.869422+0000","last_peered":"2023-10-09T09:18:51.869422+0000","last_clean":"2023-10-09T09:18:51.869422+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:18:51.869422+0000","last_undegraded":"2023-10-09T09:18:51.869422+0000","last_fullsized":"2023-10-09T09:18:51.869422+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"300'2765","reported_seq":2780,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:52.311462+0000","last_change":"2023-10-09T09:18:10.916706+0000","last_active":"2023-10-09T09:18:52.311462+0000","last_peered":"2023-10-09T09:18:52.311462+0000","last_clean":"2023-10-09T09:18:52.311462+0000","last_became_active":"2023-10-09T09:18:10.916284+0000","last_became_peered":"2023-10-09T09:18:10.916284+0000","last_unstale":"2023-10-09T09:18:52.311462+0000","last_undegraded":"2023-10-09T09:18:52.311462+0000","last_fullsized":"2023-10-09T09:18:52.311462+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2765,"log_dups_size":0,"ondisk_log_size":2765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:20.902101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2765,"num_write_kb":176960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.857146+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:18:51.857146+0000","last_peered":"2023-10-09T09:18:51.857146+0000","last_clean":"2023-10-09T09:18:51.857146+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:18:51.857146+0000","last_undegraded":"2023-10-09T09:18:51.857146+0000","last_fullsized":"2023-10-09T09:18:51.857146+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":190,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869674+0000","last_change":"2023-10-09T09:17:58.262763+0000","last_active":"2023-10-09T09:18:51.869674+0000","last_peered":"2023-10-09T09:18:51.869674+0000","last_clean":"2023-10-09T09:18:51.869674+0000","last_became_active":"2023-10-09T09:17:58.262304+0000","last_became_peered":"2023-10-09T09:17:58.262304+0000","last_unstale":"2023-10-09T09:18:51.869674+0000","last_undegraded":"2023-10-09T09:18:51.869674+0000","last_fullsized":"2023-10-09T09:18:51.869674+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"301'2984","reported_seq":2998,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:55.608825+0000","last_change":"2023-10-09T09:18:10.916054+0000","last_active":"2023-10-09T09:18:55.608825+0000","last_peered":"2023-10-09T09:18:55.608825+0000","last_clean":"2023-10-09T09:18:55.608825+0000","last_became_active":"2023-10-09T09:18:10.915135+0000","last_became_peered":"2023-10-09T09:18:10.915135+0000","last_unstale":"2023-10-09T09:18:55.608825+0000","last_undegraded":"2023-10-09T09:18:55.608825+0000","last_fullsized":"2023-10-09T09:18:55.608825+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2984,"log_dups_size":0,"ondisk_log_size":2984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195559424,"num_objects":2984,"num_object_clones":0,"num_object_copies":8952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2984,"num_write_kb":190976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.868932+0000","last_change":"2023-10-09T09:17:58.262853+0000","last_active":"2023-10-09T09:18:51.868932+0000","last_peered":"2023-10-09T09:18:51.868932+0000","last_clean":"2023-10-09T09:18:51.868932+0000","last_became_active":"2023-10-09T09:17:58.262349+0000","last_became_peered":"2023-10-09T09:17:58.262349+0000","last_unstale":"2023-10-09T09:18:51.868932+0000","last_undegraded":"2023-10-09T09:18:51.868932+0000","last_fullsized":"2023-10-09T09:18:51.868932+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":181,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869443+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:18:51.869443+0000","last_peered":"2023-10-09T09:18:51.869443+0000","last_clean":"2023-10-09T09:18:51.869443+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:18:51.869443+0000","last_undegraded":"2023-10-09T09:18:51.869443+0000","last_fullsized":"2023-10-09T09:18:51.869443+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"299'2747","reported_seq":2758,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.699695+0000","last_change":"2023-10-09T09:18:10.917634+0000","last_active":"2023-10-09T09:18:51.699695+0000","last_peered":"2023-10-09T09:18:51.699695+0000","last_clean":"2023-10-09T09:18:51.699695+0000","last_became_active":"2023-10-09T09:18:10.914359+0000","last_became_peered":"2023-10-09T09:18:10.914359+0000","last_unstale":"2023-10-09T09:18:51.699695+0000","last_undegraded":"2023-10-09T09:18:51.699695+0000","last_fullsized":"2023-10-09T09:18:51.699695+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2747,"log_dups_size":0,"ondisk_log_size":2747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:37.047604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180027392,"num_objects":2747,"num_object_clones":0,"num_object_copies":8241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2747,"num_write_kb":175808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.265698+0000","last_change":"2023-10-09T09:14:48.306639+0000","last_active":"2023-10-09T09:18:23.265698+0000","last_peered":"2023-10-09T09:18:23.265698+0000","last_clean":"2023-10-09T09:18:23.265698+0000","last_became_active":"2023-10-09T09:14:48.305729+0000","last_became_peered":"2023-10-09T09:14:48.305729+0000","last_unstale":"2023-10-09T09:18:23.265698+0000","last_undegraded":"2023-10-09T09:18:23.265698+0000","last_fullsized":"2023-10-09T09:18:23.265698+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:08.892989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.267165+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:18:23.267165+0000","last_peered":"2023-10-09T09:18:23.267165+0000","last_clean":"2023-10-09T09:18:23.267165+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:18:23.267165+0000","last_undegraded":"2023-10-09T09:18:23.267165+0000","last_fullsized":"2023-10-09T09:18:23.267165+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"301'2862","reported_seq":2877,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:53.706516+0000","last_change":"2023-10-09T09:18:10.371956+0000","last_active":"2023-10-09T09:18:53.706516+0000","last_peered":"2023-10-09T09:18:53.706516+0000","last_clean":"2023-10-09T09:18:53.706516+0000","last_became_active":"2023-10-09T09:18:10.371596+0000","last_became_peered":"2023-10-09T09:18:10.371596+0000","last_unstale":"2023-10-09T09:18:53.706516+0000","last_undegraded":"2023-10-09T09:18:53.706516+0000","last_fullsized":"2023-10-09T09:18:53.706516+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2862,"log_dups_size":0,"ondisk_log_size":2862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:03:59.187705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187564032,"num_objects":2862,"num_object_clones":0,"num_object_copies":8586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2862,"num_write_kb":183168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.266889+0000","last_change":"2023-10-09T09:14:48.299630+0000","last_active":"2023-10-09T09:18:23.266889+0000","last_peered":"2023-10-09T09:18:23.266889+0000","last_clean":"2023-10-09T09:18:23.266889+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:18:23.266889+0000","last_undegraded":"2023-10-09T09:18:23.266889+0000","last_fullsized":"2023-10-09T09:18:23.266889+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854277+0000","last_change":"2023-10-09T09:17:58.255559+0000","last_active":"2023-10-09T09:18:51.854277+0000","last_peered":"2023-10-09T09:18:51.854277+0000","last_clean":"2023-10-09T09:18:51.854277+0000","last_became_active":"2023-10-09T09:17:58.255100+0000","last_became_peered":"2023-10-09T09:17:58.255100+0000","last_unstale":"2023-10-09T09:18:51.854277+0000","last_undegraded":"2023-10-09T09:18:51.854277+0000","last_fullsized":"2023-10-09T09:18:51.854277+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":55,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.265513+0000","last_change":"2023-10-09T09:15:23.413972+0000","last_active":"2023-10-09T09:18:23.265513+0000","last_peered":"2023-10-09T09:18:23.265513+0000","last_clean":"2023-10-09T09:18:23.265513+0000","last_became_active":"2023-10-09T09:15:23.413531+0000","last_became_peered":"2023-10-09T09:15:23.413531+0000","last_unstale":"2023-10-09T09:18:23.265513+0000","last_undegraded":"2023-10-09T09:18:23.265513+0000","last_fullsized":"2023-10-09T09:18:23.265513+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:54.516595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"299'2710","reported_seq":2724,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.681407+0000","last_change":"2023-10-09T09:18:10.914938+0000","last_active":"2023-10-09T09:18:51.681407+0000","last_peered":"2023-10-09T09:18:51.681407+0000","last_clean":"2023-10-09T09:18:51.681407+0000","last_became_active":"2023-10-09T09:18:10.914146+0000","last_became_peered":"2023-10-09T09:18:10.914146+0000","last_unstale":"2023-10-09T09:18:51.681407+0000","last_undegraded":"2023-10-09T09:18:51.681407+0000","last_fullsized":"2023-10-09T09:18:51.681407+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2710,"log_dups_size":0,"ondisk_log_size":2710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:15.765459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177602560,"num_objects":2710,"num_object_clones":0,"num_object_copies":8130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2710,"num_write_kb":173440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":98,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854154+0000","last_change":"2023-10-09T09:14:48.319163+0000","last_active":"2023-10-09T09:18:51.854154+0000","last_peered":"2023-10-09T09:18:51.854154+0000","last_clean":"2023-10-09T09:18:51.854154+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:18:51.854154+0000","last_undegraded":"2023-10-09T09:18:51.854154+0000","last_fullsized":"2023-10-09T09:18:51.854154+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":163,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869503+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:18:51.869503+0000","last_peered":"2023-10-09T09:18:51.869503+0000","last_clean":"2023-10-09T09:18:51.869503+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:18:51.869503+0000","last_undegraded":"2023-10-09T09:18:51.869503+0000","last_fullsized":"2023-10-09T09:18:51.869503+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":55,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.266713+0000","last_change":"2023-10-09T09:17:58.259851+0000","last_active":"2023-10-09T09:18:23.266713+0000","last_peered":"2023-10-09T09:18:23.266713+0000","last_clean":"2023-10-09T09:18:23.266713+0000","last_became_active":"2023-10-09T09:17:58.259311+0000","last_became_peered":"2023-10-09T09:17:58.259311+0000","last_unstale":"2023-10-09T09:18:23.266713+0000","last_undegraded":"2023-10-09T09:18:23.266713+0000","last_fullsized":"2023-10-09T09:18:23.266713+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:31.053670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"301'2985","reported_seq":3000,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:55.597857+0000","last_change":"2023-10-09T09:18:10.915886+0000","last_active":"2023-10-09T09:18:55.597857+0000","last_peered":"2023-10-09T09:18:55.597857+0000","last_clean":"2023-10-09T09:18:55.597857+0000","last_became_active":"2023-10-09T09:18:10.915159+0000","last_became_peered":"2023-10-09T09:18:10.915159+0000","last_unstale":"2023-10-09T09:18:55.597857+0000","last_undegraded":"2023-10-09T09:18:55.597857+0000","last_fullsized":"2023-10-09T09:18:55.597857+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2985,"log_dups_size":0,"ondisk_log_size":2985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195624960,"num_objects":2985,"num_object_clones":0,"num_object_copies":8955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2985,"num_write_kb":191040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.849397+0000","last_change":"2023-10-09T09:17:58.254057+0000","last_active":"2023-10-09T09:18:51.849397+0000","last_peered":"2023-10-09T09:18:51.849397+0000","last_clean":"2023-10-09T09:18:51.849397+0000","last_became_active":"2023-10-09T09:17:58.253674+0000","last_became_peered":"2023-10-09T09:17:58.253674+0000","last_unstale":"2023-10-09T09:18:51.849397+0000","last_undegraded":"2023-10-09T09:18:51.849397+0000","last_fullsized":"2023-10-09T09:18:51.849397+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:20.803316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.265034+0000","last_change":"2023-10-09T09:17:58.253433+0000","last_active":"2023-10-09T09:18:23.265034+0000","last_peered":"2023-10-09T09:18:23.265034+0000","last_clean":"2023-10-09T09:18:23.265034+0000","last_became_active":"2023-10-09T09:17:58.253124+0000","last_became_peered":"2023-10-09T09:17:58.253124+0000","last_unstale":"2023-10-09T09:18:23.265034+0000","last_undegraded":"2023-10-09T09:18:23.265034+0000","last_fullsized":"2023-10-09T09:18:23.265034+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:31.811437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.266978+0000","last_change":"2023-10-09T09:14:48.321029+0000","last_active":"2023-10-09T09:18:23.266978+0000","last_peered":"2023-10-09T09:18:23.266978+0000","last_clean":"2023-10-09T09:18:23.266978+0000","last_became_active":"2023-10-09T09:14:48.320435+0000","last_became_peered":"2023-10-09T09:14:48.320435+0000","last_unstale":"2023-10-09T09:18:23.266978+0000","last_undegraded":"2023-10-09T09:18:23.266978+0000","last_fullsized":"2023-10-09T09:18:23.266978+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:20.309472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"301'3090","reported_seq":3105,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:55.613645+0000","last_change":"2023-10-09T09:18:10.917223+0000","last_active":"2023-10-09T09:18:55.613645+0000","last_peered":"2023-10-09T09:18:55.613645+0000","last_clean":"2023-10-09T09:18:55.613645+0000","last_became_active":"2023-10-09T09:18:10.916265+0000","last_became_peered":"2023-10-09T09:18:10.916265+0000","last_unstale":"2023-10-09T09:18:55.613645+0000","last_undegraded":"2023-10-09T09:18:55.613645+0000","last_fullsized":"2023-10-09T09:18:55.613645+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":3090,"log_dups_size":0,"ondisk_log_size":3090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202506240,"num_objects":3090,"num_object_clones":0,"num_object_copies":9270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3090,"num_write_kb":197760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":116,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.853957+0000","last_change":"2023-10-09T09:17:58.255222+0000","last_active":"2023-10-09T09:18:51.853957+0000","last_peered":"2023-10-09T09:18:51.853957+0000","last_clean":"2023-10-09T09:18:51.853957+0000","last_became_active":"2023-10-09T09:17:58.254890+0000","last_became_peered":"2023-10-09T09:17:58.254890+0000","last_unstale":"2023-10-09T09:18:51.853957+0000","last_undegraded":"2023-10-09T09:18:51.853957+0000","last_fullsized":"2023-10-09T09:18:51.853957+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":539,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.859696+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:18:51.859696+0000","last_peered":"2023-10-09T09:18:51.859696+0000","last_clean":"2023-10-09T09:18:51.859696+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:18:51.859696+0000","last_undegraded":"2023-10-09T09:18:51.859696+0000","last_fullsized":"2023-10-09T09:18:51.859696+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":128,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854362+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:18:51.854362+0000","last_peered":"2023-10-09T09:18:51.854362+0000","last_clean":"2023-10-09T09:18:51.854362+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:18:51.854362+0000","last_undegraded":"2023-10-09T09:18:51.854362+0000","last_fullsized":"2023-10-09T09:18:51.854362+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":159,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.855915+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:18:51.855915+0000","last_peered":"2023-10-09T09:18:51.855915+0000","last_clean":"2023-10-09T09:18:51.855915+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:18:51.855915+0000","last_undegraded":"2023-10-09T09:18:51.855915+0000","last_fullsized":"2023-10-09T09:18:51.855915+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"301'3056","reported_seq":3067,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:55.602396+0000","last_change":"2023-10-09T09:18:10.915823+0000","last_active":"2023-10-09T09:18:55.602396+0000","last_peered":"2023-10-09T09:18:55.602396+0000","last_clean":"2023-10-09T09:18:55.602396+0000","last_became_active":"2023-10-09T09:18:10.915054+0000","last_became_peered":"2023-10-09T09:18:10.915054+0000","last_unstale":"2023-10-09T09:18:55.602396+0000","last_undegraded":"2023-10-09T09:18:55.602396+0000","last_fullsized":"2023-10-09T09:18:55.602396+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":3056,"log_dups_size":0,"ondisk_log_size":3056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200278016,"num_objects":3056,"num_object_clones":0,"num_object_copies":9168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3056,"num_write_kb":195584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.265272+0000","last_change":"2023-10-09T09:14:48.301697+0000","last_active":"2023-10-09T09:18:23.265272+0000","last_peered":"2023-10-09T09:18:23.265272+0000","last_clean":"2023-10-09T09:18:23.265272+0000","last_became_active":"2023-10-09T09:14:48.301379+0000","last_became_peered":"2023-10-09T09:14:48.301379+0000","last_unstale":"2023-10-09T09:18:23.265272+0000","last_undegraded":"2023-10-09T09:18:23.265272+0000","last_fullsized":"2023-10-09T09:18:23.265272+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:11.963433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":335,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.847648+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:18:50.847648+0000","last_peered":"2023-10-09T09:18:50.847648+0000","last_clean":"2023-10-09T09:18:50.847648+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:18:50.847648+0000","last_undegraded":"2023-10-09T09:18:50.847648+0000","last_fullsized":"2023-10-09T09:18:50.847648+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":341,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.849306+0000","last_change":"2023-10-09T09:17:58.260309+0000","last_active":"2023-10-09T09:18:50.849306+0000","last_peered":"2023-10-09T09:18:50.849306+0000","last_clean":"2023-10-09T09:18:50.849306+0000","last_became_active":"2023-10-09T09:17:58.259848+0000","last_became_peered":"2023-10-09T09:17:58.259848+0000","last_unstale":"2023-10-09T09:18:50.849306+0000","last_undegraded":"2023-10-09T09:18:50.849306+0000","last_fullsized":"2023-10-09T09:18:50.849306+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:52:37.788240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"299'2774","reported_seq":2788,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.702982+0000","last_change":"2023-10-09T09:18:10.373879+0000","last_active":"2023-10-09T09:18:51.702982+0000","last_peered":"2023-10-09T09:18:51.702982+0000","last_clean":"2023-10-09T09:18:51.702982+0000","last_became_active":"2023-10-09T09:18:10.373558+0000","last_became_peered":"2023-10-09T09:18:10.373558+0000","last_unstale":"2023-10-09T09:18:51.702982+0000","last_undegraded":"2023-10-09T09:18:51.702982+0000","last_fullsized":"2023-10-09T09:18:51.702982+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2774,"log_dups_size":0,"ondisk_log_size":2774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:19.114672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181796864,"num_objects":2774,"num_object_clones":0,"num_object_copies":8322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2774,"num_write_kb":177536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":81,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.860130+0000","last_change":"2023-10-09T09:09:44.439090+0000","last_active":"2023-10-09T09:18:51.860130+0000","last_peered":"2023-10-09T09:18:51.860130+0000","last_clean":"2023-10-09T09:18:51.860130+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:18:51.860130+0000","last_undegraded":"2023-10-09T09:18:51.860130+0000","last_fullsized":"2023-10-09T09:18:51.860130+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.266271+0000","last_change":"2023-10-09T09:17:58.260007+0000","last_active":"2023-10-09T09:18:23.266271+0000","last_peered":"2023-10-09T09:18:23.266271+0000","last_clean":"2023-10-09T09:18:23.266271+0000","last_became_active":"2023-10-09T09:17:58.259635+0000","last_became_peered":"2023-10-09T09:17:58.259635+0000","last_unstale":"2023-10-09T09:18:23.266271+0000","last_undegraded":"2023-10-09T09:18:23.266271+0000","last_fullsized":"2023-10-09T09:18:23.266271+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:40.824291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":311,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.842818+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:18:50.842818+0000","last_peered":"2023-10-09T09:18:50.842818+0000","last_clean":"2023-10-09T09:18:50.842818+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:18:50.842818+0000","last_undegraded":"2023-10-09T09:18:50.842818+0000","last_fullsized":"2023-10-09T09:18:50.842818+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"299'2825","reported_seq":2839,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.682098+0000","last_change":"2023-10-09T09:18:10.371750+0000","last_active":"2023-10-09T09:18:51.682098+0000","last_peered":"2023-10-09T09:18:51.682098+0000","last_clean":"2023-10-09T09:18:51.682098+0000","last_became_active":"2023-10-09T09:18:10.371244+0000","last_became_peered":"2023-10-09T09:18:10.371244+0000","last_unstale":"2023-10-09T09:18:51.682098+0000","last_undegraded":"2023-10-09T09:18:51.682098+0000","last_fullsized":"2023-10-09T09:18:51.682098+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":2825,"log_dups_size":0,"ondisk_log_size":2825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:49.380410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139200,"num_objects":2825,"num_object_clones":0,"num_object_copies":8475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2825,"num_write_kb":180800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":68,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.848908+0000","last_change":"2023-10-09T09:14:19.749169+0000","last_active":"2023-10-09T09:18:50.848908+0000","last_peered":"2023-10-09T09:18:50.848908+0000","last_clean":"2023-10-09T09:18:50.848908+0000","last_became_active":"2023-10-09T09:14:19.740888+0000","last_became_peered":"2023-10-09T09:14:19.740888+0000","last_unstale":"2023-10-09T09:18:50.848908+0000","last_undegraded":"2023-10-09T09:18:50.848908+0000","last_fullsized":"2023-10-09T09:18:50.848908+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:01.361061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T09:18:23.267264+0000","last_change":"2023-10-09T09:14:48.317228+0000","last_active":"2023-10-09T09:18:23.267264+0000","last_peered":"2023-10-09T09:18:23.267264+0000","last_clean":"2023-10-09T09:18:23.267264+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:18:23.267264+0000","last_undegraded":"2023-10-09T09:18:23.267264+0000","last_fullsized":"2023-10-09T09:18:23.267264+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.859445+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:18:51.859445+0000","last_peered":"2023-10-09T09:18:51.859445+0000","last_clean":"2023-10-09T09:18:51.859445+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:18:51.859445+0000","last_undegraded":"2023-10-09T09:18:51.859445+0000","last_fullsized":"2023-10-09T09:18:51.859445+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"301'3059","reported_seq":3072,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T09:18:56.113456+0000","last_change":"2023-10-09T09:18:10.373277+0000","last_active":"2023-10-09T09:18:56.113456+0000","last_peered":"2023-10-09T09:18:56.113456+0000","last_clean":"2023-10-09T09:18:56.113456+0000","last_became_active":"2023-10-09T09:18:10.372954+0000","last_became_peered":"2023-10-09T09:18:10.372954+0000","last_unstale":"2023-10-09T09:18:56.113456+0000","last_undegraded":"2023-10-09T09:18:56.113456+0000","last_fullsized":"2023-10-09T09:18:56.113456+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":3059,"log_dups_size":0,"ondisk_log_size":3059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200474624,"num_objects":3059,"num_object_clones":0,"num_object_copies":9177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3059,"num_write_kb":195776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":177,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854041+0000","last_change":"2023-10-09T09:17:58.250086+0000","last_active":"2023-10-09T09:18:51.854041+0000","last_peered":"2023-10-09T09:18:51.854041+0000","last_clean":"2023-10-09T09:18:51.854041+0000","last_became_active":"2023-10-09T09:17:58.249564+0000","last_became_peered":"2023-10-09T09:17:58.249564+0000","last_unstale":"2023-10-09T09:18:51.854041+0000","last_undegraded":"2023-10-09T09:18:51.854041+0000","last_fullsized":"2023-10-09T09:18:51.854041+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":354,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.849464+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:18:50.849464+0000","last_peered":"2023-10-09T09:18:50.849464+0000","last_clean":"2023-10-09T09:18:50.849464+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:18:50.849464+0000","last_undegraded":"2023-10-09T09:18:50.849464+0000","last_fullsized":"2023-10-09T09:18:50.849464+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":181,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869018+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:18:51.869018+0000","last_peered":"2023-10-09T09:18:51.869018+0000","last_clean":"2023-10-09T09:18:51.869018+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:18:51.869018+0000","last_undegraded":"2023-10-09T09:18:51.869018+0000","last_fullsized":"2023-10-09T09:18:51.869018+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":322,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.842726+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:18:50.842726+0000","last_peered":"2023-10-09T09:18:50.842726+0000","last_clean":"2023-10-09T09:18:50.842726+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:18:50.842726+0000","last_undegraded":"2023-10-09T09:18:50.842726+0000","last_fullsized":"2023-10-09T09:18:50.842726+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":19,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.850385+0000","last_change":"2023-10-09T09:17:58.252585+0000","last_active":"2023-10-09T09:18:51.850385+0000","last_peered":"2023-10-09T09:18:51.850385+0000","last_clean":"2023-10-09T09:18:51.850385+0000","last_became_active":"2023-10-09T09:17:58.252048+0000","last_became_peered":"2023-10-09T09:17:58.252048+0000","last_unstale":"2023-10-09T09:18:51.850385+0000","last_undegraded":"2023-10-09T09:18:51.850385+0000","last_fullsized":"2023-10-09T09:18:51.850385+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:28.682751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":106,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854002+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:18:51.854002+0000","last_peered":"2023-10-09T09:18:51.854002+0000","last_clean":"2023-10-09T09:18:51.854002+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:18:51.854002+0000","last_undegraded":"2023-10-09T09:18:51.854002+0000","last_fullsized":"2023-10-09T09:18:51.854002+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":181,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.869288+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:18:51.869288+0000","last_peered":"2023-10-09T09:18:51.869288+0000","last_clean":"2023-10-09T09:18:51.869288+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:18:51.869288+0000","last_undegraded":"2023-10-09T09:18:51.869288+0000","last_fullsized":"2023-10-09T09:18:51.869288+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":93,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.854121+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:18:51.854121+0000","last_peered":"2023-10-09T09:18:51.854121+0000","last_clean":"2023-10-09T09:18:51.854121+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:18:51.854121+0000","last_undegraded":"2023-10-09T09:18:51.854121+0000","last_fullsized":"2023-10-09T09:18:51.854121+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":163,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.868994+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:18:51.868994+0000","last_peered":"2023-10-09T09:18:51.868994+0000","last_clean":"2023-10-09T09:18:51.868994+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:18:51.868994+0000","last_undegraded":"2023-10-09T09:18:51.868994+0000","last_fullsized":"2023-10-09T09:18:51.868994+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":264,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.857125+0000","last_change":"2023-10-09T09:17:58.250076+0000","last_active":"2023-10-09T09:18:51.857125+0000","last_peered":"2023-10-09T09:18:51.857125+0000","last_clean":"2023-10-09T09:18:51.857125+0000","last_became_active":"2023-10-09T09:17:58.249548+0000","last_became_peered":"2023-10-09T09:17:58.249548+0000","last_unstale":"2023-10-09T09:18:51.857125+0000","last_undegraded":"2023-10-09T09:18:51.857125+0000","last_fullsized":"2023-10-09T09:18:51.857125+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":19,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.849536+0000","last_change":"2023-10-09T09:17:58.254648+0000","last_active":"2023-10-09T09:18:51.849536+0000","last_peered":"2023-10-09T09:18:51.849536+0000","last_clean":"2023-10-09T09:18:51.849536+0000","last_became_active":"2023-10-09T09:17:58.254270+0000","last_became_peered":"2023-10-09T09:17:58.254270+0000","last_unstale":"2023-10-09T09:18:51.849536+0000","last_undegraded":"2023-10-09T09:18:51.849536+0000","last_fullsized":"2023-10-09T09:18:51.849536+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:44.095656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":411,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T09:18:50.848824+0000","last_change":"2023-10-09T09:17:58.354204+0000","last_active":"2023-10-09T09:18:50.848824+0000","last_peered":"2023-10-09T09:18:50.848824+0000","last_clean":"2023-10-09T09:18:50.848824+0000","last_became_active":"2023-10-09T09:17:58.260303+0000","last_became_peered":"2023-10-09T09:17:58.260303+0000","last_unstale":"2023-10-09T09:18:50.848824+0000","last_undegraded":"2023-10-09T09:18:50.848824+0000","last_fullsized":"2023-10-09T09:18:50.848824+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:28.585571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":190,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T09:18:51.868528+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:18:51.868528+0000","last_peered":"2023-10-09T09:18:51.868528+0000","last_clean":"2023-10-09T09:18:51.868528+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:18:51.868528+0000","last_undegraded":"2023-10-09T09:18:51.868528+0000","last_fullsized":"2023-10-09T09:18:51.868528+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3035037696,"num_objects":46311,"num_object_clones":0,"num_object_copies":138933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46311,"num_write_kb":2963904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9181331456,"data_stored":9181331456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46311,"ondisk_log_size":46311,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":19794,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430781,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1795392,"kb_used_data":1209784,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91949760,"statfs":{"total":95995035648,"available":94156554240,"internally_reserved":0,"allocated":1238818816,"data_stored":1238226477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:18:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":2.0619999999999998,"15min":2.863},"min":{"1min":0.41099999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.607,"5min":50.515999999999998,"15min":119.498},"last":3.1120000000000001},{"interface":"front","average":{"1min":0.88200000000000001,"5min":2.0489999999999999,"15min":2.8479999999999999},"min":{"1min":0.47499999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":1.5389999999999999,"5min":50.621000000000002,"15min":119.532},"last":3.125}]},{"osd":1,"last update":"Mon Oct 9 09:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.621,"15min":1.1479999999999999},"min":{"1min":0.68300000000000005,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.873,"5min":21.800999999999998,"15min":21.800999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.74,"15min":1.0780000000000001},"min":{"1min":0.61799999999999999,"5min":0.316,"15min":0.316},"max":{"1min":1.675,"5min":21.734000000000002,"15min":21.734000000000002},"last":0.53900000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:18:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.421,"15min":2.052},"min":{"1min":0.52200000000000002,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":1.9450000000000001,"5min":8.1120000000000001,"15min":126.072},"last":1.3879999999999999},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.4930000000000001,"15min":2.0880000000000001},"min":{"1min":0.38200000000000001,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":2.407,"5min":15.597,"15min":120.729},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.5680000000000001,"15min":3.5289999999999999},"min":{"1min":0.56599999999999995,"5min":0.375,"15min":0.316},"max":{"1min":1.585,"5min":15.592000000000001,"15min":191.30799999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.998,"5min":1.391,"15min":3.5110000000000001},"min":{"1min":0.629,"5min":0.34999999999999998,"15min":0.318},"max":{"1min":1.659,"5min":8.1229999999999993,"15min":191.47900000000001},"last":1.5429999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:18:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.8560000000000001,"15min":1.4530000000000001},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.6579999999999999,"5min":11.371,"15min":13.895},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.995,"5min":1.8939999999999999,"15min":1.4730000000000001},"min":{"1min":0.51300000000000001,"5min":0.314,"15min":0.314},"max":{"1min":2.7429999999999999,"5min":8.9879999999999995,"15min":16.433},"last":0.52500000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:18:05 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":1.948,"15min":1.3300000000000001},"min":{"1min":0.47499999999999998,"5min":0.34599999999999997,"15min":0.28199999999999997},"max":{"1min":1.4179999999999999,"5min":15.616,"15min":15.616},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.319,"15min":1.6040000000000001},"min":{"1min":0.44500000000000001,"5min":0.36599999999999999,"15min":0.34499999999999997},"max":{"1min":2.3540000000000001,"5min":18.501000000000001,"15min":18.501000000000001},"last":0.628}]},{"osd":6,"last update":"Mon Oct 9 09:18:05 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.9810000000000001,"15min":1.5449999999999999},"min":{"1min":0.55000000000000004,"5min":0.39300000000000002,"15min":0.35599999999999998},"max":{"1min":2.3959999999999999,"5min":21.145,"15min":21.145},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.077,"5min":2.0920000000000001,"15min":1.661},"min":{"1min":0.59299999999999997,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.7450000000000001,"5min":12.269,"15min":19.068999999999999},"last":0.65300000000000002}]}]},{"osd":6,"up_from":42,"seq":180388627014,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2708724,"kb_used_data":1516972,"kb_used_omap":0,"kb_used_meta":1191744,"kb_avail":91036428,"statfs":{"total":95995035648,"available":93221302272,"internally_reserved":0,"allocated":1553379328,"data_stored":1552799081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1220345856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":4.1879999999999997,"15min":2.7799999999999998},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26200000000000001},"max":{"1min":5.7270000000000003,"5min":120.812,"15min":120.812},"last":1.9219999999999999},{"interface":"front","average":{"1min":1.367,"5min":4.1890000000000001,"15min":2.73},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.24099999999999999},"max":{"1min":5.883,"5min":120.857,"15min":120.857},"last":0.81699999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.617,"15min":1.2949999999999999},"min":{"1min":0.61299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.129,"5min":10.657999999999999,"15min":16.879000000000001},"last":4.1509999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.337,"15min":1.214},"min":{"1min":0.437,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3520000000000001,"5min":18.079999999999998,"15min":18.079999999999998},"last":0.72299999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.4870000000000001,"15min":1.6000000000000001},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.4820000000000002,"5min":10.590999999999999,"15min":18.756},"last":2.097},{"interface":"front","average":{"1min":1.109,"5min":1.5269999999999999,"15min":1.575},"min":{"1min":0.52200000000000002,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":2.5960000000000001,"5min":18.170000000000002,"15min":18.841999999999999},"last":0.88900000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.681,"15min":1.5920000000000001},"min":{"1min":0.60299999999999998,"5min":0.33700000000000002,"15min":0.28999999999999998},"max":{"1min":1.546,"5min":10.630000000000001,"15min":16.977},"last":3.5779999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.2929999999999999,"15min":1.6299999999999999},"min":{"1min":0.59299999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.1930000000000001,"5min":9.0839999999999996,"15min":20.573},"last":0.64100000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":2.379,"15min":1.633},"min":{"1min":0.48199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.327,"5min":29.585999999999999,"15min":29.585999999999999},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":2.306,"15min":1.6020000000000001},"min":{"1min":0.44500000000000001,"5min":0.38300000000000001,"15min":0.317},"max":{"1min":3.4420000000000002,"5min":20.318000000000001,"15min":20.318000000000001},"last":0.745}]},{"osd":5,"last update":"Mon Oct 9 09:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.1850000000000001,"15min":1.7070000000000001},"min":{"1min":0.53400000000000003,"5min":0.34300000000000003,"15min":0.30199999999999999},"max":{"1min":1.5309999999999999,"5min":18.103000000000002,"15min":18.103000000000002},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.99099999999999999,"5min":2.0739999999999998,"15min":1.5589999999999999},"min":{"1min":0.45700000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.395,"5min":11.567,"15min":13.609},"last":0.79100000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":2.129,"15min":1.6919999999999999},"min":{"1min":0.432,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.593,"5min":29.530000000000001,"15min":29.530000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.069,"5min":1.883,"15min":1.486},"min":{"1min":0.42299999999999999,"5min":0.39000000000000001,"15min":0.36199999999999999},"max":{"1min":2.4849999999999999,"5min":11.477,"15min":13.523999999999999},"last":0.83299999999999996}]}]},{"osd":1,"up_from":83,"seq":356482285961,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1852244,"kb_used_data":1381068,"kb_used_omap":9,"kb_used_meta":471158,"kb_avail":91892908,"statfs":{"total":95995035648,"available":94098337792,"internally_reserved":0,"allocated":1414213632,"data_stored":1413123070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":482466793},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:18:23 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.98499999999999999,"15min":1.7929999999999999},"min":{"1min":0.39100000000000001,"5min":0.215,"15min":0.189},"max":{"1min":1.2989999999999999,"5min":15.298,"15min":94.915999999999997},"last":1.869},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.274,"15min":2.0569999999999999},"min":{"1min":0.32900000000000001,"5min":0.23999999999999999,"15min":0.21199999999999999},"max":{"1min":1.0229999999999999,"5min":19.899000000000001,"15min":98.227999999999994},"last":0.76800000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.81200000000000006,"15min":1.4390000000000001},"min":{"1min":0.41899999999999998,"5min":0.27800000000000002,"15min":0.23799999999999999},"max":{"1min":3.3809999999999998,"5min":5.1289999999999996,"15min":46.354999999999997},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.60099999999999998,"15min":1.2110000000000001},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.252},"max":{"1min":1.0820000000000001,"5min":1.829,"15min":46.241999999999997},"last":2.3319999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:18:23 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.0680000000000001,"15min":1.79},"min":{"1min":0.48699999999999999,"5min":0.33700000000000002,"15min":0.26800000000000002},"max":{"1min":1.335,"5min":28.314,"15min":67.807000000000002},"last":2.452},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.139,"15min":1.8029999999999999},"min":{"1min":0.57899999999999996,"5min":0.315,"15min":0.22600000000000001},"max":{"1min":1.272,"5min":28.398,"15min":68.097999999999999},"last":1.8}]},{"osd":4,"last update":"Mon Oct 9 09:18:23 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.9350000000000001,"15min":1.3169999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.30099999999999999},"max":{"1min":1.7609999999999999,"5min":8.3059999999999992,"15min":8.3059999999999992},"last":2.5430000000000001},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.5649999999999999,"15min":1.1639999999999999},"min":{"1min":0.63400000000000001,"5min":0.36899999999999999,"15min":0.33100000000000002},"max":{"1min":1.6870000000000001,"5min":10.523999999999999,"15min":10.523999999999999},"last":1.841}]},{"osd":5,"last update":"Mon Oct 9 09:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.583,"15min":1.214},"min":{"1min":0.51900000000000002,"5min":0.34699999999999998,"15min":0.33800000000000002},"max":{"1min":1.3120000000000001,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.036,"5min":1.359,"15min":1.119},"min":{"1min":0.58399999999999996,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.6419999999999999,"5min":7.2160000000000002,"15min":7.6399999999999997},"last":2.496}]},{"osd":6,"last update":"Mon Oct 9 09:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.7130000000000001,"15min":1.361},"min":{"1min":0.59599999999999997,"5min":0.38300000000000001,"15min":0.33200000000000002},"max":{"1min":1.587,"5min":7.9480000000000004,"15min":7.9480000000000004},"last":1.9059999999999999},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.7370000000000001,"15min":1.3799999999999999},"min":{"1min":0.61599999999999999,"5min":0.42499999999999999,"15min":0.39000000000000001},"max":{"1min":1.5129999999999999,"5min":8.9079999999999995,"15min":8.9079999999999995},"last":0.72999999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.4350000000000001,"15min":1.1870000000000001},"min":{"1min":0.51700000000000002,"5min":0.42199999999999999,"15min":0.38500000000000001},"max":{"1min":1.849,"5min":9.6129999999999995,"15min":10.326000000000001},"last":2.4129999999999998},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.623,"15min":1.2749999999999999},"min":{"1min":0.70599999999999996,"5min":0.42099999999999999,"15min":0.38900000000000001},"max":{"1min":1.726,"5min":6.8330000000000002,"15min":7.6379999999999999},"last":2.1819999999999999}]}]},{"osd":0,"up_from":78,"seq":335007449487,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1946988,"kb_used_data":1047012,"kb_used_omap":28,"kb_used_meta":899939,"kb_avail":91798164,"statfs":{"total":95995035648,"available":94001319936,"internally_reserved":0,"allocated":1072140288,"data_stored":1071027846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":921538384},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:18:22 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.90800000000000003,"15min":1.054},"min":{"1min":0.42099999999999999,"5min":0.24099999999999999,"15min":0.193},"max":{"1min":1.1679999999999999,"5min":9.7620000000000005,"15min":9.7620000000000005},"last":1.133},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.97999999999999998,"15min":1.016},"min":{"1min":0.438,"5min":0.254,"15min":0.22800000000000001},"max":{"1min":1.1919999999999999,"5min":6.6040000000000001,"15min":10.102},"last":0.375}]},{"osd":2,"last update":"Mon Oct 9 09:18:07 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.913,"15min":2.093},"min":{"1min":0.36699999999999999,"5min":0.27200000000000002,"15min":0.23400000000000001},"max":{"1min":1.345,"5min":74.200999999999993,"15min":74.200999999999993},"last":2.7320000000000002},{"interface":"front","average":{"1min":0.873,"5min":2.125,"15min":2.9670000000000001},"min":{"1min":0.51600000000000001,"5min":0.35299999999999998,"15min":0.24399999999999999},"max":{"1min":1.365,"5min":73.983999999999995,"15min":219.00700000000001},"last":2.931}]},{"osd":3,"last update":"Mon Oct 9 09:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":1.2210000000000001,"15min":2.637},"min":{"1min":0.45300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.194,"5min":9.8239999999999998,"15min":219.041},"last":3.855},{"interface":"front","average":{"1min":0.84099999999999997,"5min":1.2210000000000001,"15min":2.157},"min":{"1min":0.46899999999999997,"5min":0.32500000000000001,"15min":0.29499999999999998},"max":{"1min":1.337,"5min":6.8079999999999998,"15min":137.94999999999999},"last":0.504}]},{"osd":4,"last update":"Mon Oct 9 09:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":2.387,"15min":1.7629999999999999},"min":{"1min":0.64500000000000002,"5min":0.34000000000000002,"15min":0.318},"max":{"1min":4.9669999999999996,"5min":14.163,"15min":14.163},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.022,"5min":2.2570000000000001,"15min":1.889},"min":{"1min":0.52100000000000002,"5min":0.35499999999999998,"15min":0.29099999999999998},"max":{"1min":5.4249999999999998,"5min":11.346,"15min":11.346},"last":1.337}]},{"osd":5,"last update":"Mon Oct 9 09:18:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.0369999999999999,"15min":1.6990000000000001},"min":{"1min":0.64000000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.252,"5min":9.9770000000000003,"15min":9.9770000000000003},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.714,"15min":1.6539999999999999},"min":{"1min":0.68300000000000005,"5min":0.40699999999999997,"15min":0.35299999999999998},"max":{"1min":4.7590000000000003,"5min":9.5909999999999993,"15min":16.364000000000001},"last":0.54500000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":2.0089999999999999,"15min":2.141},"min":{"1min":0.64900000000000002,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":3.9390000000000001,"5min":11.314,"15min":98.864000000000004},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.2190000000000001,"5min":2.226,"15min":2.2349999999999999},"min":{"1min":0.75900000000000001,"5min":0.44,"15min":0.38300000000000001},"max":{"1min":3.9540000000000002,"5min":12.561999999999999,"15min":98.878},"last":0.91400000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.8979999999999999,"15min":1.6599999999999999},"min":{"1min":0.45100000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.637,"5min":9.6780000000000008,"15min":10.157999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.085,"5min":2.153,"15min":1.8440000000000001},"min":{"1min":0.77000000000000002,"5min":0.41199999999999998,"15min":0.40100000000000002},"max":{"1min":3.468,"5min":10.734,"15min":10.734},"last":1.375}]}]},{"osd":2,"up_from":88,"seq":377957122435,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2537244,"kb_used_data":1714516,"kb_used_omap":81,"kb_used_meta":822638,"kb_avail":91207908,"statfs":{"total":95995035648,"available":93396897792,"internally_reserved":0,"allocated":1755664384,"data_stored":1754520073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":842381530},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:18:06 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.996,"15min":2.246},"min":{"1min":0.46800000000000003,"5min":0.23899999999999999,"15min":0.188},"max":{"1min":1.1100000000000001,"5min":7.6820000000000004,"15min":85.091999999999999},"last":2.5019999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.0069999999999999,"15min":2.194},"min":{"1min":0.44,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":1.405,"5min":8.7360000000000007,"15min":84.373999999999995},"last":2.6400000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:18:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":2.5129999999999999,"15min":1.615},"min":{"1min":0.39700000000000002,"5min":0.30599999999999999,"15min":0.214},"max":{"1min":1.169,"5min":122.367,"15min":122.367},"last":2.5739999999999998},{"interface":"front","average":{"1min":0.83299999999999996,"5min":2.4540000000000002,"15min":1.5029999999999999},"min":{"1min":0.44700000000000001,"5min":0.315,"15min":0.24199999999999999},"max":{"1min":1.234,"5min":122.777,"15min":122.777},"last":2.54}]},{"osd":3,"last update":"Mon Oct 9 09:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.0249999999999999,"15min":1.786},"min":{"1min":0.51600000000000001,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":1.3959999999999999,"5min":8.6519999999999992,"15min":73.221999999999994},"last":2.6800000000000002},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.21,"15min":1.851},"min":{"1min":0.39200000000000002,"5min":0.27100000000000002,"15min":0.20100000000000001},"max":{"1min":1.5009999999999999,"5min":19.321999999999999,"15min":75.840999999999994},"last":0.502}]},{"osd":4,"last update":"Mon Oct 9 09:18:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.5229999999999999,"15min":1.6359999999999999},"min":{"1min":0.377,"5min":0.35999999999999999,"15min":0.316},"max":{"1min":1.1619999999999999,"5min":14.343999999999999,"15min":17.408999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.7769999999999999,"15min":1.627},"min":{"1min":0.52600000000000002,"5min":0.42299999999999999,"15min":0.29699999999999999},"max":{"1min":1.2929999999999999,"5min":12.351000000000001,"15min":12.351000000000001},"last":2.8719999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:18:06 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.3120000000000001,"15min":1.575},"min":{"1min":0.41599999999999998,"5min":0.36699999999999999,"15min":0.35799999999999998},"max":{"1min":1.8700000000000001,"5min":8.0239999999999991,"15min":72.311000000000007},"last":2.754},{"interface":"front","average":{"1min":1.095,"5min":1.5609999999999999,"15min":1.9299999999999999},"min":{"1min":0.61299999999999999,"5min":0.371,"15min":0.36899999999999999},"max":{"1min":3.3039999999999998,"5min":13.303000000000001,"15min":73.042000000000002},"last":0.54200000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:18:06 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.913,"15min":1.6759999999999999},"min":{"1min":0.65200000000000002,"5min":0.40100000000000002,"15min":0.34300000000000003},"max":{"1min":4.798,"5min":14.714,"15min":14.714},"last":2.9529999999999998},{"interface":"front","average":{"1min":1.155,"5min":1.9930000000000001,"15min":1.6240000000000001},"min":{"1min":0.76100000000000001,"5min":0.432,"15min":0.40999999999999998},"max":{"1min":1.653,"5min":14.571999999999999,"15min":14.571999999999999},"last":2.7149999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:18:06 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.5920000000000001,"15min":1.768},"min":{"1min":0.55600000000000005,"5min":0.47699999999999998,"15min":0.40200000000000002},"max":{"1min":1.538,"5min":9.3529999999999998,"15min":67.491},"last":2.915},{"interface":"front","average":{"1min":1.105,"5min":1.367,"15min":1.4359999999999999},"min":{"1min":0.45300000000000001,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.768,"5min":4.3550000000000004,"15min":8.8789999999999996},"last":2.8119999999999998}]}]},{"osd":3,"up_from":93,"seq":399431958910,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2375784,"kb_used_data":1487648,"kb_used_omap":7,"kb_used_meta":888120,"kb_avail":91369368,"statfs":{"total":95995035648,"available":93562232832,"internally_reserved":0,"allocated":1523351552,"data_stored":1522255217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":909435496},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":2.0219999999999998,"15min":2.7749999999999999},"min":{"1min":0.47499999999999998,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":0.94499999999999995,"5min":46.433999999999997,"15min":233.22900000000001},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.78000000000000003,"5min":2.149,"15min":3.0830000000000002},"min":{"1min":0.52600000000000002,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":1.167,"5min":46.387,"15min":237.447},"last":0.64200000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.087,"15min":1.1890000000000001},"min":{"1min":0.44900000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.1859999999999999,"5min":32.520000000000003,"15min":32.520000000000003},"last":4.7320000000000002},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.036,"15min":1.0249999999999999},"min":{"1min":0.54100000000000004,"5min":0.28100000000000003,"15min":0.22900000000000001},"max":{"1min":1.032,"5min":32.585999999999999,"15min":32.585999999999999},"last":3.0800000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.92700000000000005,"15min":1.948},"min":{"1min":0.56699999999999995,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.135,"5min":8.3130000000000006,"15min":129.67699999999999},"last":2.5430000000000001},{"interface":"front","average":{"1min":0.874,"5min":0.79900000000000004,"15min":1.8560000000000001},"min":{"1min":0.53200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.238,"5min":6.0490000000000004,"15min":130.40000000000001},"last":2.8679999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.5089999999999999,"15min":1.3500000000000001},"min":{"1min":0.63400000000000001,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":1.411,"5min":7.5519999999999996,"15min":8.9879999999999995},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.8400000000000001,"15min":1.7190000000000001},"min":{"1min":0.46500000000000002,"5min":0.34999999999999998,"15min":0.313},"max":{"1min":1.4670000000000001,"5min":12.617000000000001,"15min":12.617000000000001},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.641,"15min":1.6850000000000001},"min":{"1min":0.66800000000000004,"5min":0.38700000000000001,"15min":0.38100000000000001},"max":{"1min":1.304,"5min":7.819,"15min":16.132999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.04,"5min":1.754,"15min":1.663},"min":{"1min":0.70899999999999996,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.343,"5min":9.5210000000000008,"15min":14.361000000000001},"last":0.49099999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.8660000000000001,"15min":1.8740000000000001},"min":{"1min":0.76800000000000002,"5min":0.37,"15min":0.308},"max":{"1min":1.387,"5min":9.1159999999999997,"15min":12.105},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.054,"5min":1.79,"15min":1.77},"min":{"1min":0.79700000000000004,"5min":0.40899999999999997,"15min":0.38500000000000001},"max":{"1min":1.4279999999999999,"5min":9.0199999999999996,"15min":9.5090000000000003},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:18:14 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.53,"15min":1.631},"min":{"1min":0.876,"5min":0.45100000000000001,"15min":0.41499999999999998},"max":{"1min":1.407,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.099,"5min":1.462,"15min":1.4810000000000001},"min":{"1min":0.69599999999999995,"5min":0.39700000000000002,"15min":0.38300000000000001},"max":{"1min":1.536,"5min":8.8930000000000007,"15min":9.1560000000000006},"last":0.79400000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019480,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1662204,"kb_used_data":772212,"kb_used_omap":0,"kb_used_meta":889984,"kb_avail":92082948,"statfs":{"total":95995035648,"available":94292938752,"internally_reserved":0,"allocated":790745088,"data_stored":790156496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911343616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:18:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":2.0699999999999998,"15min":2.1880000000000002},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":9.7850000000000001,"5min":16.170999999999999,"15min":116.59999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.843,"15min":2.173},"min":{"1min":0.30299999999999999,"5min":0.30199999999999999,"15min":0.25800000000000001},"max":{"1min":10.458,"5min":16.143999999999998,"15min":116.843},"last":0.40400000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.339,"15min":1.0720000000000001},"min":{"1min":0.47499999999999998,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":1.3580000000000001,"5min":7.8529999999999998,"15min":13.853999999999999},"last":1.907},{"interface":"front","average":{"1min":0.871,"5min":1.75,"15min":1.216},"min":{"1min":0.51800000000000002,"5min":0.30299999999999999,"15min":0.30099999999999999},"max":{"1min":1.4590000000000001,"5min":11.391999999999999,"15min":11.391999999999999},"last":3.2559999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.77,"15min":1.9039999999999999},"min":{"1min":0.374,"5min":0.33800000000000002,"15min":0.29299999999999998},"max":{"1min":1.716,"5min":11.358000000000001,"15min":116.7},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.79200000000000004,"5min":1.6220000000000001,"15min":1.9550000000000001},"min":{"1min":0.39700000000000002,"5min":0.34699999999999998,"15min":0.315},"max":{"1min":1.6890000000000001,"5min":11.1,"15min":116.63800000000001},"last":0.64700000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.5,"15min":3.742},"min":{"1min":0.57499999999999996,"5min":0.34799999999999998,"15min":0.29899999999999999},"max":{"1min":1.655,"5min":7.891,"15min":165.995},"last":1.7070000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.9370000000000001,"15min":4.0679999999999996},"min":{"1min":0.59799999999999998,"5min":0.34999999999999998,"15min":0.314},"max":{"1min":1.28,"5min":12.207000000000001,"15min":165.53200000000001},"last":0.499}]},{"osd":5,"last update":"Mon Oct 9 09:18:34 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":2.4409999999999998,"15min":1.5820000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.30099999999999999},"max":{"1min":1.607,"5min":13.907,"15min":13.907},"last":0.436},{"interface":"front","average":{"1min":0.79700000000000004,"5min":1.8819999999999999,"15min":1.3939999999999999},"min":{"1min":0.46500000000000002,"5min":0.376,"15min":0.34499999999999997},"max":{"1min":1.4410000000000001,"5min":11.042999999999999,"15min":11.042999999999999},"last":0.56000000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:18:06 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.319,"15min":1.742},"min":{"1min":0.49399999999999999,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":1.204,"5min":18.103999999999999,"15min":74.700999999999993},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":2.6629999999999998,"15min":1.8879999999999999},"min":{"1min":0.50800000000000001,"5min":0.42999999999999999,"15min":0.33000000000000002},"max":{"1min":1.3240000000000001,"5min":13.444000000000001,"15min":73.572999999999993},"last":1.1439999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":2.3820000000000001,"15min":1.641},"min":{"1min":0.39800000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.5680000000000001,"5min":15.484999999999999,"15min":15.484999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.794,"15min":1.5209999999999999},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.37},"max":{"1min":1.76,"5min":11.090999999999999,"15min":13.916},"last":1.75}]}]},{"osd":5,"up_from":36,"seq":154618823248,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285416,"kb_used_data":384096,"kb_used_omap":0,"kb_used_meta":901312,"kb_avail":92459736,"statfs":{"total":95995035648,"available":94678769664,"internally_reserved":0,"allocated":393314304,"data_stored":392741866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922943488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":3.7120000000000002,"15min":3.536},"min":{"1min":0.27300000000000002,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":2.052,"5min":139.81700000000001,"15min":156.97800000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.0069999999999999,"5min":3.7280000000000002,"15min":3.5670000000000002},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":3.54,"5min":141.66900000000001,"15min":157.45699999999999},"last":0.42499999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:18:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.7769999999999999,"15min":1.3049999999999999},"min":{"1min":0.54700000000000004,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.3400000000000001,"5min":11.105,"15min":11.105},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.73499999999999999,"5min":1.514,"15min":1.2110000000000001},"min":{"1min":0.54500000000000004,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.0580000000000001,"5min":10.959,"15min":10.959},"last":1.728}]},{"osd":2,"last update":"Mon Oct 9 09:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":2.0579999999999998,"15min":2.6970000000000001},"min":{"1min":0.39700000000000002,"5min":0.29299999999999998,"15min":0.27700000000000002},"max":{"1min":1.3180000000000001,"5min":46.058999999999997,"15min":156.69499999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.149,"5min":2.181,"15min":2.5329999999999999},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.7519999999999998,"5min":46.030000000000001,"15min":156.87799999999999},"last":0.52400000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.847,"15min":2.8530000000000002},"min":{"1min":0.60999999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.3,"5min":11.427,"15min":161.678},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.833,"15min":2.992},"min":{"1min":0.63100000000000001,"5min":0.31,"15min":0.31},"max":{"1min":1.151,"5min":11.019,"15min":159.43899999999999},"last":3.819}]},{"osd":4,"last update":"Mon Oct 9 09:18:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":2.6120000000000001,"15min":1.8080000000000001},"min":{"1min":0.432,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.399,"5min":13.904,"15min":13.904},"last":1.056},{"interface":"front","average":{"1min":0.98199999999999998,"5min":2.7429999999999999,"15min":1.665},"min":{"1min":0.56899999999999995,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":1.415,"5min":22.228000000000002,"15min":22.228000000000002},"last":1.5409999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":2.0579999999999998,"15min":1.611},"min":{"1min":0.44800000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2290000000000001,"5min":11.125,"15min":16.052},"last":3.3279999999999998},{"interface":"front","average":{"1min":0.97699999999999998,"5min":2.375,"15min":1.6659999999999999},"min":{"1min":0.499,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.4379999999999999,"5min":11.863,"15min":11.863},"last":1.264}]},{"osd":7,"last update":"Mon Oct 9 09:18:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":2.169,"15min":1.4790000000000001},"min":{"1min":0.61799999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.486,"5min":12.159000000000001,"15min":12.159000000000001},"last":1.083},{"interface":"front","average":{"1min":0.94699999999999995,"5min":2.1709999999999998,"15min":1.554},"min":{"1min":0.45700000000000002,"5min":0.36399999999999999,"15min":0.33400000000000002},"max":{"1min":1.4850000000000001,"5min":11.090999999999999,"15min":11.090999999999999},"last":1.77}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1084227584,"data_stored":1084227584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1437073408,"data_stored":1437073408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1136918528,"data_stored":1136918528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1558446080,"data_stored":1558446080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":800653312,"data_stored":800653312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":397803520,"data_stored":397803520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1577385984,"data_stored":1577385984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1188823040,"data_stored":1188823040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:18:59.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-09T09:18:59.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:59.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:18:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:59.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:59.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:59.304 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:59.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:59 smithi094 bash[59677]: cluster 2023-10-09T09:18:58.195427+0000 mgr.y (mgr.25421) 1253 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:18:59.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:59 smithi094 bash[59677]: audit 2023-10-09T09:18:59.042231+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:59 smithi094 bash[59677]: audit 2023-10-09T09:18:59.061895+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:59 smithi094 bash[59677]: audit 2023-10-09T09:18:59.063560+0000 mon.a (mon.0) 1678 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:18:59.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:59 smithi094 bash[59677]: audit 2023-10-09T09:18:59.064910+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:18:59.312 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:18:59 smithi094 bash[59677]: audit 2023-10-09T09:18:59.080740+0000 mon.a (mon.0) 1680 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:59.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:59.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:59.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[75179]: cluster 2023-10-09T09:18:58.195427+0000 mgr.y (mgr.25421) 1253 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:18:59.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[75179]: audit 2023-10-09T09:18:59.042231+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[75179]: audit 2023-10-09T09:18:59.061895+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[75179]: audit 2023-10-09T09:18:59.063560+0000 mon.a (mon.0) 1678 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:18:59.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[75179]: audit 2023-10-09T09:18:59.064910+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:18:59.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[75179]: audit 2023-10-09T09:18:59.080740+0000 mon.a (mon.0) 1680 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.441 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[77892]: cluster 2023-10-09T09:18:58.195427+0000 mgr.y (mgr.25421) 1253 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:18:59.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[77892]: audit 2023-10-09T09:18:59.042231+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[77892]: audit 2023-10-09T09:18:59.061895+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[77892]: audit 2023-10-09T09:18:59.063560+0000 mon.a (mon.0) 1678 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:18:59.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[77892]: audit 2023-10-09T09:18:59.064910+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:18:59.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:18:59 smithi016 bash[77892]: audit 2023-10-09T09:18:59.080740+0000 mon.a (mon.0) 1680 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:18:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:59.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:59.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:59.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:59.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:18:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:59.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:18:59.610 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 52972 52956 97.3347 70.125 0.00762562 0.0102671 2023-10-09T09:18:59.705 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:59.733 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:18:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:59.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:18:59.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:59.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:18:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:18:59.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:18:59.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:00.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:00.122 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:00.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:00.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:00.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:00.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:00.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:00.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:00.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:00.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:00.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:00.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 54126 54110 96.6142 72.125 0.0121122 0.0103451 2023-10-09T09:19:00.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:00.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:00.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:00.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:00.816 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:00.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:00.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:00.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:00.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:00.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:00 smithi016 bash[75179]: cluster 2023-10-09T09:19:00.195909+0000 mgr.y (mgr.25421) 1254 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T09:19:01.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:00 smithi016 bash[77892]: cluster 2023-10-09T09:19:00.195909+0000 mgr.y (mgr.25421) 1254 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T09:19:01.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:01.055 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:00 smithi094 bash[59677]: cluster 2023-10-09T09:19:00.195909+0000 mgr.y (mgr.25421) 1254 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T09:19:01.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:01.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:01.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:01.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:01.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:01.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:01.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:01.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:01.345 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:01.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:01.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:01.449 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:01.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:01.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:01.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:01.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:01.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:01.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 55275 55259 95.9251 71.8125 0.0111235 0.0104193 2023-10-09T09:19:01.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:01.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:01.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:01.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:01.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:01.854 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:01.882 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:01.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:01.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:01.986 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:02.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:02.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:02.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:02.138 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:02.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:02.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:02.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:02.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:02.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:02.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:02.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:02.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:02.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 56415 56399 95.258 71.25 0.0101027 0.0104913 2023-10-09T09:19:02.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:02.655 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:02.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:02.799 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:02.829 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:02.933 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:02.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:02.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:03.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:03.092 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:19:03.093 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:19:03.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:03.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:03.198 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:03.220 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:03.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:03.324 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:03.350 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:03.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:03.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:03.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:03.464 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:03 smithi016 bash[75179]: cluster 2023-10-09T09:19:02.196660+0000 mgr.y (mgr.25421) 1255 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:19:03.465 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:03 smithi016 bash[77892]: cluster 2023-10-09T09:19:02.196660+0000 mgr.y (mgr.25421) 1255 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:19:03.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:03.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:03 smithi094 bash[59677]: cluster 2023-10-09T09:19:02.196660+0000 mgr.y (mgr.25421) 1255 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:19:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:03.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:03.595 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:03.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 57548 57532 94.6145 70.8125 0.0170275 0.0105618 2023-10-09T09:19:03.619 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:03.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:03.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:03.723 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:03.742 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:03.742 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:03.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:03.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:03.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:03.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:03.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:03.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:04.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:04.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:04.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:04.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:04.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:04.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:04.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:04.389 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:04.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:04.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:04 smithi016 bash[75179]: audit 2023-10-09T09:19:04.254805+0000 mon.a (mon.0) 1681 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:04.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:04 smithi016 bash[77892]: audit 2023-10-09T09:19:04.254805+0000 mon.a (mon.0) 1681 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:04.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:04 smithi094 bash[59677]: audit 2023-10-09T09:19:04.254805+0000 mon.a (mon.0) 1681 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:04.552 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:04.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:04.566 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:19:04.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 58653 58637 93.9592 69.0625 0.00452224 0.0106373 2023-10-09T09:19:04.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:04.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:04.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:04.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:04.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:04.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:04.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:04.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:05.058 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:05.093 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:05.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:05.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:05.239 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:05.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:05.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:05.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:05.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:05.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:05.479 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:05.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:05 smithi094 bash[59677]: cluster 2023-10-09T09:19:04.197252+0000 mgr.y (mgr.25421) 1256 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:19:05.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:05.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:05.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:05.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:05.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:19:05.611072+0000 min lat: 0.00130401 max lat: 0.384087 avg lat: 0.0107698 2023-10-09T09:19:05.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:19:05.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 59414 59398 92.7992 47.5625 0.00995024 0.0107698 2023-10-09T09:19:05.625 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:05 smithi016 bash[75179]: cluster 2023-10-09T09:19:04.197252+0000 mgr.y (mgr.25421) 1256 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:19:05.626 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:05 smithi016 bash[77892]: cluster 2023-10-09T09:19:04.197252+0000 mgr.y (mgr.25421) 1256 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:19:05.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:05.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:05.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:05.741 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:05.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:05.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:05.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:05.884 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:19:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:19:05] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.33.4" 2023-10-09T09:19:05.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:05.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:06.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:06.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:06.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:06.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:06.146 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:06.219 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:06.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:06.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:06.353 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:06.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:06.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:06.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:06.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:06.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:06.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:06.613 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 60610 60594 92.3588 74.75 0.00796208 0.0108214 2023-10-09T09:19:06.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:06.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:06.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:06.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:06.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:06.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:06.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:06.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:06.922 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:06.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:06.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:07.025 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:07.056 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:07.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:07.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:07.164 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:07.193 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:07.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:07.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:07.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:07.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:07.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:07.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:07.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:07.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:07.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:07.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:07.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:07 smithi094 bash[59677]: cluster 2023-10-09T09:19:06.197882+0000 mgr.y (mgr.25421) 1257 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:19:07.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:07.580 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:07 smithi016 bash[75179]: cluster 2023-10-09T09:19:06.197882+0000 mgr.y (mgr.25421) 1257 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:19:07.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:07 smithi016 bash[77892]: cluster 2023-10-09T09:19:06.197882+0000 mgr.y (mgr.25421) 1257 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:19:07.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:07.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:07.611 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 61785 61769 91.9081 73.4375 0.0131616 0.0108733 2023-10-09T09:19:07.710 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:07.738 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:07.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:07.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:07.869 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:07.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:07.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:07.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:08.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:08.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:08.133 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:08.337 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:08.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:08.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:08.614 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 62829 62813 91.2823 65.25 0.00429642 0.0109482 2023-10-09T09:19:08.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:08.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:08.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:08.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:08.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:08.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:08.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:08.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:08.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:09.000 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:09.025 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:09.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:09.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:09.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:09.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:09.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:09.412 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:09.416 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:19:09.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:09 smithi016 bash[75179]: cluster 2023-10-09T09:19:08.198479+0000 mgr.y (mgr.25421) 1258 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:19:09.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:09 smithi016 bash[77892]: cluster 2023-10-09T09:19:08.198479+0000 mgr.y (mgr.25421) 1258 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:19:09.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:09.538 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:09.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:09 smithi094 bash[59677]: cluster 2023-10-09T09:19:08.198479+0000 mgr.y (mgr.25421) 1258 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:19:09.614 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 63892 63876 90.7176 66.4375 0.0146588 0.0110176 2023-10-09T09:19:09.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:09.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:09.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:09.805 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:09.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:09.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:09.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:09.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:10.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:10.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:10.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:10.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:10.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:10.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:10.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:10.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:10.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:10.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:10.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:10.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:10.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:10.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:10.614 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 64587 64571 89.6668 43.4375 0.00349152 0.0110592 2023-10-09T09:19:10.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:10.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:10.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:10.732 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:10.758 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:10.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:10.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:10.883 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:10.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:10.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:10.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:10.994 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:10 smithi094 bash[59677]: cluster 2023-10-09T09:19:10.198998+0000 mgr.y (mgr.25421) 1259 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:19:11.006 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:11.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:11.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:11.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:10 smithi016 bash[75179]: cluster 2023-10-09T09:19:10.198998+0000 mgr.y (mgr.25421) 1259 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:19:11.118 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:10 smithi016 bash[77892]: cluster 2023-10-09T09:19:10.198998+0000 mgr.y (mgr.25421) 1259 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:19:11.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:11.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:11.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:11.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:11.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:11.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:11.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:11.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:11.614 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 65555 65539 89.0327 60.5 0.0191306 0.0112259 2023-10-09T09:19:11.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:11.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:11.775 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:11.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:11.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:11.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:11.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:11.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:11.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:12.041 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:12.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:12.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:12.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:12.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:12.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:12.298 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:12.329 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:12.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:12.434 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:12.464 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:12.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:12.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:12.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:12.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:12.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:12.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:12.614 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 66618 66602 88.5518 66.4375 0.00873362 0.011287 2023-10-09T09:19:12.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:12.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:12.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:12.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:12.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:12.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:12.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:12.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:12.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:12.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:13.101 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:13.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:13.225 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:13.253 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:13.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:13.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:13.479 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:13.489 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:13 smithi094 bash[59677]: cluster 2023-10-09T09:19:12.199832+0000 mgr.y (mgr.25421) 1260 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T09:19:13.503 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:13.607 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:13.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 67458 67442 87.8006 52.5 0.00426389 0.0113231 2023-10-09T09:19:13.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:13.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:13.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:13.748 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:13 smithi016 bash[75179]: cluster 2023-10-09T09:19:12.199832+0000 mgr.y (mgr.25421) 1260 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T09:19:13.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:13 smithi016 bash[77892]: cluster 2023-10-09T09:19:12.199832+0000 mgr.y (mgr.25421) 1260 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T09:19:13.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:13.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:13.871 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:13.900 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:13.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:14.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:14.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:14.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:14.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:14.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:14.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:14.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:14.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:14.417 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:14.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:14.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:14.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:14.574 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:14.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 15 68350 68335 87.1477 55.8125 0.00415263 0.0114691 2023-10-09T09:19:14.677 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:14.713 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:14.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:14.820 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:14.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:14.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:14.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:14.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:14.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:15.011 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:19:15.011 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:19:15.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:15.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:15.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:15.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:15.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:15.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:15.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:15.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:15.376 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:15 smithi094 bash[59677]: cluster 2023-10-09T09:19:14.200356+0000 mgr.y (mgr.25421) 1261 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:19:15.390 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:15.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:15.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:15.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:15 smithi016 bash[75179]: cluster 2023-10-09T09:19:14.200356+0000 mgr.y (mgr.25421) 1261 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:19:15.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:15 smithi016 bash[77892]: cluster 2023-10-09T09:19:14.200356+0000 mgr.y (mgr.25421) 1261 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:19:15.517 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:15.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:15.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:15.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 69188 69172 86.451 52.3125 0.0143758 0.0115607 2023-10-09T09:19:15.620 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:15.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:15.644 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:19:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:15.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:15.696 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:19:15.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T09:19:15.748 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:15.758 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:19:15.770 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:15.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:15.806 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:19:15.841 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:19:15.874 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:15.879 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:19:15.900 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:15.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:15.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:15.928 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:19:15.970 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:19:15.977 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.977 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.977 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.977 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.977 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.977 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.978 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.978 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:19:15.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:19:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:19:15] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.33.4" 2023-10-09T09:19:16.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:16.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:16.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:16.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:16.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:16.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:16.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[75179]: audit 2023-10-09T09:19:15.578004+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.16:0/471774907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:19:16.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[75179]: audit 2023-10-09T09:19:15.732439+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.16:0/2888820090' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:19:16.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[75179]: audit 2023-10-09T09:19:15.733704+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:19:16.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[75179]: audit 2023-10-09T09:19:15.799264+0000 mon.a (mon.0) 1683 : audit 1 from='client.? 172.21.15.16:0/990527133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:19:16.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:16.286 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:19:16.297 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:16.329 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:19:16.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:16.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:16.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:16.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:16.560 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[77892]: audit 2023-10-09T09:19:15.578004+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.16:0/471774907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:19:16.560 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[77892]: audit 2023-10-09T09:19:15.732439+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.16:0/2888820090' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:19:16.560 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[77892]: audit 2023-10-09T09:19:15.733704+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:19:16.560 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:16 smithi016 bash[77892]: audit 2023-10-09T09:19:15.799264+0000 mon.a (mon.0) 1683 : audit 1 from='client.? 172.21.15.16:0/990527133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:19:16.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:16.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:16.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 70104 70088 85.8783 57.25 0.0193432 0.0116381 2023-10-09T09:19:16.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:16.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:16.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:16 smithi094 bash[59677]: audit 2023-10-09T09:19:15.578004+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.16:0/471774907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:19:16.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:16 smithi094 bash[59677]: audit 2023-10-09T09:19:15.732439+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.16:0/2888820090' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:19:16.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:16 smithi094 bash[59677]: audit 2023-10-09T09:19:15.733704+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:19:16.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:16 smithi094 bash[59677]: audit 2023-10-09T09:19:15.799264+0000 mon.a (mon.0) 1683 : audit 1 from='client.? 172.21.15.16:0/990527133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:19:16.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:16.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:16.953 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:16.982 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:16.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:16.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:17.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:17.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:17.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:17.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:17.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:17.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:17.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:17.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:17.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:17.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[75179]: cluster 2023-10-09T09:19:16.201030+0000 mgr.y (mgr.25421) 1262 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T09:19:17.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[75179]: cluster 2023-10-09T09:19:16.261649+0000 mon.a (mon.0) 1684 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:19:17.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[75179]: cluster 2023-10-09T09:19:16.272080+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T09:19:17.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[77892]: cluster 2023-10-09T09:19:16.201030+0000 mgr.y (mgr.25421) 1262 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T09:19:17.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[77892]: cluster 2023-10-09T09:19:16.261649+0000 mon.a (mon.0) 1684 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:19:17.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[77892]: cluster 2023-10-09T09:19:16.272080+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T09:19:17.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[77892]: audit 2023-10-09T09:19:16.275319+0000 mon.a (mon.0) 1685 : audit 1 from='client.? 172.21.15.16:0/990527133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:19:17.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[77892]: cluster 2023-10-09T09:19:16.275548+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2023-10-09T09:19:17.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[77892]: cluster 2023-10-09T09:19:16.279705+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T09:19:17.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:17.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:17.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:17.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:17.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 71128 71112 85.4575 64 0.015503 0.011696 2023-10-09T09:19:17.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:17.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:17 smithi094 bash[59677]: cluster 2023-10-09T09:19:16.201030+0000 mgr.y (mgr.25421) 1262 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T09:19:17.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:17 smithi094 bash[59677]: cluster 2023-10-09T09:19:16.261649+0000 mon.a (mon.0) 1684 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:19:17.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:17 smithi094 bash[59677]: cluster 2023-10-09T09:19:16.272080+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T09:19:17.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:17 smithi094 bash[59677]: audit 2023-10-09T09:19:16.275319+0000 mon.a (mon.0) 1685 : audit 1 from='client.? 172.21.15.16:0/990527133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:19:17.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:17 smithi094 bash[59677]: cluster 2023-10-09T09:19:16.275548+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2023-10-09T09:19:17.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:17 smithi094 bash[59677]: cluster 2023-10-09T09:19:16.279705+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T09:19:17.672 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:17.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:17.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:17.776 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:17.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[75179]: audit 2023-10-09T09:19:16.275319+0000 mon.a (mon.0) 1685 : audit 1 from='client.? 172.21.15.16:0/990527133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:19:17.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[75179]: cluster 2023-10-09T09:19:16.275548+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2023-10-09T09:19:17.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:17 smithi016 bash[75179]: cluster 2023-10-09T09:19:16.279705+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T09:19:17.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:17.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:17.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:17.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:17.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:18.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:18.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:18.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:18.170 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:18.202 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:18.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:18.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:18.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:18.333 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:18.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:18.440 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:18.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:18.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:18.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:18.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:18.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:18.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:18.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 72134 72118 85.0313 62.875 0.00841205 0.0117545 2023-10-09T09:19:18.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:18.739 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:18.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:18.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:18.842 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:18.868 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:18.869 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T09:19:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:18.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:18.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:19.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:19.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:19.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:19.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:19.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:19.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:19.395 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:19 smithi016 bash[77892]: cluster 2023-10-09T09:19:18.201553+0000 mgr.y (mgr.25421) 1263 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T09:19:19.395 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:19 smithi016 bash[77892]: audit 2023-10-09T09:19:19.257909+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:19.395 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:19 smithi016 bash[75179]: cluster 2023-10-09T09:19:18.201553+0000 mgr.y (mgr.25421) 1263 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T09:19:19.396 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:19 smithi016 bash[75179]: audit 2023-10-09T09:19:19.257909+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:19.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:19.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:19.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:19.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 73167 73151 84.6521 64.5625 0.0131025 0.011807 2023-10-09T09:19:19.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:19.657 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:19 smithi094 bash[59677]: cluster 2023-10-09T09:19:18.201553+0000 mgr.y (mgr.25421) 1263 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T09:19:19.657 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:19 smithi094 bash[59677]: audit 2023-10-09T09:19:19.257909+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:19.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:19.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:19.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:19.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:19.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:19.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:19.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:20.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:20.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:20.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:20.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:20.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:20.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:20.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:20.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:20.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:20.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:20.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:20.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:20.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:20.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:20.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:20.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:20.615 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 74123 74107 84.1993 59.75 0.0134639 0.0118708 2023-10-09T09:19:20.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:20.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:20.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:20.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:20.764 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:20.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:20.869 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:20.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:20.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:20.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:20.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:21.007 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:20 smithi094 bash[59677]: cluster 2023-10-09T09:19:20.203405+0000 mgr.y (mgr.25421) 1264 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:19:21.033 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:21.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:21.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:21.164 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:21.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:21.203 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:19:21.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:20 smithi016 bash[75179]: cluster 2023-10-09T09:19:20.203405+0000 mgr.y (mgr.25421) 1264 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:19:21.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:20 smithi016 bash[77892]: cluster 2023-10-09T09:19:20.203405+0000 mgr.y (mgr.25421) 1264 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T09:19:21.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:21.292 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:21.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:21.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:21.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:21.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:21.530 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:21.565 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:21.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:21.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 75090 75074 83.7749 60.4375 0.0151321 0.0119306 2023-10-09T09:19:21.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:21.697 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:21.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:21.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:21.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:19:21.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:21.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:19:21.869 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:21.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:21.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:21.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:21.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:22.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:22.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:22.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:22.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:22.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:22.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:22.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:22.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:22.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:22.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:22.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 75979 75963 83.2799 55.5625 0.0083271 0.0120005 2023-10-09T09:19:22.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:22.678 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:22.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:22.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:22.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:22.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:22.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:22.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:22.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:23.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:23.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:23.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:23.190 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:23.212 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:23.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:23.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:23.328 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:23 smithi016 bash[77892]: cluster 2023-10-09T09:19:22.204141+0000 mgr.y (mgr.25421) 1265 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T09:19:23.329 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:23 smithi016 bash[75179]: cluster 2023-10-09T09:19:22.204141+0000 mgr.y (mgr.25421) 1265 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T09:19:23.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:23.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:23.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:23.462 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:23 smithi094 bash[59677]: cluster 2023-10-09T09:19:22.204141+0000 mgr.y (mgr.25421) 1265 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T09:19:23.481 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:23.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:23.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:23.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 76923 76907 82.8612 59 0.00983956 0.012061 2023-10-09T09:19:23.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:23.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:23.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:23.722 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:23.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:23.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:23.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:23.869 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:19:23.870 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:19:23.870 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 3, 4, 6, 5, 7, 1, 2] 2023-10-09T09:19:23.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T09:19:23.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:23.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:23.991 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:24.028 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:24.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:24.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:24.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:24.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:24.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:24.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:24.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:24.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:24.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:24.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:24.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:24.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:24.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:24.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:24.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:24.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 77823 77807 82.41 56.25 0.00862182 0.0121283 2023-10-09T09:19:24.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:24.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:24.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:24.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:24.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:24.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:24.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:25.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:25.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:25.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:25.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:25.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:25.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:25.382 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:25.405 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:25.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:25.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:25 smithi094 bash[59677]: cluster 2023-10-09T09:19:24.204678+0000 mgr.y (mgr.25421) 1266 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T09:19:25.509 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:25.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:25.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:25.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:19:25.615819+0000 min lat: 0.00130401 max lat: 0.391267 avg lat: 0.0121923 2023-10-09T09:19:25.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:19:25.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 78733 78717 81.9844 56.875 0.0154835 0.0121923 2023-10-09T09:19:25.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:25.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:25.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:25.739 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:19:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:19:25] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:25 smithi016 bash[75179]: cluster 2023-10-09T09:19:24.204678+0000 mgr.y (mgr.25421) 1266 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T09:19:25.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:25 smithi016 bash[77892]: cluster 2023-10-09T09:19:24.204678+0000 mgr.y (mgr.25421) 1266 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T09:19:25.774 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:25.799 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:25.903 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:25.929 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:25.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:26.032 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:26.060 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:26.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:26.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:26.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:26.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:26.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:26.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:26.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:26.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:26.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:26.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 79646 79630 81.5758 57.0625 0.0188525 0.012253 2023-10-09T09:19:26.671 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:26.680 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:26.699 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:26.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:26.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:26.826 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:26.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:26.878 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:19:26.879 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:19:26.879 INFO:teuthology.orchestra.run.smithi094.stdout: "complaint_time": 30, 2023-10-09T09:19:26.879 INFO:teuthology.orchestra.run.smithi094.stdout: "num_blocked_ops": 0 2023-10-09T09:19:26.879 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:19:26.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:26.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:26.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:26.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:27.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:27.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:27.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:27.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:27.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:27.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:27.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:27.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:27.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:27.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:27.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:27.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:27 smithi094 bash[59677]: cluster 2023-10-09T09:19:26.205382+0000 mgr.y (mgr.25421) 1267 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-09T09:19:27.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:27.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:27 smithi016 bash[75179]: cluster 2023-10-09T09:19:26.205382+0000 mgr.y (mgr.25421) 1267 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-09T09:19:27.519 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:27 smithi016 bash[77892]: cluster 2023-10-09T09:19:26.205382+0000 mgr.y (mgr.25421) 1267 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-09T09:19:27.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:27.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:27.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 80529 80513 81.1501 55.1875 0.0194925 0.0123166 2023-10-09T09:19:27.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:27.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:27.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:27.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:27.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:27.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:27.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:27.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:27.892 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:27.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:27.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:27.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:28.023 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:28.046 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:28.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:28.116 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:19:28.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:28.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:28.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:28.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:28.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:28.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:28.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:28.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:28.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:28.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:28.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 81238 81222 80.5653 44.3125 0.0130086 0.0124066 2023-10-09T09:19:28.683 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:28.704 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:28.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:28.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:28.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:28.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:28.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:28.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:28.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:29.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:29.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:29.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:29.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:29.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:29.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:29.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:29.361 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:29.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:29.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:29.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:29.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:29 smithi016 bash[75179]: cluster 2023-10-09T09:19:28.205920+0000 mgr.y (mgr.25421) 1268 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-10-09T09:19:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:29 smithi016 bash[77892]: cluster 2023-10-09T09:19:28.205920+0000 mgr.y (mgr.25421) 1268 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-10-09T09:19:29.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:29.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:29.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:29.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 82159 82143 80.2058 57.5625 0.016088 0.0124619 2023-10-09T09:19:29.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:29.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:29.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:29.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:29 smithi094 bash[59677]: cluster 2023-10-09T09:19:28.205920+0000 mgr.y (mgr.25421) 1268 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-10-09T09:19:29.767 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:29.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:29.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:29.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:29.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:29.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:29.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:30.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:30.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:30.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:30.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:30.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:30.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:30.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:30.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:30.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:30.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:30.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:30.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:30.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:30.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:30.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:30.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 83091 83075 79.868 58.25 0.0263452 0.0125155 2023-10-09T09:19:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:30.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:30.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:30.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:30.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:30.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:30.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:30.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:30.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:30.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:31.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:31.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:30 smithi016 bash[77892]: cluster 2023-10-09T09:19:30.208980+0000 mgr.y (mgr.25421) 1269 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:19:31.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:30 smithi016 bash[75179]: cluster 2023-10-09T09:19:30.208980+0000 mgr.y (mgr.25421) 1269 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:19:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:31.132 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:31.140 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:30 smithi094 bash[59677]: cluster 2023-10-09T09:19:30.208980+0000 mgr.y (mgr.25421) 1269 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:19:31.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:31.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:31.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:31.298 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:31.402 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:31.426 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:31.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:31.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:31.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:31.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 84023 84007 79.5404 58.25 0.00675053 0.0125665 2023-10-09T09:19:31.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:31.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:31.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:31.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:31.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:31.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:31.956 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:31.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:32.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:32.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:32.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:32.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:32.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:32.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:32.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:32.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:32.360 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:32.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:32.463 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:32.498 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:32.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:32.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:32.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:32.616 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 84976 84960 79.2421 59.5625 0.0154617 0.0126145 2023-10-09T09:19:32.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:32.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:32.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:32.730 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:32.753 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:32.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:32.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:32.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:32.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:32.888 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:19:32.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:32.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:32.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:33.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:33.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:33.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:33.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:33.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:33.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:33.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:33.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:33 smithi016 bash[75179]: cluster 2023-10-09T09:19:32.209695+0000 mgr.y (mgr.25421) 1270 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:19:33.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:33 smithi016 bash[77892]: cluster 2023-10-09T09:19:32.209695+0000 mgr.y (mgr.25421) 1270 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:19:33.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:33.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:33.545 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:33.557 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:33 smithi094 bash[59677]: cluster 2023-10-09T09:19:32.209695+0000 mgr.y (mgr.25421) 1270 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T09:19:33.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:33.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:33.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 85877 85861 78.9049 56.3125 0.0191102 0.0126678 2023-10-09T09:19:33.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:33.698 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:33.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:33.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:33.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:33.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:33.970 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:33.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:34.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:34.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:34.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:34.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:34.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:34.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:34.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:34.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:34.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:34 smithi016 bash[75179]: audit 2023-10-09T09:19:34.257412+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:34.352 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:34 smithi016 bash[77892]: audit 2023-10-09T09:19:34.257412+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:34.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:34.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:34.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:34.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:34 smithi094 bash[59677]: audit 2023-10-09T09:19:34.257412+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:34.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:34.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:34.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:34.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:34.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 86748 86732 78.5502 54.4375 0.00452813 0.0127243 2023-10-09T09:19:34.630 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:34.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:34.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:34.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:34.761 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:34.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:34.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:34.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:34.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:34.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:34.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:35.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:35.126 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:35.150 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:35.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:35.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:35.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:35.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:35.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:35.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:35.463 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:35 smithi094 bash[59677]: cluster 2023-10-09T09:19:34.210198+0000 mgr.y (mgr.25421) 1271 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2023-10-09T09:19:35.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:35.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:35.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:35.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 87643 87627 78.2271 55.9375 0.013322 0.0127781 2023-10-09T09:19:35.653 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:35.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:35.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:35.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:35 smithi016 bash[75179]: cluster 2023-10-09T09:19:34.210198+0000 mgr.y (mgr.25421) 1271 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2023-10-09T09:19:35.756 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:35 smithi016 bash[77892]: cluster 2023-10-09T09:19:34.210198+0000 mgr.y (mgr.25421) 1271 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2023-10-09T09:19:35.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:35.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:35.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:35.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:35.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:35.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:36.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:36.066 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:19:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:19:35] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T09:19:36.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:36.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:36.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:36.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:36.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:36.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:36.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:36.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:36.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:36.594 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:36.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 88563 88547 77.9351 57.5 0.0291077 0.0128254 2023-10-09T09:19:36.626 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:36.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:36.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:36.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:36.761 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:36.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:36.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:36.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:36.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:36.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:37.004 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:37.032 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:37.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:37.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:37.286 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:37.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:37.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:37.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:37.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:37.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:37.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:37.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:37 smithi094 bash[59677]: cluster 2023-10-09T09:19:36.212385+0000 mgr.y (mgr.25421) 1272 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T09:19:37.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:37.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:37.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 89470 89454 77.64 56.6875 0.0167106 0.0128742 2023-10-09T09:19:37.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:37 smithi016 bash[75179]: cluster 2023-10-09T09:19:36.212385+0000 mgr.y (mgr.25421) 1272 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T09:19:37.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:37 smithi016 bash[77892]: cluster 2023-10-09T09:19:36.212385+0000 mgr.y (mgr.25421) 1272 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T09:19:37.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:37.717 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:37.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:37.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:37.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:37.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:37.992 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:37.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:38.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:38.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:38.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:38.235 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:38.268 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:38.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:38.286 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:19:38.286 INFO:teuthology.orchestra.run.smithi094.stdout: "size": 20, 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 600, 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [ 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.45290.0:167510 9.13 9.93121913 (undecoded) ondisk+write+known_if_redirected e238)", 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:10:34.562501+0000", 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 543.71967344300003, 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.11602164, 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "reached pg", 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.287 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.45290", 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/2881052605", 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 167510 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562501+0000", 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.288 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562501+0000", 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562499+0000", 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.3100000000000006e-06 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562508+0000", 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0609e-05 2023-10-09T09:19:38.289 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562519+0000", 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4989999999999994e-06 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562527+0000", 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9993e-05 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.290 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562587+0000", 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.8379999999999997e-06 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:34.562597+0000", 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1159125089999999 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.291 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.678509+0000", 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3142000000000001e-05 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:10:35.678523+0000", 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.11602164 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.292 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162039 10.9 10:9b7e05ee:::benchmark_data_smithi016_193276_object54816:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408488+0000", 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.87368640400001, 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.478613953, 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162039 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.293 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.294 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.294 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.294 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408488+0000", 2023-10-09T09:19:38.294 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.294 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.294 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.294 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.296 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408488+0000", 2023-10-09T09:19:38.296 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999971 2023-10-09T09:19:38.296 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.296 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.296 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.296 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408485+0000", 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2616000000000001e-05 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408498+0000", 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7674000000000001e-05 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.297 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408515+0000", 2023-10-09T09:19:38.298 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0917999999999999e-05 2023-10-09T09:19:38.298 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.298 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.298 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.298 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408526+0000", 2023-10-09T09:19:38.298 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.2999999999999997e-05 2023-10-09T09:19:38.298 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408619+0000", 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.8566000000000001e-05 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.299 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408638+0000", 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2724654879999999 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.681103+0000", 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000467274 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.681571+0000", 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.088937732000000005 2023-10-09T09:19:38.300 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.770508+0000", 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0038938110000000001 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.774402+0000", 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00022509099999999999 2023-10-09T09:19:38.301 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.774627+0000", 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024453556000000001 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.799081+0000", 2023-10-09T09:19:38.302 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.084655864999999997 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.883737+0000", 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.1014999999999999e-05 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.883778+0000", 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010028599999999999 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.303 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.883878+0000", 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000142755 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.884021+0000", 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003081121 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.304 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.887102+0000", 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.478613953 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162044 10.9 10:9fd3c7e2:::benchmark_data_smithi016_193276_object54821:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408656+0000", 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.873518808, 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4814079580000001, 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.305 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162044 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408656+0000", 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.306 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408656+0000", 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408654+0000", 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8460000000000002e-06 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.307 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408661+0000", 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1215e-05 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408672+0000", 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8939999999999993e-06 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.308 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408680+0000", 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000366471 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.409047+0000", 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5270000000000001e-05 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.309 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.409062+0000", 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.272544023 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.681606+0000", 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00048089999999999998 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.310 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.682087+0000", 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.10246718 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.784554+0000", 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6771000000000001e-05 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.311 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.784631+0000", 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012778 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.784759+0000", 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.014334308 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.312 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.799093+0000", 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.088096002000000007 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.887189+0000", 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0309999999999996e-05 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.313 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.887259+0000", 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.6646999999999998e-05 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.887356+0000", 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8085999999999998e-05 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.887424+0000", 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0026395479999999998 2023-10-09T09:19:38.314 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.890063+0000", 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4814079580000001 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162103 10.9 10:9f77513b:::benchmark_data_smithi016_193276_object54880:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.315 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408695+0000", 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.87347933300001, 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4838368769999999, 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162103 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.316 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408695+0000", 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408695+0000", 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408694+0000", 2023-10-09T09:19:38.317 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.0819999999999998e-06 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408699+0000", 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8349999999999993e-06 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408708+0000", 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.6320000000000002e-06 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.318 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408715+0000", 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00078582999999999995 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.409501+0000", 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3295999999999999e-05 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.409514+0000", 2023-10-09T09:19:38.319 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2726128299999999 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.682127+0000", 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00043112899999999998 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.682558+0000", 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.112037873 2023-10-09T09:19:38.320 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.321 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.321 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.321 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.794596+0000", 2023-10-09T09:19:38.321 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.9697000000000005e-05 2023-10-09T09:19:38.321 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.321 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.321 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.794685+0000", 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013751599999999999 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.794823+0000", 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.033170962999999998 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.322 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.827994+0000", 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.059319487999999997 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.887313+0000", 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0027905289999999999 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.323 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.890104+0000", 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.3305e-05 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.890147+0000", 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.5995999999999994e-05 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.324 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.890243+0000", 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002288712 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892532+0000", 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4838368769999999 2023-10-09T09:19:38.325 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162109 10.9 10:9881d4dc:::benchmark_data_smithi016_193276_object54886:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408727+0000", 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.873447642, 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4867992809999999, 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.326 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162109 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408727+0000", 2023-10-09T09:19:38.327 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408727+0000", 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408726+0000", 2023-10-09T09:19:38.328 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4830000000000001e-06 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408730+0000", 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.1980000000000007e-06 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408740+0000", 2023-10-09T09:19:38.329 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0889999999999996e-06 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408746+0000", 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.001065814 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.330 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.409811+0000", 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4105e-05 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.409826+0000", 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.272767057 2023-10-09T09:19:38.331 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.682593+0000", 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00041518399999999997 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.332 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.683008+0000", 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.124478213 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.807486+0000", 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012001400000000001 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.333 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.807606+0000", 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000227755 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.807834+0000", 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.020171946999999999 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.828006+0000", 2023-10-09T09:19:38.334 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.062120841000000003 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.890127+0000", 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0024641480000000002 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892591+0000", 2023-10-09T09:19:38.335 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019752499999999999 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892788+0000", 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013366100000000001 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.336 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892922+0000", 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0026040339999999999 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.895526+0000", 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4867992809999999 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.337 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162121 10.9 10:93ca2354:::benchmark_data_smithi016_193276_object54898:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408765+0000", 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.87340885, 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4893015140000001, 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162121 2023-10-09T09:19:38.338 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408765+0000", 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408765+0000", 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.339 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408765+0000", 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.1820000000000002e-06 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408768+0000", 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3619999999999997e-06 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.340 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408773+0000", 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.014e-06 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408778+0000", 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0016542670000000001 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.410432+0000", 2023-10-09T09:19:38.341 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2687000000000001e-05 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.410444+0000", 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2726274850000001 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.683072+0000", 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00051808799999999999 2023-10-09T09:19:38.342 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.683590+0000", 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.133826097 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.817416+0000", 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012485799999999999 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.343 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.817541+0000", 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000178569 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.817720+0000", 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024691838000000001 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.344 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.842411+0000", 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.050221037000000003 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892632+0000", 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6115999999999998e-05 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892669+0000", 2023-10-09T09:19:38.345 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00291493 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.895583+0000", 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0840999999999995e-05 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.895654+0000", 2023-10-09T09:19:38.346 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0024126709999999999 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.898067+0000", 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4893015140000001 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162141 10.9 10:9f2bb4be:::benchmark_data_smithi016_193276_object54918:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.347 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408783+0000", 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.87339083699999, 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.495912578, 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162141 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.348 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408783+0000", 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408783+0000", 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999995 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408783+0000", 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.683e-06 2023-10-09T09:19:38.349 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408786+0000", 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.2169999999999996e-06 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408790+0000", 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.146e-06 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.350 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408793+0000", 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0021080029999999998 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.410901+0000", 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4854999999999999e-05 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.410916+0000", 2023-10-09T09:19:38.351 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2727889079999999 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.683705+0000", 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00044193100000000001 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.684147+0000", 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.143902848 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.352 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.828050+0000", 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.5662999999999997e-05 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.828125+0000", 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000104551 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.353 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.828230+0000", 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024181551999999999 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.852411+0000", 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.040298089000000002 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892710+0000", 2023-10-09T09:19:38.354 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.8825000000000002e-05 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.892748+0000", 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0092431100000000006 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.901992+0000", 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.1631999999999995e-05 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.355 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.902053+0000", 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0026429370000000002 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.904696+0000", 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.495912578 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.356 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.357 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.357 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162142 10.9 10:9005006f:::benchmark_data_smithi016_193276_object54919:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.357 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408832+0000", 2023-10-09T09:19:38.357 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.87334205100001, 2023-10-09T09:19:38.357 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4984686950000001, 2023-10-09T09:19:38.357 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.357 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162142 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408832+0000", 2023-10-09T09:19:38.358 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408832+0000", 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.359 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408831+0000", 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4949999999999996e-06 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408838+0000", 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2858e-05 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.360 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408851+0000", 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8180000000000004e-06 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408858+0000", 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002175415 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.361 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.411034+0000", 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3659999999999997e-06 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.411040+0000", 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2731448249999999 2023-10-09T09:19:38.362 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.684185+0000", 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00039227000000000001 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.363 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.684577+0000", 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15243431499999999 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.837012+0000", 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00012203500000000001 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.837134+0000", 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00020147599999999999 2023-10-09T09:19:38.364 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.837335+0000", 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.031033427999999998 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.868369+0000", 2023-10-09T09:19:38.365 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.029750875999999999 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.898119+0000", 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.5790000000000001e-05 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.366 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.898155+0000", 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0065880340000000004 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.904743+0000", 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.4084999999999994e-05 2023-10-09T09:19:38.367 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.368 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.368 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.368 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.904807+0000", 2023-10-09T09:19:38.368 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002493589 2023-10-09T09:19:38.368 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.368 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.368 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.907301+0000", 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4984686950000001 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162157 10.9 10:9a83bb11:::benchmark_data_smithi016_193276_object54934:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.369 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408873+0000", 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.87330174799999, 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5011095839999999, 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162157 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.370 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408873+0000", 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408873+0000", 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.371 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408872+0000", 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.0690000000000001e-06 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408877+0000", 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.4530000000000008e-06 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.372 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408886+0000", 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6779999999999997e-06 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408892+0000", 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003067574 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.411959+0000", 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4005000000000001e-05 2023-10-09T09:19:38.373 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.411974+0000", 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.272640752 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.684614+0000", 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00041048499999999998 2023-10-09T09:19:38.374 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.685025+0000", 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.16287038300000001 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.847895+0000", 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0001177 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.375 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.848013+0000", 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00025749799999999999 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.848270+0000", 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.030125010000000001 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.878395+0000", 2023-10-09T09:19:38.376 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.019795212 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.898191+0000", 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3512e-05 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.898224+0000", 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0091258020000000006 2023-10-09T09:19:38.377 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.907350+0000", 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.8e-05 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.907408+0000", 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0025743060000000002 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.378 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.909982+0000", 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5011095839999999 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162173 10.9 10:9ac0acd1:::benchmark_data_smithi016_193276_object54950:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e271)", 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:23.408903+0000", 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 254.873271117, 2023-10-09T09:19:38.379 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5038397649999999, 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162173 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408903+0000", 2023-10-09T09:19:38.380 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408903+0000", 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408902+0000", 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.4569999999999998e-06 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.381 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408907+0000", 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.1810000000000007e-06 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408916+0000", 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.9730000000000002e-06 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.382 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.408922+0000", 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0033760000000000001 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.412298+0000", 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.3271000000000001e-05 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.383 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:23.412311+0000", 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2727519140000001 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.685063+0000", 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00046688300000000002 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.685530+0000", 2023-10-09T09:19:38.384 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.171623158 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.857153+0000", 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.1710000000000004e-05 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.857215+0000", 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000109448 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.385 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.857324+0000", 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.021084333 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.878409+0000", 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.023516031 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.386 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.901925+0000", 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.4484999999999999e-05 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.901959+0000", 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0081814269999999998 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.910141+0000", 2023-10-09T09:19:38.387 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6575e-05 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.910197+0000", 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0025456580000000001 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.912743+0000", 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.5038397649999999 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.388 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162175 10.9 10:97d3e194:::benchmark_data_smithi016_193276_object54952:head [delete] snapc 0=[] ondisk+write+known_if_redirected e272)", 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:24.607575+0000", 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 253.67459912499999, 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.30973709300000002, 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.389 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.390 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162175 2023-10-09T09:19:38.390 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.390 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.390 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.390 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.390 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607575+0000", 2023-10-09T09:19:38.390 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607575+0000", 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607573+0000", 2023-10-09T09:19:38.391 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1942e-05 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607585+0000", 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.3975e-05 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.392 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607609+0000", 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0149000000000001e-05 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607619+0000", 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.026044147 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.393 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.633663+0000", 2023-10-09T09:19:38.394 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1419e-05 2023-10-09T09:19:38.394 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.394 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.394 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.394 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.633674+0000", 2023-10-09T09:19:38.394 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.051890709 2023-10-09T09:19:38.394 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.685565+0000", 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00049901500000000001 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.686064+0000", 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.182347439 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.395 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.868411+0000", 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00013992099999999999 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.868551+0000", 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00019843000000000001 2023-10-09T09:19:38.396 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.868750+0000", 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.014904015 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.883654+0000", 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.026371103 2023-10-09T09:19:38.397 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.910025+0000", 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.0227999999999999e-05 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.910055+0000", 2023-10-09T09:19:38.398 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0046429940000000001 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.914698+0000", 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8973000000000005e-05 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.399 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.914767+0000", 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0025452360000000002 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.917312+0000", 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.30973709300000002 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.400 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.401 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.401 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.401 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.401 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162177 10.9 10:931564da:::benchmark_data_smithi016_193276_object54954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e272)", 2023-10-09T09:19:38.401 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:24.607798+0000", 2023-10-09T09:19:38.401 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 253.67437613499999, 2023-10-09T09:19:38.401 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.31660349300000001, 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162177 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.402 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607798+0000", 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607798+0000", 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.403 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607797+0000", 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7830000000000004e-06 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607806+0000", 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4837999999999999e-05 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607820+0000", 2023-10-09T09:19:38.404 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6760000000000007e-06 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.607828+0000", 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.025871966 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.633700+0000", 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.8259999999999999e-06 2023-10-09T09:19:38.405 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for peered", 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.633709+0000", 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.052387847000000001 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.686097+0000", 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000457792 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.406 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.686555+0000", 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.19189999499999999 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.878455+0000", 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.4037999999999996e-05 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.878549+0000", 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016031500000000001 2023-10-09T09:19:38.407 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.878709+0000", 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0084110790000000001 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.887120+0000", 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.022964004 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.408 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.910084+0000", 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.8626999999999999e-05 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.910113+0000", 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0090753150000000005 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.919188+0000", 2023-10-09T09:19:38.409 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.3983e-05 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.919252+0000", 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.005149753 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.924402+0000", 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.31660349300000001 2023-10-09T09:19:38.410 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162190 10.9 10:99a8de58:::benchmark_data_smithi016_193276_object54967:head [delete] snapc 0=[] ondisk+write+known_if_redirected e272)", 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:24.663237+0000", 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 253.61893703699999, 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.36294790100000002, 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.411 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162190 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.663237+0000", 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.412 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.663237+0000", 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.663235+0000", 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.7020000000000007e-06 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.663244+0000", 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.349e-05 2023-10-09T09:19:38.413 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.663258+0000", 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8800000000000002e-06 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.663264+0000", 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.26121204999999997 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.414 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.924476+0000", 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00049463100000000004 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.924971+0000", 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.039153282999999997 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.415 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.964124+0000", 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.9545999999999999e-05 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.964224+0000", 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00017373599999999999 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.964398+0000", 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.024964397999999999 2023-10-09T09:19:38.416 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.989362+0000", 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.033711418999999999 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023073+0000", 2023-10-09T09:19:38.417 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.9073999999999999e-05 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023113+0000", 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00029436600000000001 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023407+0000", 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.0792999999999999e-05 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.418 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023468+0000", 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0027174870000000002 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.026185+0000", 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.36294790100000002 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.419 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162207 10.9 10:9cd5dc4b:::benchmark_data_smithi016_193276_object54984:head [delete] snapc 0=[] ondisk+write+known_if_redirected e272)", 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:24.747564+0000", 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 253.53461044299999, 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.28129569700000001, 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.420 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162207 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.747564+0000", 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.747564+0000", 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.421 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.747562+0000", 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1936000000000001e-05 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.747574+0000", 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4813e-05 2023-10-09T09:19:38.422 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.747589+0000", 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.0339999999999999e-06 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.747596+0000", 2023-10-09T09:19:38.423 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.17742765899999999 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.925024+0000", 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00027497300000000003 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.925299+0000", 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.049496793999999997 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.424 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.974796+0000", 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000100405 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.974896+0000", 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00015949700000000001 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.975055+0000", 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.014313782000000001 2023-10-09T09:19:38.425 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.989369+0000", 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.033778324999999998 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023148+0000", 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.3102000000000003e-05 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.426 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023181+0000", 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0030811829999999999 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.026262+0000", 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.601e-05 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.427 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.026328+0000", 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0025317120000000002 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.028860+0000", 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.28129569700000001 2023-10-09T09:19:38.428 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:162210 10.9 10:98f4a47e:::benchmark_data_smithi016_193276_object54987:head [delete] snapc 0=[] ondisk+write+known_if_redirected e272)", 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:24.773186+0000", 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 253.50898838399999, 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.26223212899999998, 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.429 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.430 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.430 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 162210 2023-10-09T09:19:38.430 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.430 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.430 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.430 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.430 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.773186+0000", 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.773186+0000", 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.773184+0000", 2023-10-09T09:19:38.431 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.0586e-05 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.773194+0000", 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.1189999999999999e-05 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.773215+0000", 2023-10-09T09:19:38.432 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.1970000000000002e-06 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.773225+0000", 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.15211406 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.433 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.925339+0000", 2023-10-09T09:19:38.434 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00039451200000000002 2023-10-09T09:19:38.434 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.434 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.434 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.434 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.925733+0000", 2023-10-09T09:19:38.434 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.059426193000000002 2023-10-09T09:19:38.434 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.985159+0000", 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010869700000000001 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.435 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.985268+0000", 2023-10-09T09:19:38.436 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000218729 2023-10-09T09:19:38.436 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.436 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.436 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.436 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:24.985487+0000", 2023-10-09T09:19:38.436 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.019207444000000001 2023-10-09T09:19:38.436 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.004694+0000", 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.018518140999999998 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.437 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023212+0000", 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 2.9958999999999999e-05 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.023242+0000", 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0091686100000000006 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.438 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.032411+0000", 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.8708000000000006e-05 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.032490+0000", 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.002928389 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:25.035418+0000", 2023-10-09T09:19:38.439 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.26223212899999998 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:170540 10.9 10:9250e130:::benchmark_data_smithi016_193276_object63317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e276)", 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:43.178850+0000", 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 235.10332438699999, 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.26448032199999999, 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.440 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 170540 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.178850+0000", 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.441 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.178850+0000", 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.999999 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.178849+0000", 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.651e-06 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.442 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.178854+0000", 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2451e-05 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.178866+0000", 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3940000000000004e-06 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.443 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.178872+0000", 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.079508905000000005 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.258381+0000", 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00040601 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for degraded object", 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.258787+0000", 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.069051926999999999 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.444 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.327838+0000", 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000119863 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.327958+0000", 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00021240399999999999 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T09:19:38.445 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.328171+0000", 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.014481348999999999 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.342652+0000", 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.9351000000000002e-05 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.446 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.342701+0000", 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0061724509999999998 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.348874+0000", 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.090927809999999998 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.439802+0000", 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 8.4549999999999995e-05 2023-10-09T09:19:38.447 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.439886+0000", 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.003444024 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:43.443330+0000", 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.26448032199999999 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.448 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:171291 10.c 10:32a36ee2:::benchmark_data_smithi016_193276_object64068:head [delete] snapc 0=[] ondisk+write+known_if_redirected e276)", 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:44.847134+0000", 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 233.435040501, 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.26432096700000002, 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.449 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 171291 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847134+0000", 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847134+0000", 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999971 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.450 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847131+0000", 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.046e-05 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847141+0000", 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.7980000000000001e-05 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.451 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847159+0000", 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.4059999999999999e-06 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847167+0000", 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00035908899999999999 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.452 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847526+0000", 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00020433300000000001 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847730+0000", 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000149056 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.847879+0000", 2023-10-09T09:19:38.453 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0048764919999999996 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852756+0000", 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.020878035 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.873634+0000", 2023-10-09T09:19:38.454 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.23772815899999999 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:45.111362+0000", 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2821999999999997e-05 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:45.111415+0000", 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4.0123999999999997e-05 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.455 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.457 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:45.111455+0000", 2023-10-09T09:19:38.457 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.26432096700000002 2023-10-09T09:19:38.457 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.457 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.457 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.457 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:171294 10.c 10:3d18d6ee:::benchmark_data_smithi016_193276_object64071:head [delete] snapc 0=[] ondisk+write+known_if_redirected e276)", 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:44.852849+0000", 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 233.42932574299999, 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.25876453199999999, 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.458 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 171294 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852849+0000", 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.459 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852849+0000", 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999986 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852847+0000", 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6410000000000002e-06 2023-10-09T09:19:38.460 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852855+0000", 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.2018000000000001e-05 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852867+0000", 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.9440000000000005e-06 2023-10-09T09:19:38.461 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.462 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.462 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.462 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852875+0000", 2023-10-09T09:19:38.462 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00010346899999999999 2023-10-09T09:19:38.462 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.462 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.462 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.852978+0000", 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000210232 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.853188+0000", 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00035575599999999999 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:19:38.463 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.853544+0000", 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0050911250000000002 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.858635+0000", 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.015316697000000001 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.464 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:44.873952+0000", 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.23754797799999999 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:45.111500+0000", 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.7009999999999996e-05 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.465 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:45.111577+0000", 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 3.6207000000000001e-05 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:45.111613+0000", 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.25876453199999999 2023-10-09T09:19:38.466 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35619.0:174619 10.c 10:37a5f75e:::benchmark_data_smithi016_193276_object67396:head [delete] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:15:52.562546+0000", 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 225.719627952, 2023-10-09T09:19:38.467 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.27450838700000002, 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35619", 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 174619 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.468 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562546+0000", 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562546+0000", 2023-10-09T09:19:38.469 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999976 2023-10-09T09:19:38.470 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.470 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.470 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.470 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562544+0000", 2023-10-09T09:19:38.470 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 9.0750000000000004e-06 2023-10-09T09:19:38.471 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.471 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.471 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.471 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562553+0000", 2023-10-09T09:19:38.471 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.4387e-05 2023-10-09T09:19:38.471 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.471 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562568+0000", 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 7.6170000000000001e-06 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562575+0000", 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.2614999999999996e-05 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.472 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562638+0000", 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014154200000000001 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562779+0000", 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000180407 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:19:38.473 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.562960+0000", 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.013597303999999999 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.576557+0000", 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0040306580000000003 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.580588+0000", 2023-10-09T09:19:38.474 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.25630201699999999 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.836890+0000", 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000107323 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.836997+0000", 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.7627000000000002e-05 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.475 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:15:52.837055+0000", 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.27450838700000002 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: "description": "osd_op(client.35859.0:81030 11.4 11:2ec13296:::benchmark_data_smithi016_218422_object81029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: "initiated_at": "2023-10-09T09:19:28.116211+0000", 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: "age": 10.165963106, 2023-10-09T09:19:38.476 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.34591163000000003, 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "type_data": { 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "client_info": { 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "client": "client.35859", 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "tid": 81030 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "events": [ 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "initiated", 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116211+0000", 2023-10-09T09:19:38.477 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "throttled", 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116211+0000", 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 4294967295.9999981 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "header_read", 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116209+0000", 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.4190999999999997e-05 2023-10-09T09:19:38.478 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "all_read", 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116263+0000", 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 1.1776e-05 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "dispatched", 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116275+0000", 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 6.8730000000000001e-06 2023-10-09T09:19:38.479 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "queued_for_pg", 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116282+0000", 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.6240999999999999e-05 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "reached_pg", 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116338+0000", 2023-10-09T09:19:38.480 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00014788400000000001 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "started", 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116486+0000", 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.00016145600000000001 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.116648+0000", 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.0095346649999999995 2023-10-09T09:19:38.481 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "op_commit", 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.126182+0000", 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.000488142 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.126670+0000", 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.33534635200000001 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.482 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.462017+0000", 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.2268000000000002e-05 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "commit_sent", 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.462069+0000", 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 5.3786999999999998e-05 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: }, 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: { 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "event": "done", 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "time": "2023-10-09T09:19:28.462123+0000", 2023-10-09T09:19:38.483 INFO:teuthology.orchestra.run.smithi094.stdout: "duration": 0.34591163000000003 2023-10-09T09:19:38.484 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.484 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.484 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.484 INFO:teuthology.orchestra.run.smithi094.stdout: } 2023-10-09T09:19:38.484 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2023-10-09T09:19:38.484 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:19:38.484 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:38.509 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:38.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:38.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:38.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 90362 90346 77.3401 55.75 0.0197423 0.0129232 2023-10-09T09:19:38.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:38.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:38.759 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:38.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:38.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:38.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:38.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:38.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:38.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:39.023 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:39.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:39.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:39.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:39.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:39.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:39.296 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:39 smithi094 bash[59677]: cluster 2023-10-09T09:19:38.213032+0000 mgr.y (mgr.25421) 1273 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T09:19:39.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:39.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:39.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:39.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:19:39.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:39.452 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:39 smithi016 bash[75179]: cluster 2023-10-09T09:19:38.213032+0000 mgr.y (mgr.25421) 1273 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T09:19:39.452 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:39 smithi016 bash[77892]: cluster 2023-10-09T09:19:38.213032+0000 mgr.y (mgr.25421) 1273 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T09:19:39.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:39.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:39.579 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:39.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:39.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:39.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 91177 91161 76.9832 50.9375 0.0345233 0.0129838 2023-10-09T09:19:39.710 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:39.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:39.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:39.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:39.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:39.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:39.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:39.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:39.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:40.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:40.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:40.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:40.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:40.246 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:40.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:40.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:40.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:40.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:40.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:40.544 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:40.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:40.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 92041 92025 76.6767 54 0.016072 0.013036 2023-10-09T09:19:40.650 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:40.675 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:40.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:40.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:40.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:40 smithi094 bash[59677]: cluster 2023-10-09T09:19:40.213542+0000 mgr.y (mgr.25421) 1274 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-10-09T09:19:40.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:40.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:40.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:41.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:40 smithi016 bash[75179]: cluster 2023-10-09T09:19:40.213542+0000 mgr.y (mgr.25421) 1274 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-10-09T09:19:41.069 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:40 smithi016 bash[77892]: cluster 2023-10-09T09:19:40.213542+0000 mgr.y (mgr.25421) 1274 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-10-09T09:19:41.071 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:41.102 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:41.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:41.207 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:41.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:41.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:41.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:41.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:41.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:41.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:41.488 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:19:41.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:41.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:41.560 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:19:41.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:41.617 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 92730 92714 76.2344 43.0625 0.00852066 0.0131116 2023-10-09T09:19:41.621 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:19:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:41.645 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:41.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:41.669 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:19:41.710 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:19:41.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:41.762 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:19:41.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:41.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:41.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:41.796 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:19:41.842 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:19:41.846 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.2. 2023-10-09T09:19:41.851 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.851 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.851 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.851 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.851 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.851 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.851 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.852 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:19:41.853 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:19:41.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:41.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:41.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:41.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:41 smithi094 bash[59677]: audit 2023-10-09T09:19:41.441579+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.16:0/3065211296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:19:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:41 smithi094 bash[59677]: audit 2023-10-09T09:19:41.542392+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.16:0/2473616323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:19:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:41 smithi094 bash[59677]: audit 2023-10-09T09:19:41.543673+0000 mon.a (mon.0) 1689 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:19:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:41 smithi094 bash[59677]: audit 2023-10-09T09:19:41.743602+0000 mon.a (mon.0) 1690 : audit 1 from='client.? 172.21.15.16:0/93218436' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:19:41.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:41 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.743857+0000 mon.a (mon.0) 1691 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:19:41.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[75179]: debug 2023-10-09T09:19:41.736+0000 7fa046a4d700 -1 mon.a@0(leader).osd e302 definitely_dead 0 2023-10-09T09:19:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[75179]: audit 2023-10-09T09:19:41.441579+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.16:0/3065211296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:19:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[75179]: audit 2023-10-09T09:19:41.542392+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.16:0/2473616323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:19:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[75179]: audit 2023-10-09T09:19:41.543673+0000 mon.a (mon.0) 1689 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:19:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[75179]: audit 2023-10-09T09:19:41.743602+0000 mon.a (mon.0) 1690 : audit 1 from='client.? 172.21.15.16:0/93218436' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:19:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.743857+0000 mon.a (mon.0) 1691 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:19:42.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[77892]: audit 2023-10-09T09:19:41.441579+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.16:0/3065211296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:19:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[77892]: audit 2023-10-09T09:19:41.542392+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.16:0/2473616323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:19:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[77892]: audit 2023-10-09T09:19:41.543673+0000 mon.a (mon.0) 1689 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:19:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[77892]: audit 2023-10-09T09:19:41.743602+0000 mon.a (mon.0) 1690 : audit 1 from='client.? 172.21.15.16:0/93218436' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:19:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:41 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.743857+0000 mon.a (mon.0) 1691 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:19:42.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:42.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:42.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:42.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:42.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:42.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:42.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:42.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:42.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:42.419 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:42.454 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:42.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:42.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:42.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:42.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 92911 92895 75.3912 11.3125 0.00430808 0.0131284 2023-10-09T09:19:42.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:42.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:42.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:42.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:42.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:42.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:42.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:42.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:42.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:43.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:43.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:43.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:43.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.832483+0000 mon.a (mon.0) 1692 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:19:43.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.832518+0000 mon.a (mon.0) 1693 : cluster 1 Cluster is now healthy 2023-10-09T09:19:43.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: audit 2023-10-09T09:19:41.838756+0000 mon.a (mon.0) 1694 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:19:43.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: audit 2023-10-09T09:19:41.839038+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.16:0/93218436' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:19:43.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.839199+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T09:19:43.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.847213+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.848319+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.848921+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:41.849314+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.214212+0000 mgr.y (mgr.25421) 1275 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T09:19:43.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.832483+0000 mon.a (mon.0) 1692 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:19:43.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.832518+0000 mon.a (mon.0) 1693 : cluster 1 Cluster is now healthy 2023-10-09T09:19:43.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: audit 2023-10-09T09:19:41.838756+0000 mon.a (mon.0) 1694 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:19:43.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: audit 2023-10-09T09:19:41.839038+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.16:0/93218436' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:19:43.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.839199+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T09:19:43.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.847213+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.848319+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.848921+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:41.849314+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:42 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.214212+0000 mgr.y (mgr.25421) 1275 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T09:19:43.211 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:43.224 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.832483+0000 mon.a (mon.0) 1692 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:19:43.224 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.832518+0000 mon.a (mon.0) 1693 : cluster 1 Cluster is now healthy 2023-10-09T09:19:43.224 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: audit 2023-10-09T09:19:41.838756+0000 mon.a (mon.0) 1694 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:19:43.225 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: audit 2023-10-09T09:19:41.839038+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.16:0/93218436' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:19:43.225 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.839199+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T09:19:43.225 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.847213+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.225 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.848319+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.225 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.848921+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.225 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:41.849314+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T09:19:43.225 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:42 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.214212+0000 mgr.y (mgr.25421) 1275 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T09:19:43.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:43.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:43.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:43.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:43.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:43.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:43.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:43.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:43.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:43.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:43.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:43.607 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:43.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 92911 92895 74.4247 0 - 0.0131284 2023-10-09T09:19:43.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:43.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:43.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:43.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:43.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:43.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:43.878 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:43.887 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.856284+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.856284+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.856438+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.858016+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.858718+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.858822+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.860795+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.861501+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.861806+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.888 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.861844+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.889 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.862378+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.889 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:43 smithi094 bash[59677]: cluster 2023-10-09T09:19:42.865674+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:43.903 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:43.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:44.010 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:44.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:44.148 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.174 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:44.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.856284+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.856284+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.856438+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.858016+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.858718+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.858822+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.860795+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.861501+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.861806+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.861844+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.862378+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[75179]: cluster 2023-10-09T09:19:42.865674+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.856284+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.856284+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.856438+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.858016+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.858718+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.858822+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T09:19:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.860795+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.861501+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.861806+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.861844+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.862378+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:43 smithi016 bash[77892]: cluster 2023-10-09T09:19:42.865674+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T09:19:44.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:44.405 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:44.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:44.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:44.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:44.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:44.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 15 93104 93089 73.6361 6.0625 0.00940719 0.0134625 2023-10-09T09:19:44.671 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.695 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:44.798 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:44.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:44.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:44.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.935 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:44 smithi094 bash[59677]: cluster 2023-10-09T09:19:43.855927+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:44.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:44 smithi094 bash[59677]: cluster 2023-10-09T09:19:43.856878+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:44.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:44 smithi094 bash[59677]: cluster 2023-10-09T09:19:43.857518+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T09:19:44.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:44 smithi094 bash[59677]: cluster 2023-10-09T09:19:43.858586+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:44.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:44 smithi094 bash[59677]: cluster 2023-10-09T09:19:43.859396+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:44.936 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:44 smithi094 bash[59677]: cluster 2023-10-09T09:19:44.214760+0000 mgr.y (mgr.25421) 1276 : cluster 0 pgmap v1401: 89 pgs: 6 peering, 83 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T09:19:44.952 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:44.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:45.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:45.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:45.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:45.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:45.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:45.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:45.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[75179]: cluster 2023-10-09T09:19:43.855927+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[75179]: cluster 2023-10-09T09:19:43.856878+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[75179]: cluster 2023-10-09T09:19:43.857518+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T09:19:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[75179]: cluster 2023-10-09T09:19:43.858586+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[75179]: cluster 2023-10-09T09:19:43.859396+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[75179]: cluster 2023-10-09T09:19:44.214760+0000 mgr.y (mgr.25421) 1276 : cluster 0 pgmap v1401: 89 pgs: 6 peering, 83 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T09:19:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[77892]: cluster 2023-10-09T09:19:43.855927+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[77892]: cluster 2023-10-09T09:19:43.856878+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[77892]: cluster 2023-10-09T09:19:43.857518+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T09:19:45.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[77892]: cluster 2023-10-09T09:19:43.858586+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[77892]: cluster 2023-10-09T09:19:43.859396+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T09:19:45.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:44 smithi016 bash[77892]: cluster 2023-10-09T09:19:44.214760+0000 mgr.y (mgr.25421) 1276 : cluster 0 pgmap v1401: 89 pgs: 6 peering, 83 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T09:19:45.324 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:45.352 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:45.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:45.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:45.455 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:45.480 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:45.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:45.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:45.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:45.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:45.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-10-09T09:19:45.617739+0000 min lat: 0.00130401 max lat: 2.37467 avg lat: 0.0134609 2023-10-09T09:19:45.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:19:45.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 93135 93119 72.7392 1.875 0.00450364 0.0134609 2023-10-09T09:19:45.716 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:45.748 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:45.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:45.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:45.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:45.866 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:19:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:19:45] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T09:19:45.868 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:45 smithi016 bash[77892]: cluster 2023-10-09T09:19:44.862392+0000 mon.a (mon.0) 1699 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:19:45.878 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:45.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:45.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:45.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:45 smithi094 bash[59677]: cluster 2023-10-09T09:19:44.862392+0000 mon.a (mon.0) 1699 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:19:46.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:46.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:46.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:46.121 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:45 smithi016 bash[75179]: cluster 2023-10-09T09:19:44.862392+0000 mon.a (mon.0) 1699 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:19:46.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:46.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:46.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:46.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:46.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:19:46.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:46.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:19:46.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:46.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:46.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:46.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 93135 93119 71.8412 0 - 0.0134609 2023-10-09T09:19:46.656 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:46.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:46.787 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:46.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:46.870 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:46.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:46.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:46.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:47.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:47.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:47.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:47.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:46 smithi016 bash[75179]: cluster 2023-10-09T09:19:46.215540+0000 mgr.y (mgr.25421) 1277 : cluster 0 pgmap v1402: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 peering, 82 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 5843/278556 objects degraded (2.098%); 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:19:47.186 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:46 smithi016 bash[77892]: cluster 2023-10-09T09:19:46.215540+0000 mgr.y (mgr.25421) 1277 : cluster 0 pgmap v1402: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 peering, 82 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 5843/278556 objects degraded (2.098%); 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:19:47.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:47.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:47.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:46 smithi094 bash[59677]: cluster 2023-10-09T09:19:46.215540+0000 mgr.y (mgr.25421) 1277 : cluster 0 pgmap v1402: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 peering, 82 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 5843/278556 objects degraded (2.098%); 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T09:19:47.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:47.331 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:47.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:47.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:47.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:47.561 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:47.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:47.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:47.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 93135 93119 70.9651 0 - 0.0134609 2023-10-09T09:19:47.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:47.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:47.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:47.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:47.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:47.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:47.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:47.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:47.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:47.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:48.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:47 smithi016 bash[75179]: cluster 2023-10-09T09:19:46.864888+0000 mon.a (mon.0) 1700 : cluster 3 Health check failed: Degraded data redundancy: 5843/278556 objects degraded (2.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:19:48.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:47 smithi016 bash[77892]: cluster 2023-10-09T09:19:46.864888+0000 mon.a (mon.0) 1700 : cluster 3 Health check failed: Degraded data redundancy: 5843/278556 objects degraded (2.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:19:48.078 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:48.106 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:48.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:48.209 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:48.221 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:47 smithi094 bash[59677]: cluster 2023-10-09T09:19:46.864888+0000 mon.a (mon.0) 1700 : cluster 3 Health check failed: Degraded data redundancy: 5843/278556 objects degraded (2.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:19:48.235 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:48.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:48.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:48.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:48.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:48.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:48.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:48.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:48.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:48.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:48.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 93135 93119 70.1101 0 - 0.0134609 2023-10-09T09:19:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:48.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:48.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:48.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:48.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:48.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:48.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:48.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:48.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:48.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:48.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:48 smithi016 bash[75179]: cluster 2023-10-09T09:19:48.216365+0000 mgr.y (mgr.25421) 1278 : cluster 0 pgmap v1403: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 10 MiB/s wr, 164 op/s; 17456/278883 objects degraded (6.259%); 11604/278883 objects misplaced (4.161%); 2.3 MiB/s, 1 keys/s, 37 objects/s recovering 2023-10-09T09:19:49.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:48 smithi016 bash[77892]: cluster 2023-10-09T09:19:48.216365+0000 mgr.y (mgr.25421) 1278 : cluster 0 pgmap v1403: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 10 MiB/s wr, 164 op/s; 17456/278883 objects degraded (6.259%); 11604/278883 objects misplaced (4.161%); 2.3 MiB/s, 1 keys/s, 37 objects/s recovering 2023-10-09T09:19:49.007 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:49.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:49.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:49.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:49.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:49.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:49.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:48 smithi094 bash[59677]: cluster 2023-10-09T09:19:48.216365+0000 mgr.y (mgr.25421) 1278 : cluster 0 pgmap v1403: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 10 MiB/s wr, 164 op/s; 17456/278883 objects degraded (6.259%); 11604/278883 objects misplaced (4.161%); 2.3 MiB/s, 1 keys/s, 37 objects/s recovering 2023-10-09T09:19:49.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:49.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:49.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:49.374 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:19:49.374 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:19:49.374 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T09:19:49.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T09:19:49.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:49.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:49.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:49.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:49.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 93135 93119 69.2755 0 - 0.0134609 2023-10-09T09:19:49.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:49.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:49.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:49.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:49.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:49.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:49.895 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:49.929 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:49.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:49.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:49 smithi016 bash[75179]: cluster 2023-10-09T09:19:48.875896+0000 mon.a (mon.0) 1701 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T09:19:49.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:49 smithi016 bash[75179]: audit 2023-10-09T09:19:49.262152+0000 mon.a (mon.0) 1702 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:19:50.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:49 smithi016 bash[75179]: audit 2023-10-09T09:19:49.264572+0000 mon.a (mon.0) 1703 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:49 smithi016 bash[77892]: cluster 2023-10-09T09:19:48.875896+0000 mon.a (mon.0) 1701 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T09:19:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:49 smithi016 bash[77892]: audit 2023-10-09T09:19:49.262152+0000 mon.a (mon.0) 1702 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:19:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:49 smithi016 bash[77892]: audit 2023-10-09T09:19:49.264572+0000 mon.a (mon.0) 1703 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:50.032 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:50.045 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:49 smithi094 bash[59677]: cluster 2023-10-09T09:19:48.875896+0000 mon.a (mon.0) 1701 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T09:19:50.045 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:49 smithi094 bash[59677]: audit 2023-10-09T09:19:49.262152+0000 mon.a (mon.0) 1702 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:19:50.045 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:49 smithi094 bash[59677]: audit 2023-10-09T09:19:49.264572+0000 mon.a (mon.0) 1703 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:50.062 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:50.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:50.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:50.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:50.295 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:50.321 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:50.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:50.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:50.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:50.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:50.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 93135 93119 68.4605 0 - 0.0134609 2023-10-09T09:19:50.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:50.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:50.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:50.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:50.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:50.924 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:50.925 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:50.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:50.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:50.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:50.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:50 smithi016 bash[77892]: cluster 2023-10-09T09:19:50.216827+0000 mgr.y (mgr.25421) 1279 : cluster 0 pgmap v1404: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 8.2 MiB/s wr, 130 op/s; 17456/278883 objects degraded (6.259%); 11604/278883 objects misplaced (4.161%); 1.8 MiB/s, 1 keys/s, 29 objects/s recovering 2023-10-09T09:19:51.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:50 smithi016 bash[75179]: cluster 2023-10-09T09:19:50.216827+0000 mgr.y (mgr.25421) 1279 : cluster 0 pgmap v1404: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 8.2 MiB/s wr, 130 op/s; 17456/278883 objects degraded (6.259%); 11604/278883 objects misplaced (4.161%); 1.8 MiB/s, 1 keys/s, 29 objects/s recovering 2023-10-09T09:19:51.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:51.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:51.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:51.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:51.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:51.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:50 smithi094 bash[59677]: cluster 2023-10-09T09:19:50.216827+0000 mgr.y (mgr.25421) 1279 : cluster 0 pgmap v1404: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 8.2 MiB/s wr, 130 op/s; 17456/278883 objects degraded (6.259%); 11604/278883 objects misplaced (4.161%); 1.8 MiB/s, 1 keys/s, 29 objects/s recovering 2023-10-09T09:19:51.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:51.346 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:51.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:51.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:51.577 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:51.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:51.618 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 93135 93119 67.6645 0 - 0.0134609 2023-10-09T09:19:51.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:51.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:51.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:51.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:51.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:51.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:51.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:51.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:51.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:52.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:52.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:52.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:52.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:52.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:52.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:52.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:52.359 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:52.388 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:52.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:52.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:52.492 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:52.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:52.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:52.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:52.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 93135 93119 66.8868 0 - 0.0134609 2023-10-09T09:19:52.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:52.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:52.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:52.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:52.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:52.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:52.889 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:52.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:52.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:53.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:53.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:53.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:53.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:53.286 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:53.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:53.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:53.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:53.428 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:53 smithi016 bash[75179]: cluster 2023-10-09T09:19:52.217510+0000 mgr.y (mgr.25421) 1280 : cluster 0 pgmap v1405: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.9 MiB/s wr, 61 op/s; 17456/278883 objects degraded (6.259%); 11363/278883 objects misplaced (4.074%); 3.2 MiB/s, 1 keys/s, 51 objects/s recovering 2023-10-09T09:19:53.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:53 smithi016 bash[77892]: cluster 2023-10-09T09:19:52.217510+0000 mgr.y (mgr.25421) 1280 : cluster 0 pgmap v1405: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.9 MiB/s wr, 61 op/s; 17456/278883 objects degraded (6.259%); 11363/278883 objects misplaced (4.074%); 3.2 MiB/s, 1 keys/s, 51 objects/s recovering 2023-10-09T09:19:53.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:53.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:53 smithi094 bash[59677]: cluster 2023-10-09T09:19:52.217510+0000 mgr.y (mgr.25421) 1280 : cluster 0 pgmap v1405: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.9 MiB/s wr, 61 op/s; 17456/278883 objects degraded (6.259%); 11363/278883 objects misplaced (4.074%); 3.2 MiB/s, 1 keys/s, 51 objects/s recovering 2023-10-09T09:19:53.544 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:53.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:53.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 93135 93119 66.1268 0 - 0.0134609 2023-10-09T09:19:53.678 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:53.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:53.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:53.808 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:53.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:53.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:53.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:53.967 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:19:53.971 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:53.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:54.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:54.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:54.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:54.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:54.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:54.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:54.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:54.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:54.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:54.575 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:54.605 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:54.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:54.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 93135 93119 65.3838 0 - 0.0134609 2023-10-09T09:19:54.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:54.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:54.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:54.862 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:54.966 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:54.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:54.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:54.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:55.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:55.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:55.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:55.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:55.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:55.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:55.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:55.356 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:55.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:55 smithi094 bash[59677]: cluster 2023-10-09T09:19:54.218103+0000 mgr.y (mgr.25421) 1281 : cluster 0 pgmap v1406: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.5 MiB/s wr, 55 op/s; 17456/278883 objects degraded (6.259%); 11125/278883 objects misplaced (3.989%); 4.3 MiB/s, 0 keys/s, 69 objects/s recovering 2023-10-09T09:19:55.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:55 smithi094 bash[59677]: cluster 2023-10-09T09:19:54.817461+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:19:55.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:55 smithi094 bash[59677]: audit 2023-10-09T09:19:55.265647+0000 mon.a (mon.0) 1705 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:19:55.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:55.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:55.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:55.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:55 smithi016 bash[75179]: cluster 2023-10-09T09:19:54.218103+0000 mgr.y (mgr.25421) 1281 : cluster 0 pgmap v1406: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.5 MiB/s wr, 55 op/s; 17456/278883 objects degraded (6.259%); 11125/278883 objects misplaced (3.989%); 4.3 MiB/s, 0 keys/s, 69 objects/s recovering 2023-10-09T09:19:55.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:55 smithi016 bash[75179]: cluster 2023-10-09T09:19:54.817461+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:19:55.498 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:55 smithi016 bash[75179]: audit 2023-10-09T09:19:55.265647+0000 mon.a (mon.0) 1705 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:19:55.498 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:55 smithi016 bash[77892]: cluster 2023-10-09T09:19:54.218103+0000 mgr.y (mgr.25421) 1281 : cluster 0 pgmap v1406: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.5 MiB/s wr, 55 op/s; 17456/278883 objects degraded (6.259%); 11125/278883 objects misplaced (3.989%); 4.3 MiB/s, 0 keys/s, 69 objects/s recovering 2023-10-09T09:19:55.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:55 smithi016 bash[77892]: cluster 2023-10-09T09:19:54.817461+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:19:55.499 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:55 smithi016 bash[77892]: audit 2023-10-09T09:19:55.265647+0000 mon.a (mon.0) 1705 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:19:55.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:55.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:55.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:55.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:55.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 16 93135 93119 64.6573 0 - 0.0134609 2023-10-09T09:19:55.644 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:55.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:55.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:55.749 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:19:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:19:55] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T09:19:55.777 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:55.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:55.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:55.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:55.908 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:55.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:55.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:56.011 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:56.039 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:56.143 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:56.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:56.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:56.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:56.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:56.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:56.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:56.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:56.529 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:56.556 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:56.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 91 16 93135 93119 63.9469 0 - 0.0134609 2023-10-09T09:19:56.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:56.688 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:56.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:56.818 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:56.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:56.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:56.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:56.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:57.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:57.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:57.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:57.201 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:57.304 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:57.335 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:57.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:57.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:57.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:57.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:57.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:57 smithi094 bash[59677]: cluster 2023-10-09T09:19:56.219261+0000 mgr.y (mgr.25421) 1282 : cluster 0 pgmap v1407: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.0 MiB/s wr, 48 op/s; 17456/278883 objects degraded (6.259%); 11125/278883 objects misplaced (3.989%); 3.7 MiB/s, 0 keys/s, 59 objects/s recovering 2023-10-09T09:19:57.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:57.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:57.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:57.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 92 16 93135 93119 63.2518 0 - 0.0134609 2023-10-09T09:19:57.637 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:57 smithi016 bash[75179]: cluster 2023-10-09T09:19:56.219261+0000 mgr.y (mgr.25421) 1282 : cluster 0 pgmap v1407: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.0 MiB/s wr, 48 op/s; 17456/278883 objects degraded (6.259%); 11125/278883 objects misplaced (3.989%); 3.7 MiB/s, 0 keys/s, 59 objects/s recovering 2023-10-09T09:19:57.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:57 smithi016 bash[77892]: cluster 2023-10-09T09:19:56.219261+0000 mgr.y (mgr.25421) 1282 : cluster 0 pgmap v1407: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.0 MiB/s wr, 48 op/s; 17456/278883 objects degraded (6.259%); 11125/278883 objects misplaced (3.989%); 3.7 MiB/s, 0 keys/s, 59 objects/s recovering 2023-10-09T09:19:57.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:57.719 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:57.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:57.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:57.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:57.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:19:57.954 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:57.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:19:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:57.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:19:58.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:58.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:58.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:58.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:58.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:58.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:58.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:58.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:58.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:58.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:58.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:58.464 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:58.494 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:58.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:58.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:58.597 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:58.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 93 16 93135 93119 62.5717 0 - 0.0134609 2023-10-09T09:19:58.624 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:58.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:58.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:58.727 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:58.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:19:58.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:58.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:19:58.859 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:58.886 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:58.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:58.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:58.989 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:59.016 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:59.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:59.149 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:19:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:19:59.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:59.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:59.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:59.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:19:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:19:59.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:59.514 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:59 smithi016 bash[75179]: cluster 2023-10-09T09:19:58.220086+0000 mgr.y (mgr.25421) 1283 : cluster 0 pgmap v1408: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 581 KiB/s wr, 9 op/s; 17456/278883 objects degraded (6.259%); 10609/278883 objects misplaced (3.804%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T09:19:59.514 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:19:59 smithi016 bash[75179]: audit 2023-10-09T09:19:59.149619+0000 mon.a (mon.0) 1706 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:19:59.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:59 smithi016 bash[77892]: cluster 2023-10-09T09:19:58.220086+0000 mgr.y (mgr.25421) 1283 : cluster 0 pgmap v1408: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 581 KiB/s wr, 9 op/s; 17456/278883 objects degraded (6.259%); 10609/278883 objects misplaced (3.804%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T09:19:59.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:19:59 smithi016 bash[77892]: audit 2023-10-09T09:19:59.149619+0000 mon.a (mon.0) 1706 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:19:59.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:19:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:19:59.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 94 16 93135 93119 61.9061 0 - 0.0134609 2023-10-09T09:19:59.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:59.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:19:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:19:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:59 smithi094 bash[59677]: cluster 2023-10-09T09:19:58.220086+0000 mgr.y (mgr.25421) 1283 : cluster 0 pgmap v1408: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 581 KiB/s wr, 9 op/s; 17456/278883 objects degraded (6.259%); 10609/278883 objects misplaced (3.804%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T09:19:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:19:59 smithi094 bash[59677]: audit 2023-10-09T09:19:59.149619+0000 mon.a (mon.0) 1706 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:19:59.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:59.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:19:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:19:59.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:59.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:19:59.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:19:59.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:00.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:00.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:00.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:00.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:00.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:00.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:00.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:00.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:00.404 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:00.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:00.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:00.565 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:00.619 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 94.2077 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 93135 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 61.7883 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 39.2117 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 173.75 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 0 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 988 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 627.387 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2780 2023-10-09T09:20:00.620 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 0 2023-10-09T09:20:00.621 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.016173 2023-10-09T09:20:00.621 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.209616 2023-10-09T09:20:00.621 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 17.9731 2023-10-09T09:20:00.621 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00130401 2023-10-09T09:20:00.648 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:20:00.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:00.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:00.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:00.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:00.836 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:00.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:00.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:00.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[75179]: cluster 2023-10-09T09:20:00.000105+0000 mon.a (mon.0) 1707 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded 2023-10-09T09:20:00.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[75179]: cluster 2023-10-09T09:20:00.000141+0000 mon.a (mon.0) 1708 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded 2023-10-09T09:20:00.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[75179]: cluster 2023-10-09T09:20:00.000151+0000 mon.a (mon.0) 1709 : cluster 3 pg 11.2 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T09:20:00.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[75179]: cluster 2023-10-09T09:20:00.000160+0000 mon.a (mon.0) 1710 : cluster 3 pg 11.4 is active+recovery_wait+undersized+degraded+remapped, acting [7,6] 2023-10-09T09:20:00.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[75179]: cluster 2023-10-09T09:20:00.000169+0000 mon.a (mon.0) 1711 : cluster 3 pg 11.c is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-10-09T09:20:00.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[75179]: cluster 2023-10-09T09:20:00.220956+0000 mgr.y (mgr.25421) 1284 : cluster 0 pgmap v1409: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 15 op/s; 23142/279438 objects degraded (8.282%); 10609/279438 objects misplaced (3.797%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:20:00.952 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[77892]: cluster 2023-10-09T09:20:00.000105+0000 mon.a (mon.0) 1707 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded 2023-10-09T09:20:00.952 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[77892]: cluster 2023-10-09T09:20:00.000141+0000 mon.a (mon.0) 1708 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded 2023-10-09T09:20:00.953 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[77892]: cluster 2023-10-09T09:20:00.000151+0000 mon.a (mon.0) 1709 : cluster 3 pg 11.2 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T09:20:00.953 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[77892]: cluster 2023-10-09T09:20:00.000160+0000 mon.a (mon.0) 1710 : cluster 3 pg 11.4 is active+recovery_wait+undersized+degraded+remapped, acting [7,6] 2023-10-09T09:20:00.953 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[77892]: cluster 2023-10-09T09:20:00.000169+0000 mon.a (mon.0) 1711 : cluster 3 pg 11.c is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-10-09T09:20:00.953 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:00 smithi016 bash[77892]: cluster 2023-10-09T09:20:00.220956+0000 mgr.y (mgr.25421) 1284 : cluster 0 pgmap v1409: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 15 op/s; 23142/279438 objects degraded (8.282%); 10609/279438 objects misplaced (3.797%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:20:00.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:00.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:01.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:01.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:01.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:01.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:00 smithi094 bash[59677]: cluster 2023-10-09T09:20:00.000105+0000 mon.a (mon.0) 1707 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded 2023-10-09T09:20:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:00 smithi094 bash[59677]: cluster 2023-10-09T09:20:00.000141+0000 mon.a (mon.0) 1708 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 17456/278883 objects degraded (6.259%), 3 pgs degraded 2023-10-09T09:20:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:00 smithi094 bash[59677]: cluster 2023-10-09T09:20:00.000151+0000 mon.a (mon.0) 1709 : cluster 3 pg 11.2 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T09:20:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:00 smithi094 bash[59677]: cluster 2023-10-09T09:20:00.000160+0000 mon.a (mon.0) 1710 : cluster 3 pg 11.4 is active+recovery_wait+undersized+degraded+remapped, acting [7,6] 2023-10-09T09:20:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:00 smithi094 bash[59677]: cluster 2023-10-09T09:20:00.000169+0000 mon.a (mon.0) 1711 : cluster 3 pg 11.c is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-10-09T09:20:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:00 smithi094 bash[59677]: cluster 2023-10-09T09:20:00.220956+0000 mgr.y (mgr.25421) 1284 : cluster 0 pgmap v1409: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 15 op/s; 23142/279438 objects degraded (8.282%); 10609/279438 objects misplaced (3.797%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:20:01.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:01.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:01.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:01.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:01.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:01.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:01.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:01.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:01.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:01.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:01.671 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:01.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:01.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:01.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:01.802 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:01.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:01.833 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:01 smithi016 bash[75179]: cluster 2023-10-09T09:20:00.820920+0000 mon.a (mon.0) 1712 : cluster 3 Health check update: Degraded data redundancy: 23142/279438 objects degraded (8.282%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:01.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:01 smithi016 bash[77892]: cluster 2023-10-09T09:20:00.820920+0000 mon.a (mon.0) 1712 : cluster 3 Health check update: Degraded data redundancy: 23142/279438 objects degraded (8.282%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:01.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:01.917 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:01 smithi094 bash[59677]: cluster 2023-10-09T09:20:00.820920+0000 mon.a (mon.0) 1712 : cluster 3 Health check update: Degraded data redundancy: 23142/279438 objects degraded (8.282%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:01.937 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:01.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:02.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:02.080 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:02.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:02.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:02.184 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:02.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:02.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:02.324 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:02.700 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:02.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:02.803 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:02.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:02.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:02.943 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:02.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:02 smithi094 bash[59677]: cluster 2023-10-09T09:20:02.221746+0000 mgr.y (mgr.25421) 1285 : cluster 0 pgmap v1410: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 36 op/s; 23142/278682 objects degraded (8.304%); 10387/278682 objects misplaced (3.727%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T09:20:02.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:02.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:02.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:02 smithi016 bash[75179]: cluster 2023-10-09T09:20:02.221746+0000 mgr.y (mgr.25421) 1285 : cluster 0 pgmap v1410: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 36 op/s; 23142/278682 objects degraded (8.304%); 10387/278682 objects misplaced (3.727%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T09:20:02.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:02 smithi016 bash[77892]: cluster 2023-10-09T09:20:02.221746+0000 mgr.y (mgr.25421) 1285 : cluster 0 pgmap v1410: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 36 op/s; 23142/278682 objects degraded (8.304%); 10387/278682 objects misplaced (3.727%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T09:20:03.087 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:03.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:03.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:03.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:03.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:03.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:03.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:03.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:03.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:03.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:03.547 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:03.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:03.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:03.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:03.679 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:03.703 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:03.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:03.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:03.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:03.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:03.963 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:03.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:04.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:04.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:04.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:04.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:04.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:04.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:04.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:04.442 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:04 smithi016 bash[75179]: audit 2023-10-09T09:20:04.257970+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:04.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:04 smithi016 bash[77892]: audit 2023-10-09T09:20:04.257970+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:04.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:04.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:04 smithi094 bash[59677]: audit 2023-10-09T09:20:04.257970+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:04.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:04.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:04.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:04.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:04.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:04.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:04.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:04.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:04.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:04.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:05.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:05.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:05.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:05.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:05.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:05.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:05.280 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:05.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:05.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:05.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:05.427 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:05.466 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:05.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:05.571 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:05.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:05 smithi094 bash[59677]: cluster 2023-10-09T09:20:04.222361+0000 mgr.y (mgr.25421) 1286 : cluster 0 pgmap v1411: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 47 op/s; 28229/278283 objects degraded (10.144%); 5186/278283 objects misplaced (1.864%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:20:05.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:05 smithi094 bash[59677]: audit 2023-10-09T09:20:05.279182+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:05.609 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:05.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:05.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:05.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:05.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:05 smithi016 bash[77892]: cluster 2023-10-09T09:20:04.222361+0000 mgr.y (mgr.25421) 1286 : cluster 0 pgmap v1411: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 47 op/s; 28229/278283 objects degraded (10.144%); 5186/278283 objects misplaced (1.864%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:20:05.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:05 smithi016 bash[77892]: audit 2023-10-09T09:20:05.279182+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:05.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:05 smithi016 bash[75179]: cluster 2023-10-09T09:20:04.222361+0000 mgr.y (mgr.25421) 1286 : cluster 0 pgmap v1411: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 986 KiB/s wr, 47 op/s; 28229/278283 objects degraded (10.144%); 5186/278283 objects misplaced (1.864%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:20:05.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:05 smithi016 bash[75179]: audit 2023-10-09T09:20:05.279182+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:05.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:05.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:05.854 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:05.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:05.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:05.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:05.991 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:20:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:20:05] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T09:20:06.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:06.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:06.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:06.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:06.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:06.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:06.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:06.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:06.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:06.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:06.557 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:06.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:06.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:06 smithi016 bash[75179]: audit 2023-10-09T09:20:05.645796+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:06.647 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:06 smithi016 bash[75179]: audit 2023-10-09T09:20:05.659389+0000 mon.a (mon.0) 1716 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:06.647 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:06 smithi016 bash[77892]: audit 2023-10-09T09:20:05.645796+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:06.647 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:06 smithi016 bash[77892]: audit 2023-10-09T09:20:05.659389+0000 mon.a (mon.0) 1716 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:06.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:06.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:06.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:06.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:06.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:06.930 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:06.939 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:06 smithi094 bash[59677]: audit 2023-10-09T09:20:05.645796+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:06.939 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:06 smithi094 bash[59677]: audit 2023-10-09T09:20:05.659389+0000 mon.a (mon.0) 1716 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:06.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:06.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:06.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:07.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:07.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:07.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:07.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:07.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:07.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:07.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:07.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:07.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:07.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:07.479 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:07.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:07.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:07.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:07.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:07.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:07.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:07.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:07.761 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:07.772 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:07 smithi094 bash[59677]: cluster 2023-10-09T09:20:06.223057+0000 mgr.y (mgr.25421) 1287 : cluster 0 pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 185 op/s; 22549/273408 objects degraded (8.247%); 10729/273408 objects misplaced (3.924%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:20:07.772 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:07 smithi094 bash[59677]: cluster 2023-10-09T09:20:06.657819+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 22549/273408 objects degraded (8.247%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:07.787 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:07.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:07.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:07.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:07.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:07.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:07 smithi016 bash[75179]: cluster 2023-10-09T09:20:06.223057+0000 mgr.y (mgr.25421) 1287 : cluster 0 pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 185 op/s; 22549/273408 objects degraded (8.247%); 10729/273408 objects misplaced (3.924%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:20:08.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:07 smithi016 bash[75179]: cluster 2023-10-09T09:20:06.657819+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 22549/273408 objects degraded (8.247%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:08.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:07 smithi016 bash[77892]: cluster 2023-10-09T09:20:06.223057+0000 mgr.y (mgr.25421) 1287 : cluster 0 pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 185 op/s; 22549/273408 objects degraded (8.247%); 10729/273408 objects misplaced (3.924%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:20:08.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:07 smithi016 bash[77892]: cluster 2023-10-09T09:20:06.657819+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 22549/273408 objects degraded (8.247%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:08.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:08.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:08.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:08.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:08.205 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:20:08.205 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:20:08.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:08.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:08.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:08.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:08.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:08.389 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:08.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:08.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:08.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:08.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:08.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:08.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:08.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:08.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:08.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:08.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:08.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:08.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:08.880 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:08.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:08.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:08.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:09.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:09.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:09.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:09.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:09.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:09.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:09.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:09.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:09.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:09.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:09.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:09.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:09.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:09.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:09.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:09 smithi016 bash[75179]: cluster 2023-10-09T09:20:08.223655+0000 mgr.y (mgr.25421) 1288 : cluster 0 pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 300 op/s; 22549/269265 objects degraded (8.374%); 10634/269265 objects misplaced (3.949%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:20:09.916 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:09 smithi016 bash[77892]: cluster 2023-10-09T09:20:08.223655+0000 mgr.y (mgr.25421) 1288 : cluster 0 pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 300 op/s; 22549/269265 objects degraded (8.374%); 10634/269265 objects misplaced (3.949%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:20:09.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:09.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:09.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:09.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:09.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:09 smithi094 bash[59677]: cluster 2023-10-09T09:20:08.223655+0000 mgr.y (mgr.25421) 1288 : cluster 0 pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 300 op/s; 22549/269265 objects degraded (8.374%); 10634/269265 objects misplaced (3.949%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:20:10.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:10.116 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:10.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:10.222 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:10.252 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:10.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:10.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:10.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:10.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:10.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:10.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:10.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:10.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:10.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:10.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:10.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:10.793 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:10.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:10.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:10.905 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:10 smithi094 bash[59677]: cluster 2023-10-09T09:20:10.224631+0000 mgr.y (mgr.25421) 1289 : cluster 0 pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 331 op/s; 22623/268146 objects degraded (8.437%); 10447/268146 objects misplaced (3.896%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T09:20:10.932 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:10.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:10.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:11.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:11.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:11.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:11.182 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:11.213 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:11.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:11.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:10 smithi016 bash[77892]: cluster 2023-10-09T09:20:10.224631+0000 mgr.y (mgr.25421) 1289 : cluster 0 pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 331 op/s; 22623/268146 objects degraded (8.437%); 10447/268146 objects misplaced (3.896%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T09:20:11.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:10 smithi016 bash[75179]: cluster 2023-10-09T09:20:10.224631+0000 mgr.y (mgr.25421) 1289 : cluster 0 pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 331 op/s; 22623/268146 objects degraded (8.437%); 10447/268146 objects misplaced (3.896%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T09:20:11.319 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:11.347 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:11.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:11.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:11.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:11.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:11.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:11.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:11.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:11.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:11.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:11.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:11.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:11.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:11.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:11.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:11.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:12.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:12.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:12.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:12.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:12.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:12.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:12.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:12.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:12.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:12.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:12.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:12.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:12.551 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:12.582 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:12.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.587 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:12 smithi094 bash[59677]: cluster 2023-10-09T09:20:12.276670+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 22623/261543 objects degraded (8.650%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:12.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:12.677 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:12 smithi016 bash[75179]: cluster 2023-10-09T09:20:12.276670+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 22623/261543 objects degraded (8.650%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:12.678 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:12 smithi016 bash[77892]: cluster 2023-10-09T09:20:12.276670+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 22623/261543 objects degraded (8.650%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:12.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:12.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:12.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:12.814 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:12.837 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:12.939 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:12.969 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:12.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:12.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:13.002 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:20:13.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:13.104 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:13.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:13.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:13.258 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:13.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:13.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:13.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:13.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:13.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:13.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:13.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:13 smithi094 bash[59677]: cluster 2023-10-09T09:20:12.225366+0000 mgr.y (mgr.25421) 1290 : cluster 0 pgmap v1415: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 499 op/s; 22623/261543 objects degraded (8.650%); 10369/261543 objects misplaced (3.965%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:20:13.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:13.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:13.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:13.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:13.644 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:13.672 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:13.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:13 smithi016 bash[75179]: cluster 2023-10-09T09:20:12.225366+0000 mgr.y (mgr.25421) 1290 : cluster 0 pgmap v1415: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 499 op/s; 22623/261543 objects degraded (8.650%); 10369/261543 objects misplaced (3.965%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:20:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:13 smithi016 bash[77892]: cluster 2023-10-09T09:20:12.225366+0000 mgr.y (mgr.25421) 1290 : cluster 0 pgmap v1415: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 499 op/s; 22623/261543 objects degraded (8.650%); 10369/261543 objects misplaced (3.965%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:20:13.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:13.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:13.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:13.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:13.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:13.955 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:13.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:13.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:14.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:14.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:14.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:14.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:14.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:14.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:14.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:14.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:14.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:14.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:14.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:14.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:14.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:14.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:14.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:14.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:14.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:14.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:14.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:14.730 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:14.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:14.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:14.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:14.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:14.894 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:14.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:14.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:15.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:15.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:15.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:15.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:15.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:15.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:15.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:15.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:15.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:15.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:15 smithi016 bash[77892]: cluster 2023-10-09T09:20:14.225990+0000 mgr.y (mgr.25421) 1291 : cluster 0 pgmap v1416: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 508 op/s; 22623/260439 objects degraded (8.686%); 10369/260439 objects misplaced (3.981%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T09:20:15.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:15.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:15.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:15.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:15.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:15.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:15.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:15.727 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:15.736 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:15 smithi094 bash[59677]: cluster 2023-10-09T09:20:14.225990+0000 mgr.y (mgr.25421) 1291 : cluster 0 pgmap v1416: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 508 op/s; 22623/260439 objects degraded (8.686%); 10369/260439 objects misplaced (3.981%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T09:20:15.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:15.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:15.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:15.755 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:20:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:20:15] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T09:20:15.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:15 smithi016 bash[75179]: cluster 2023-10-09T09:20:14.225990+0000 mgr.y (mgr.25421) 1291 : cluster 0 pgmap v1416: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 508 op/s; 22623/260439 objects degraded (8.686%); 10369/260439 objects misplaced (3.981%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T09:20:15.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:15.883 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:15.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:15.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:16.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:16.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:16.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:16.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:16.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:16.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:16.314 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:16.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:16.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:16.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:16.553 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:16.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:16.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:16.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:16.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:16.825 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:16 smithi094 bash[59677]: audit 2023-10-09T09:20:15.698538+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:16.825 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:16 smithi094 bash[59677]: audit 2023-10-09T09:20:15.705547+0000 mon.a (mon.0) 1720 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:16.825 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:16 smithi094 bash[59677]: audit 2023-10-09T09:20:16.275680+0000 mon.a (mon.0) 1721 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:20:16.825 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:16 smithi094 bash[59677]: audit 2023-10-09T09:20:16.277023+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:20:16.825 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:16 smithi094 bash[59677]: audit 2023-10-09T09:20:16.283013+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:16.840 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:16.946 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:16.972 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:16.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:16.997 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:20:16.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[75179]: audit 2023-10-09T09:20:15.698538+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:16.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[75179]: audit 2023-10-09T09:20:15.705547+0000 mon.a (mon.0) 1720 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[75179]: audit 2023-10-09T09:20:16.275680+0000 mon.a (mon.0) 1721 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[75179]: audit 2023-10-09T09:20:16.277023+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[75179]: audit 2023-10-09T09:20:16.283013+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[77892]: audit 2023-10-09T09:20:15.698538+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[77892]: audit 2023-10-09T09:20:15.705547+0000 mon.a (mon.0) 1720 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[77892]: audit 2023-10-09T09:20:16.275680+0000 mon.a (mon.0) 1721 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[77892]: audit 2023-10-09T09:20:16.277023+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:20:17.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:16 smithi016 bash[77892]: audit 2023-10-09T09:20:16.283013+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:17.060 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:20:17.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:17.095 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:20:17.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:17.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:17.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:17.131 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:20:17.168 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:20:17.220 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:17.229 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:20:17.244 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:17.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:17.273 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:20:17.315 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:20:17.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.328 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.328 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:20:17.346 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:17.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:17.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:17.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:17.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:17.520 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:17.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:17.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:17.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:17.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:17.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[75179]: cluster 2023-10-09T09:20:16.226707+0000 mgr.y (mgr.25421) 1292 : cluster 0 pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 635 op/s; 22682/255471 objects degraded (8.879%); 10191/255471 objects misplaced (3.989%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T09:20:17.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[75179]: audit 2023-10-09T09:20:16.956389+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.16:0/2485826945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:17.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[75179]: audit 2023-10-09T09:20:17.679964+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.16:0/85556078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:20:17.749 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:20:17.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:17.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:17.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[75179]: audit 2023-10-09T09:20:17.680758+0000 mon.a (mon.0) 1724 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:20:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[75179]: debug 2023-10-09T09:20:17.827+0000 7fa046a4d700 -1 mon.a@0(leader).osd e306 definitely_dead 0 2023-10-09T09:20:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[77892]: cluster 2023-10-09T09:20:16.226707+0000 mgr.y (mgr.25421) 1292 : cluster 0 pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 635 op/s; 22682/255471 objects degraded (8.879%); 10191/255471 objects misplaced (3.989%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T09:20:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[77892]: audit 2023-10-09T09:20:16.956389+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.16:0/2485826945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[77892]: audit 2023-10-09T09:20:17.679964+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.16:0/85556078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:20:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:17 smithi016 bash[77892]: audit 2023-10-09T09:20:17.680758+0000 mon.a (mon.0) 1724 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:20:17.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:17.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:17.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:17.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:17.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:17 smithi094 bash[59677]: cluster 2023-10-09T09:20:16.226707+0000 mgr.y (mgr.25421) 1292 : cluster 0 pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 69 KiB/s wr, 635 op/s; 22682/255471 objects degraded (8.879%); 10191/255471 objects misplaced (3.989%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T09:20:17.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:17 smithi094 bash[59677]: audit 2023-10-09T09:20:16.956389+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.16:0/2485826945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:17.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:17 smithi094 bash[59677]: audit 2023-10-09T09:20:17.679964+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.16:0/85556078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:20:17.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:17 smithi094 bash[59677]: audit 2023-10-09T09:20:17.680758+0000 mon.a (mon.0) 1724 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:20:18.022 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:18.045 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:18.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:18.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:18.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:18.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:18.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:18.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:18.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:18.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:18.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:18.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:18.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:18.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:18.750 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.2. 2023-10-09T09:20:18.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:18.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:18.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:18.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: cluster 2023-10-09T09:20:17.705133+0000 mon.a (mon.0) 1725 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:18.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: cluster 2023-10-09T09:20:17.733166+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: cluster 2023-10-09T09:20:17.736053+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: audit 2023-10-09T09:20:17.737227+0000 mon.a (mon.0) 1726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:20:18.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: cluster 2023-10-09T09:20:17.737720+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T09:20:18.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: cluster 2023-10-09T09:20:17.737986+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: cluster 2023-10-09T09:20:17.741395+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[77892]: audit 2023-10-09T09:20:17.834905+0000 mon.a (mon.0) 1728 : audit 0 from='client.? 172.21.15.16:0/695217520' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:20:18.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: cluster 2023-10-09T09:20:17.705133+0000 mon.a (mon.0) 1725 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:18.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: cluster 2023-10-09T09:20:17.733166+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: cluster 2023-10-09T09:20:17.736053+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.977 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: audit 2023-10-09T09:20:17.737227+0000 mon.a (mon.0) 1726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:20:18.977 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: cluster 2023-10-09T09:20:17.737720+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T09:20:18.977 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: cluster 2023-10-09T09:20:17.737986+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.977 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: cluster 2023-10-09T09:20:17.741395+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.977 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:18 smithi016 bash[75179]: audit 2023-10-09T09:20:17.834905+0000 mon.a (mon.0) 1728 : audit 0 from='client.? 172.21.15.16:0/695217520' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:20:18.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:18.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: cluster 2023-10-09T09:20:17.705133+0000 mon.a (mon.0) 1725 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: cluster 2023-10-09T09:20:17.733166+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: cluster 2023-10-09T09:20:17.736053+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: audit 2023-10-09T09:20:17.737227+0000 mon.a (mon.0) 1726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: cluster 2023-10-09T09:20:17.737720+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: cluster 2023-10-09T09:20:17.737986+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: cluster 2023-10-09T09:20:17.741395+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T09:20:18.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:18 smithi094 bash[59677]: audit 2023-10-09T09:20:17.834905+0000 mon.a (mon.0) 1728 : audit 0 from='client.? 172.21.15.16:0/695217520' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:20:19.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:19.127 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:19.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:19.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:19.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:19.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:19.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:19.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:19.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:19.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:19.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:19.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:19.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:19.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:19.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:19.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:19.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: cluster 2023-10-09T09:20:18.227253+0000 mgr.y (mgr.25421) 1293 : cluster 0 pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 22682/251739 objects degraded (9.010%); 10114/251739 objects misplaced (4.018%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: cluster 2023-10-09T09:20:18.722961+0000 mon.a (mon.0) 1729 : cluster 3 Health check update: Degraded data redundancy: 22682/251739 objects degraded (9.010%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: audit 2023-10-09T09:20:18.739469+0000 mon.a (mon.0) 1730 : audit 1 from='client.? 172.21.15.16:0/695217520' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: cluster 2023-10-09T09:20:18.739538+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: cluster 2023-10-09T09:20:18.739961+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: cluster 2023-10-09T09:20:18.740816+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: audit 2023-10-09T09:20:19.264338+0000 mon.a (mon.0) 1732 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[75179]: audit 2023-10-09T09:20:19.265737+0000 mon.a (mon.0) 1733 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: cluster 2023-10-09T09:20:18.227253+0000 mgr.y (mgr.25421) 1293 : cluster 0 pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 22682/251739 objects degraded (9.010%); 10114/251739 objects misplaced (4.018%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: cluster 2023-10-09T09:20:18.722961+0000 mon.a (mon.0) 1729 : cluster 3 Health check update: Degraded data redundancy: 22682/251739 objects degraded (9.010%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: audit 2023-10-09T09:20:18.739469+0000 mon.a (mon.0) 1730 : audit 1 from='client.? 172.21.15.16:0/695217520' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: cluster 2023-10-09T09:20:18.739538+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: cluster 2023-10-09T09:20:18.739961+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: cluster 2023-10-09T09:20:18.740816+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: audit 2023-10-09T09:20:19.264338+0000 mon.a (mon.0) 1732 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:19.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:19 smithi016 bash[77892]: audit 2023-10-09T09:20:19.265737+0000 mon.a (mon.0) 1733 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:19.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:19.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:19.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:19.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:19.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: cluster 2023-10-09T09:20:18.227253+0000 mgr.y (mgr.25421) 1293 : cluster 0 pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 22682/251739 objects degraded (9.010%); 10114/251739 objects misplaced (4.018%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: cluster 2023-10-09T09:20:18.722961+0000 mon.a (mon.0) 1729 : cluster 3 Health check update: Degraded data redundancy: 22682/251739 objects degraded (9.010%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: audit 2023-10-09T09:20:18.739469+0000 mon.a (mon.0) 1730 : audit 1 from='client.? 172.21.15.16:0/695217520' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: cluster 2023-10-09T09:20:18.739538+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: cluster 2023-10-09T09:20:18.739961+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: cluster 2023-10-09T09:20:18.740816+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: audit 2023-10-09T09:20:19.264338+0000 mon.a (mon.0) 1732 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:19.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:19 smithi094 bash[59677]: audit 2023-10-09T09:20:19.265737+0000 mon.a (mon.0) 1733 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:20.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:20.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:20.174 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:20.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:20.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:20.312 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:20.336 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:20.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:20.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:20.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:20.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:20.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:20.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:20.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:20.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:20.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:20.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.739073+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.742851+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.742992+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.744839+0000 mon.a (mon.0) 1734 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.745043+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.745590+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.745928+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.747134+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[75179]: cluster 2023-10-09T09:20:19.748167+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.739073+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.742851+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.840 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:20.867 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:20.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:20.970 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:20.979 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.739073+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.979 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.742851+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.742992+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.744839+0000 mon.a (mon.0) 1734 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-10-09T09:20:20.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.745043+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.745590+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.745928+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.747134+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.980 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:20 smithi094 bash[59677]: cluster 2023-10-09T09:20:19.748167+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.994 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:20.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:20.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.742992+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:20.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.744839+0000 mon.a (mon.0) 1734 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-10-09T09:20:21.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.745043+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:21.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.745590+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:21.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.745928+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:21.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.747134+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:21.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:20 smithi016 bash[77892]: cluster 2023-10-09T09:20:19.748167+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T09:20:21.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:21.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:21.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:21.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:21.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:21.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:21.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:21.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:21.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:21.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:21.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:21.495 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:21.531 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:21.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:21.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:21.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:21.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:21.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:21.810 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T09:20:21.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:21.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:21.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:21.924 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:21 smithi094 bash[59677]: cluster 2023-10-09T09:20:20.227764+0000 mgr.y (mgr.25421) 1294 : cluster 0 pgmap v1422: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22543/250710 objects degraded (8.992%); 4936/250710 objects misplaced (1.969%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T09:20:21.942 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:21.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:21.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:21 smithi016 bash[75179]: cluster 2023-10-09T09:20:20.227764+0000 mgr.y (mgr.25421) 1294 : cluster 0 pgmap v1422: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22543/250710 objects degraded (8.992%); 4936/250710 objects misplaced (1.969%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T09:20:22.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:21 smithi016 bash[77892]: cluster 2023-10-09T09:20:20.227764+0000 mgr.y (mgr.25421) 1294 : cluster 0 pgmap v1422: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22543/250710 objects degraded (8.992%); 4936/250710 objects misplaced (1.969%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T09:20:22.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:22.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:22.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:22.144 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:20:22.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:20:22.188 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:22.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:22.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:22.320 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:22.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:22.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:22.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:22.475 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:22.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:22.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:22.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:22.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:22.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:22.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:22.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:22.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:22.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:22.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:22.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:22.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:22.986 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:23.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:23.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:23.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:23.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:23.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:23.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:23.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:23.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:23.410 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:23.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:23.513 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:23.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:23.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:23.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:23.678 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:23.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:20:23.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:23.814 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:23.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:23.919 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:23.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:23 smithi094 bash[59677]: cluster 2023-10-09T09:20:22.228555+0000 mgr.y (mgr.25421) 1295 : cluster 0 pgmap v1423: 89 pgs: 1 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 12949/245379 objects degraded (5.277%); 2.1 MiB/s, 1 keys/s, 34 objects/s recovering 2023-10-09T09:20:23.944 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:23.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:23.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:23.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:23 smithi016 bash[75179]: cluster 2023-10-09T09:20:22.228555+0000 mgr.y (mgr.25421) 1295 : cluster 0 pgmap v1423: 89 pgs: 1 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 12949/245379 objects degraded (5.277%); 2.1 MiB/s, 1 keys/s, 34 objects/s recovering 2023-10-09T09:20:24.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:23 smithi016 bash[77892]: cluster 2023-10-09T09:20:22.228555+0000 mgr.y (mgr.25421) 1295 : cluster 0 pgmap v1423: 89 pgs: 1 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 12949/245379 objects degraded (5.277%); 2.1 MiB/s, 1 keys/s, 34 objects/s recovering 2023-10-09T09:20:24.047 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:24.073 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:24.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:24.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:24.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:24.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:24.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:24.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:24.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:24.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:24.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:24.482 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:24.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:24.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:24.585 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:24.622 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:24.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:24.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:24.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:24.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:24.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:24.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:24.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:24.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:24.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:24.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:24 smithi016 bash[75179]: cluster 2023-10-09T09:20:24.229106+0000 mgr.y (mgr.25421) 1296 : cluster 0 pgmap v1424: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 2748/244284 objects degraded (1.125%); 1.3 MiB/s, 1 keys/s, 40 objects/s recovering 2023-10-09T09:20:25.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:24 smithi016 bash[75179]: cluster 2023-10-09T09:20:24.746235+0000 mon.a (mon.0) 1735 : cluster 3 Health check update: Degraded data redundancy: 2748/244284 objects degraded (1.125%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:25.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:24 smithi016 bash[77892]: cluster 2023-10-09T09:20:24.229106+0000 mgr.y (mgr.25421) 1296 : cluster 0 pgmap v1424: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 2748/244284 objects degraded (1.125%); 1.3 MiB/s, 1 keys/s, 40 objects/s recovering 2023-10-09T09:20:25.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:24 smithi016 bash[77892]: cluster 2023-10-09T09:20:24.746235+0000 mon.a (mon.0) 1735 : cluster 3 Health check update: Degraded data redundancy: 2748/244284 objects degraded (1.125%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:25.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:25.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:25.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:25.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:25.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:24 smithi094 bash[59677]: cluster 2023-10-09T09:20:24.229106+0000 mgr.y (mgr.25421) 1296 : cluster 0 pgmap v1424: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 2748/244284 objects degraded (1.125%); 1.3 MiB/s, 1 keys/s, 40 objects/s recovering 2023-10-09T09:20:25.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:24 smithi094 bash[59677]: cluster 2023-10-09T09:20:24.746235+0000 mon.a (mon.0) 1735 : cluster 3 Health check update: Degraded data redundancy: 2748/244284 objects degraded (1.125%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:25.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:25.288 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:25.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:25.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:25.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:25.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:25.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:25.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:25.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:25.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:25.807 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:25.910 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:25.934 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:25.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:25.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:20:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:20:25] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T09:20:26.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:26.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:26.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:26.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:26.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:26.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:26.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:26.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:26.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:26.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:26.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:26.435 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:20:26.443 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:26.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:26.472 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:26.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:26.573 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:26.585 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:26 smithi094 bash[59677]: audit 2023-10-09T09:20:25.310743+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:26.601 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:26.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:26.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:26.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:26.714 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:26 smithi016 bash[75179]: audit 2023-10-09T09:20:25.310743+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:26.714 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:26 smithi016 bash[77892]: audit 2023-10-09T09:20:25.310743+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:26.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:26.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:26.810 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:20:26.810 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:20:26.810 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T09:20:26.811 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:20:26.811 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:20:26.811 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T09:20:26.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:26.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:26.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:26.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:26.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:27.019 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:27.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:27.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:27.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:27.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:27.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:27.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:27.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:27.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:27.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:27 smithi016 bash[75179]: cluster 2023-10-09T09:20:26.229673+0000 mgr.y (mgr.25421) 1297 : cluster 0 pgmap v1425: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3430/238059 objects degraded (1.441%); 1.1 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T09:20:27.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:27 smithi016 bash[77892]: cluster 2023-10-09T09:20:26.229673+0000 mgr.y (mgr.25421) 1297 : cluster 0 pgmap v1425: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3430/238059 objects degraded (1.441%); 1.1 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T09:20:27.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:27.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:27.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:27.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:27.676 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:27 smithi094 bash[59677]: cluster 2023-10-09T09:20:26.229673+0000 mgr.y (mgr.25421) 1297 : cluster 0 pgmap v1425: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3430/238059 objects degraded (1.441%); 1.1 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T09:20:27.692 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:27.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:27.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:27.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:27.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:27.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:27.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:28.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:28.089 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:28.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:28.195 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:28.223 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:28.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:28.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:28.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:28.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:28.381 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:20:28.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:28.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:28.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:28.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:28.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:28.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:28.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:28.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:28.744 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:28.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:28.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:28.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:28.874 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:28.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:28.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:28.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.011 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:29.115 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:29.142 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:29.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:29.248 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:29.272 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:29.375 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.385 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:29 smithi094 bash[59677]: cluster 2023-10-09T09:20:28.230321+0000 mgr.y (mgr.25421) 1298 : cluster 0 pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 3430/233151 objects degraded (1.471%); 930 KiB/s, 1 keys/s, 55 objects/s recovering 2023-10-09T09:20:29.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:29.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:29.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:29.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:29.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:29 smithi016 bash[75179]: cluster 2023-10-09T09:20:28.230321+0000 mgr.y (mgr.25421) 1298 : cluster 0 pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 3430/233151 objects degraded (1.471%); 930 KiB/s, 1 keys/s, 55 objects/s recovering 2023-10-09T09:20:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:29 smithi016 bash[77892]: cluster 2023-10-09T09:20:28.230321+0000 mgr.y (mgr.25421) 1298 : cluster 0 pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 3430/233151 objects degraded (1.471%); 930 KiB/s, 1 keys/s, 55 objects/s recovering 2023-10-09T09:20:29.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:29.796 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:29.899 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:29.927 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:29.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:29.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:30.029 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:30.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:30.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:30.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:30.194 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:30.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:30.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:30.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:30.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:30.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:30.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:30.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:30.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:30.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:30 smithi094 bash[59677]: cluster 2023-10-09T09:20:29.828037+0000 mon.a (mon.0) 1737 : cluster 3 Health check update: Degraded data redundancy: 3430/233151 objects degraded (1.471%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:30.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:30.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:30 smithi016 bash[75179]: cluster 2023-10-09T09:20:29.828037+0000 mon.a (mon.0) 1737 : cluster 3 Health check update: Degraded data redundancy: 3430/233151 objects degraded (1.471%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:30.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:30 smithi016 bash[77892]: cluster 2023-10-09T09:20:29.828037+0000 mon.a (mon.0) 1737 : cluster 3 Health check update: Degraded data redundancy: 3430/233151 objects degraded (1.471%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:30.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:30.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:30.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:30.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:30.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:30.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:30.839 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:20:30.839 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:20:30.839 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T09:20:30.840 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T09:20:30.840 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:20:30.852 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:30.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:30.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:30.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:31.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:31.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:31.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:31.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:31.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:31.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:31.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:31.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:31.414 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:31 smithi016 bash[75179]: cluster 2023-10-09T09:20:30.231040+0000 mgr.y (mgr.25421) 1299 : cluster 0 pgmap v1427: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 3430/231768 objects degraded (1.480%); 268 KiB/s, 0 keys/s, 40 objects/s recovering 2023-10-09T09:20:31.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:31 smithi016 bash[77892]: cluster 2023-10-09T09:20:30.231040+0000 mgr.y (mgr.25421) 1299 : cluster 0 pgmap v1427: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 3430/231768 objects degraded (1.480%); 268 KiB/s, 0 keys/s, 40 objects/s recovering 2023-10-09T09:20:31.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:31.546 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:31.572 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:31.631 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:20:31.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:31 smithi094 bash[59677]: cluster 2023-10-09T09:20:30.231040+0000 mgr.y (mgr.25421) 1299 : cluster 0 pgmap v1427: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 3430/231768 objects degraded (1.480%); 268 KiB/s, 0 keys/s, 40 objects/s recovering 2023-10-09T09:20:31.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:31.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:31.817 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:31.849 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:31.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:31.958 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:31.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:31.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:32.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:32.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:32.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:32.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:32.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:32.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:32.481 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:32.507 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:32.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:32.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:32.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:32.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:32.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:32.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:32.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:32.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:32.892 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:32.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:32.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:33.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[75179]: cluster 2023-10-09T09:20:31.853766+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T09:20:33.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[75179]: cluster 2023-10-09T09:20:31.854800+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T09:20:33.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[75179]: cluster 2023-10-09T09:20:31.856898+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-10-09T09:20:33.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[75179]: cluster 2023-10-09T09:20:32.231692+0000 mgr.y (mgr.25421) 1300 : cluster 0 pgmap v1429: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3430/223308 objects degraded (1.536%); 96 KiB/s, 39 objects/s recovering 2023-10-09T09:20:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[77892]: cluster 2023-10-09T09:20:31.853766+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T09:20:33.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[77892]: cluster 2023-10-09T09:20:31.854800+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T09:20:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[77892]: cluster 2023-10-09T09:20:31.856898+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-10-09T09:20:33.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:32 smithi016 bash[77892]: cluster 2023-10-09T09:20:32.231692+0000 mgr.y (mgr.25421) 1300 : cluster 0 pgmap v1429: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3430/223308 objects degraded (1.536%); 96 KiB/s, 39 objects/s recovering 2023-10-09T09:20:33.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:33.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:33.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:33.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:33.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:32 smithi094 bash[59677]: cluster 2023-10-09T09:20:31.853766+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T09:20:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:32 smithi094 bash[59677]: cluster 2023-10-09T09:20:31.854800+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T09:20:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:32 smithi094 bash[59677]: cluster 2023-10-09T09:20:31.856898+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-10-09T09:20:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:32 smithi094 bash[59677]: cluster 2023-10-09T09:20:32.231692+0000 mgr.y (mgr.25421) 1300 : cluster 0 pgmap v1429: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3430/223308 objects degraded (1.536%); 96 KiB/s, 39 objects/s recovering 2023-10-09T09:20:33.274 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:33.301 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:33.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:33.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:33.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:33.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:33.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:33.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:33.687 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:33.715 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:33.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:33.818 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:33.844 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:33.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:33.959 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:33 smithi094 bash[59677]: cluster 2023-10-09T09:20:32.860129+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:33.959 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:33 smithi094 bash[59677]: cluster 2023-10-09T09:20:32.862055+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-10-09T09:20:33.959 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:33 smithi094 bash[59677]: cluster 2023-10-09T09:20:32.891031+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:33.960 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:33 smithi094 bash[59677]: cluster 2023-10-09T09:20:32.891033+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:33.960 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:33 smithi094 bash[59677]: cluster 2023-10-09T09:20:32.926593+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:33.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:33.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:34.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:34.124 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:34.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:34.148 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[75179]: cluster 2023-10-09T09:20:32.860129+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[75179]: cluster 2023-10-09T09:20:32.862055+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[75179]: cluster 2023-10-09T09:20:32.891031+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[75179]: cluster 2023-10-09T09:20:32.891033+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[75179]: cluster 2023-10-09T09:20:32.926593+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[77892]: cluster 2023-10-09T09:20:32.860129+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[77892]: cluster 2023-10-09T09:20:32.862055+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[77892]: cluster 2023-10-09T09:20:32.891031+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[77892]: cluster 2023-10-09T09:20:32.891033+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:33 smithi016 bash[77892]: cluster 2023-10-09T09:20:32.926593+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T09:20:34.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:20:34.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:34.251 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:34.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:34.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:34.358 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:34.384 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:34.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:34.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:34.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:34.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:34.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:34.621 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:34.650 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:34.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:34.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:34.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:34.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:34.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:34.872 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:34 smithi094 bash[59677]: cluster 2023-10-09T09:20:34.232228+0000 mgr.y (mgr.25421) 1301 : cluster 0 pgmap v1431: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 3430/222069 objects degraded (1.545%); 0 B/s, 63 objects/s recovering 2023-10-09T09:20:34.872 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:34 smithi094 bash[59677]: audit 2023-10-09T09:20:34.258655+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:34.872 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:34 smithi094 bash[59677]: cluster 2023-10-09T09:20:34.833649+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 3430/223308 objects degraded (1.536%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:34.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:34.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:34.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:35.038 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:35.066 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:35.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:35.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:35.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:35.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:34 smithi016 bash[75179]: cluster 2023-10-09T09:20:34.232228+0000 mgr.y (mgr.25421) 1301 : cluster 0 pgmap v1431: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 3430/222069 objects degraded (1.545%); 0 B/s, 63 objects/s recovering 2023-10-09T09:20:35.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:34 smithi016 bash[75179]: audit 2023-10-09T09:20:34.258655+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:35.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:34 smithi016 bash[75179]: cluster 2023-10-09T09:20:34.833649+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 3430/223308 objects degraded (1.536%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:35.187 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:34 smithi016 bash[77892]: cluster 2023-10-09T09:20:34.232228+0000 mgr.y (mgr.25421) 1301 : cluster 0 pgmap v1431: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 3430/222069 objects degraded (1.545%); 0 B/s, 63 objects/s recovering 2023-10-09T09:20:35.187 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:34 smithi016 bash[77892]: audit 2023-10-09T09:20:34.258655+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:35.187 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:34 smithi016 bash[77892]: cluster 2023-10-09T09:20:34.833649+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 3430/223308 objects degraded (1.536%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:35.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:35.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:35.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:35.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:35.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:35.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:35.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:35.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:35.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:35.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:35.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:35.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:35.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:35.729 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:35.739 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:20:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:20:35] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-10-09T09:20:35.757 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:35.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:35.860 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:35.887 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:35.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:35.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:35.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:36.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:36.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:36.153 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:36.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:36.291 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:36.392 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:20:36.395 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:36.402 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:36 smithi094 bash[59677]: audit 2023-10-09T09:20:35.329021+0000 mon.a (mon.0) 1742 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:36.402 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:36 smithi094 bash[59677]: audit 2023-10-09T09:20:36.307515+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.16:0/41684622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:36.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:36.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:36.460 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:20:36.511 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:20:36.525 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:36.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:36.561 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:20:36.598 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:20:36.640 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:20:36.650 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:36.674 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:36.684 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:20:36.726 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:20:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:36 smithi016 bash[75179]: audit 2023-10-09T09:20:35.329021+0000 mon.a (mon.0) 1742 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:36 smithi016 bash[75179]: audit 2023-10-09T09:20:36.307515+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.16:0/41684622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:36.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:36 smithi016 bash[77892]: audit 2023-10-09T09:20:35.329021+0000 mon.a (mon.0) 1742 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:36.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:36 smithi016 bash[77892]: audit 2023-10-09T09:20:36.307515+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.16:0/41684622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:36.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:36.806 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:36.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:36.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:36.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:36.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:36.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:37.049 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:37.075 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:37.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:37.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:37.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:37.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:37.318 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:37.350 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:37.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:37 smithi016 bash[75179]: cluster 2023-10-09T09:20:36.232968+0000 mgr.y (mgr.25421) 1302 : cluster 0 pgmap v1432: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 2748/216786 objects degraded (1.268%); 104 KiB/s, 70 objects/s recovering 2023-10-09T09:20:37.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:37 smithi016 bash[75179]: audit 2023-10-09T09:20:36.970534+0000 mon.a (mon.0) 1743 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:20:37.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:37 smithi016 bash[75179]: audit 2023-10-09T09:20:36.970580+0000 mon.b (mon.1) 33 : audit 1 from='client.? 172.21.15.16:0/217730037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:20:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:37.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:37.355 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:20:37.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:37.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:37 smithi016 bash[77892]: cluster 2023-10-09T09:20:36.232968+0000 mgr.y (mgr.25421) 1302 : cluster 0 pgmap v1432: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 2748/216786 objects degraded (1.268%); 104 KiB/s, 70 objects/s recovering 2023-10-09T09:20:37.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:37 smithi016 bash[77892]: audit 2023-10-09T09:20:36.970534+0000 mon.a (mon.0) 1743 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:20:37.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:37 smithi016 bash[77892]: audit 2023-10-09T09:20:36.970580+0000 mon.b (mon.1) 33 : audit 1 from='client.? 172.21.15.16:0/217730037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:20:37.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:37.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:37.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:37.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:37.594 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:37 smithi094 bash[59677]: cluster 2023-10-09T09:20:36.232968+0000 mgr.y (mgr.25421) 1302 : cluster 0 pgmap v1432: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 2748/216786 objects degraded (1.268%); 104 KiB/s, 70 objects/s recovering 2023-10-09T09:20:37.594 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:37 smithi094 bash[59677]: audit 2023-10-09T09:20:36.970534+0000 mon.a (mon.0) 1743 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:20:37.594 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:37 smithi094 bash[59677]: audit 2023-10-09T09:20:36.970580+0000 mon.b (mon.1) 33 : audit 1 from='client.? 172.21.15.16:0/217730037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:20:37.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:37.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:37.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:37.741 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:37.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:37.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:37.844 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:37.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:37.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:37.974 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:37.999 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:38.102 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:38.125 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:38.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:38.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:38.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:38.349 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:38.375 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:38.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:38.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:38.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[75179]: cluster 2023-10-09T09:20:37.332118+0000 mon.a (mon.0) 1744 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:38.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[75179]: cluster 2023-10-09T09:20:37.336999+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[75179]: audit 2023-10-09T09:20:37.340531+0000 mon.a (mon.0) 1745 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:20:38.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[75179]: cluster 2023-10-09T09:20:37.340781+0000 mon.a (mon.0) 1746 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-10-09T09:20:38.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[75179]: cluster 2023-10-09T09:20:37.354963+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[75179]: cluster 2023-10-09T09:20:37.359372+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[77892]: cluster 2023-10-09T09:20:37.332118+0000 mon.a (mon.0) 1744 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:38.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[77892]: cluster 2023-10-09T09:20:37.336999+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[77892]: audit 2023-10-09T09:20:37.340531+0000 mon.a (mon.0) 1745 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:20:38.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[77892]: cluster 2023-10-09T09:20:37.340781+0000 mon.a (mon.0) 1746 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-10-09T09:20:38.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[77892]: cluster 2023-10-09T09:20:37.354963+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:38 smithi016 bash[77892]: cluster 2023-10-09T09:20:37.359372+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.507 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:38.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:38.610 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:38.618 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:38 smithi094 bash[59677]: cluster 2023-10-09T09:20:37.332118+0000 mon.a (mon.0) 1744 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:38.618 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:38 smithi094 bash[59677]: cluster 2023-10-09T09:20:37.336999+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.618 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:38 smithi094 bash[59677]: audit 2023-10-09T09:20:37.340531+0000 mon.a (mon.0) 1745 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:20:38.618 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:38 smithi094 bash[59677]: cluster 2023-10-09T09:20:37.340781+0000 mon.a (mon.0) 1746 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-10-09T09:20:38.619 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:38 smithi094 bash[59677]: cluster 2023-10-09T09:20:37.354963+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.619 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:38 smithi094 bash[59677]: cluster 2023-10-09T09:20:37.359372+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T09:20:38.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:38.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:38.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:38.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:38.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:38.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:38.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:39.011 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:39.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:39.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:39.139 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:39.166 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:39.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:39.269 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:39.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:39.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:39.350 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:20:39.350 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T09:20:39.351 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:20:39.365 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:39 smithi016 bash[75179]: cluster 2023-10-09T09:20:38.233496+0000 mgr.y (mgr.25421) 1303 : cluster 0 pgmap v1434: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 2748/212676 objects degraded (1.292%); 130 KiB/s, 89 objects/s recovering 2023-10-09T09:20:39.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:39 smithi016 bash[77892]: cluster 2023-10-09T09:20:38.233496+0000 mgr.y (mgr.25421) 1303 : cluster 0 pgmap v1434: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 2748/212676 objects degraded (1.292%); 130 KiB/s, 89 objects/s recovering 2023-10-09T09:20:39.393 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:39 smithi094 bash[59677]: cluster 2023-10-09T09:20:38.233496+0000 mgr.y (mgr.25421) 1303 : cluster 0 pgmap v1434: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 2748/212676 objects degraded (1.292%); 130 KiB/s, 89 objects/s recovering 2023-10-09T09:20:39.451 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:39.484 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:39.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:39.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:39.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:39.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:39.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:39.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:39.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:39.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:39.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:39.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:39.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:40.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:40.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:40.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:40.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:40.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:40.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:40.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:40.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:40.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:40.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:40.482 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:40 smithi094 bash[59677]: cluster 2023-10-09T09:20:39.375287+0000 mon.a (mon.0) 1747 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-10-09T09:20:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:40 smithi094 bash[59677]: cluster 2023-10-09T09:20:39.376267+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:40 smithi094 bash[59677]: cluster 2023-10-09T09:20:39.376424+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:40 smithi094 bash[59677]: cluster 2023-10-09T09:20:39.378096+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:40 smithi094 bash[59677]: cluster 2023-10-09T09:20:39.834903+0000 mon.a (mon.0) 1748 : cluster 3 Health check update: Degraded data redundancy: 2748/212676 objects degraded (1.292%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[75179]: cluster 2023-10-09T09:20:39.375287+0000 mon.a (mon.0) 1747 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-10-09T09:20:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[75179]: cluster 2023-10-09T09:20:39.376267+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[75179]: cluster 2023-10-09T09:20:39.376424+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[75179]: cluster 2023-10-09T09:20:39.378096+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[75179]: cluster 2023-10-09T09:20:39.834903+0000 mon.a (mon.0) 1748 : cluster 3 Health check update: Degraded data redundancy: 2748/212676 objects degraded (1.292%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:40.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[77892]: cluster 2023-10-09T09:20:39.375287+0000 mon.a (mon.0) 1747 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-10-09T09:20:40.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[77892]: cluster 2023-10-09T09:20:39.376267+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[77892]: cluster 2023-10-09T09:20:39.376424+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[77892]: cluster 2023-10-09T09:20:39.378096+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T09:20:40.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:40 smithi016 bash[77892]: cluster 2023-10-09T09:20:39.834903+0000 mon.a (mon.0) 1748 : cluster 3 Health check update: Degraded data redundancy: 2748/212676 objects degraded (1.292%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:40.506 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:40.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:40.609 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:40.636 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:40.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:40.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:40.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:40.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:40.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:40.901 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:40.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:40.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:41.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:41.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:41.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:20:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:41.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:41.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:41.294 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:41.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:41.388 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[75179]: cluster 2023-10-09T09:20:40.234359+0000 mgr.y (mgr.25421) 1304 : cluster 0 pgmap v1436: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 2748/212070 objects degraded (1.296%); 113 KiB/s, 43 objects/s recovering 2023-10-09T09:20:41.388 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[75179]: cluster 2023-10-09T09:20:40.374974+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[75179]: cluster 2023-10-09T09:20:40.375236+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[75179]: cluster 2023-10-09T09:20:40.376507+0000 mon.a (mon.0) 1749 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-10-09T09:20:41.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[75179]: cluster 2023-10-09T09:20:40.382715+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[75179]: cluster 2023-10-09T09:20:40.423814+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.418 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:41.450 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:41.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:41.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:41 smithi094 bash[59677]: cluster 2023-10-09T09:20:40.234359+0000 mgr.y (mgr.25421) 1304 : cluster 0 pgmap v1436: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 2748/212070 objects degraded (1.296%); 113 KiB/s, 43 objects/s recovering 2023-10-09T09:20:41.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:41 smithi094 bash[59677]: cluster 2023-10-09T09:20:40.374974+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:41 smithi094 bash[59677]: cluster 2023-10-09T09:20:40.375236+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.568 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:41 smithi094 bash[59677]: cluster 2023-10-09T09:20:40.376507+0000 mon.a (mon.0) 1749 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-10-09T09:20:41.568 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:41 smithi094 bash[59677]: cluster 2023-10-09T09:20:40.382715+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.568 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:41 smithi094 bash[59677]: cluster 2023-10-09T09:20:40.423814+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:41.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:41.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[77892]: cluster 2023-10-09T09:20:40.234359+0000 mgr.y (mgr.25421) 1304 : cluster 0 pgmap v1436: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 2748/212070 objects degraded (1.296%); 113 KiB/s, 43 objects/s recovering 2023-10-09T09:20:41.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[77892]: cluster 2023-10-09T09:20:40.374974+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[77892]: cluster 2023-10-09T09:20:40.375236+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[77892]: cluster 2023-10-09T09:20:40.376507+0000 mon.a (mon.0) 1749 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-10-09T09:20:41.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[77892]: cluster 2023-10-09T09:20:40.382715+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:41 smithi016 bash[77892]: cluster 2023-10-09T09:20:40.423814+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T09:20:41.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:41.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:41.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:41.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:41.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:20:41.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:41.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:41.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:41.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:42.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:42.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:42.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:42.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:42.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:42.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:42.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:42.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:42.411 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:42.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:42.415 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:20:42.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:42.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:42.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:42.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:42.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:42.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:42.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:42.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:42.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:42.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:42.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:42.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:43.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:43.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:43.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:43.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:43.205 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:43.232 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:43.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:43.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:43.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:43.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:43.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:43.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:43.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:43 smithi094 bash[59677]: cluster 2023-10-09T09:20:42.235169+0000 mgr.y (mgr.25421) 1305 : cluster 0 pgmap v1438: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 2087/205893 objects degraded (1.014%); 149 KiB/s, 4 objects/s recovering 2023-10-09T09:20:43.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:43.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:43.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:43.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:43.634 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:43.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:43.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:43.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:43.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:43 smithi016 bash[75179]: cluster 2023-10-09T09:20:42.235169+0000 mgr.y (mgr.25421) 1305 : cluster 0 pgmap v1438: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 2087/205893 objects degraded (1.014%); 149 KiB/s, 4 objects/s recovering 2023-10-09T09:20:43.747 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:43 smithi016 bash[77892]: cluster 2023-10-09T09:20:42.235169+0000 mgr.y (mgr.25421) 1305 : cluster 0 pgmap v1438: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 2087/205893 objects degraded (1.014%); 149 KiB/s, 4 objects/s recovering 2023-10-09T09:20:43.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:43.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:43.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:43.904 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:43.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:43.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:44.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:44.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:44.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:44.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:44.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:44.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:44.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:44.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:44.405 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:44.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:44.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:44.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:44.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:44.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:44.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:44.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:44.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:44.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:44.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:44.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:44.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:44.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:45.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:45.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:45.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:45.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:45.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:45.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:45.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:45.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:45.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:45.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:45.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:45.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:45.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:45.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:45.643 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:20:45.643 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:20:45.643 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T09:20:45.643 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:20:45.644 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:20:45.644 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T09:20:45.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:45.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:45.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:45 smithi094 bash[59677]: cluster 2023-10-09T09:20:44.235816+0000 mgr.y (mgr.25421) 1306 : cluster 0 pgmap v1439: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 2087/203844 objects degraded (1.024%); 130 KiB/s, 56 objects/s recovering 2023-10-09T09:20:45.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:45 smithi094 bash[59677]: cluster 2023-10-09T09:20:44.836327+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 2087/203844 objects degraded (1.024%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:45.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:45 smithi094 bash[59677]: audit 2023-10-09T09:20:45.347359+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:45.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:45 smithi016 bash[77892]: cluster 2023-10-09T09:20:44.235816+0000 mgr.y (mgr.25421) 1306 : cluster 0 pgmap v1439: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 2087/203844 objects degraded (1.024%); 130 KiB/s, 56 objects/s recovering 2023-10-09T09:20:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:45 smithi016 bash[77892]: cluster 2023-10-09T09:20:44.836327+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 2087/203844 objects degraded (1.024%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:45 smithi016 bash[77892]: audit 2023-10-09T09:20:45.347359+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:45 smithi016 bash[75179]: cluster 2023-10-09T09:20:44.235816+0000 mgr.y (mgr.25421) 1306 : cluster 0 pgmap v1439: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 2087/203844 objects degraded (1.024%); 130 KiB/s, 56 objects/s recovering 2023-10-09T09:20:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:45 smithi016 bash[75179]: cluster 2023-10-09T09:20:44.836327+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 2087/203844 objects degraded (1.024%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:45 smithi016 bash[75179]: audit 2023-10-09T09:20:45.347359+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:20:45.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:45.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:45.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:45.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:45.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:45.927 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:45.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:45.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:46.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:20:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:20:45] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-10-09T09:20:46.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:46.057 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:46.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:46.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:46.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:46.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:46.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:46.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:46.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:46.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:46.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:46.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:46.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:46.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:46.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:46.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:46.728 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:46.755 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:46.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:46.794 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:20:46.795 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:20:46.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:46.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:46.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:46.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:46.997 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:47.024 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:47.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:47.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:47.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:47.293 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:47.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:47.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:47.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:47.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:47.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:47.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:47.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:47.689 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:47 smithi016 bash[75179]: cluster 2023-10-09T09:20:46.236514+0000 mgr.y (mgr.25421) 1307 : cluster 0 pgmap v1440: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 2087/198636 objects degraded (1.051%); 112 KiB/s, 63 objects/s recovering 2023-10-09T09:20:47.689 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:47 smithi016 bash[77892]: cluster 2023-10-09T09:20:46.236514+0000 mgr.y (mgr.25421) 1307 : cluster 0 pgmap v1440: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 2087/198636 objects degraded (1.051%); 112 KiB/s, 63 objects/s recovering 2023-10-09T09:20:47.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:47.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:47 smithi094 bash[59677]: cluster 2023-10-09T09:20:46.236514+0000 mgr.y (mgr.25421) 1307 : cluster 0 pgmap v1440: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 2087/198636 objects degraded (1.051%); 112 KiB/s, 63 objects/s recovering 2023-10-09T09:20:47.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:47.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:47.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:47.957 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:47.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:47.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:48.062 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:48.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:48.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:48.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:48.190 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:48.212 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:48.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:48.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:48.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:48.341 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:48.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:48.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:48.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:48.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:48.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:48.587 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:48.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:48.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:48.725 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:48.754 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:48.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:48.862 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:48.888 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:48.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:48.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:49.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:49.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:49.166 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:49.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:49.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:49.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:49 smithi094 bash[59677]: cluster 2023-10-09T09:20:48.237222+0000 mgr.y (mgr.25421) 1308 : cluster 0 pgmap v1441: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 2087/193731 objects degraded (1.077%); 101 KiB/s, 56 objects/s recovering 2023-10-09T09:20:49.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:49 smithi094 bash[59677]: audit 2023-10-09T09:20:49.259049+0000 mon.a (mon.0) 1752 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:49.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:49.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:49.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:49.550 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:49 smithi016 bash[75179]: cluster 2023-10-09T09:20:48.237222+0000 mgr.y (mgr.25421) 1308 : cluster 0 pgmap v1441: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 2087/193731 objects degraded (1.077%); 101 KiB/s, 56 objects/s recovering 2023-10-09T09:20:49.550 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:49 smithi016 bash[75179]: audit 2023-10-09T09:20:49.259049+0000 mon.a (mon.0) 1752 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:49.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:49 smithi016 bash[77892]: cluster 2023-10-09T09:20:48.237222+0000 mgr.y (mgr.25421) 1308 : cluster 0 pgmap v1441: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 2087/193731 objects degraded (1.077%); 101 KiB/s, 56 objects/s recovering 2023-10-09T09:20:49.550 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:49 smithi016 bash[77892]: audit 2023-10-09T09:20:49.259049+0000 mon.a (mon.0) 1752 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:49.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:49.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:49.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:49.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:49.850 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:49.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:49.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:49.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:50.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:50.119 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:50.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:50.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:50.213 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:20:50.213 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:20:50.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:50.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:50.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:50.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:50.358 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:50.397 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:50.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:50.480 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:20:50.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:50 smithi016 bash[75179]: cluster 2023-10-09T09:20:49.837508+0000 mon.a (mon.0) 1753 : cluster 3 Health check update: Degraded data redundancy: 2087/193731 objects degraded (1.077%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:50.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:50 smithi016 bash[77892]: cluster 2023-10-09T09:20:49.837508+0000 mon.a (mon.0) 1753 : cluster 3 Health check update: Degraded data redundancy: 2087/193731 objects degraded (1.077%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:50.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:50.529 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:50.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:50.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:50.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:50.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:50.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:50.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:50.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:50 smithi094 bash[59677]: cluster 2023-10-09T09:20:49.837508+0000 mon.a (mon.0) 1753 : cluster 3 Health check update: Degraded data redundancy: 2087/193731 objects degraded (1.077%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:50.762 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:50.785 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:50.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:50.892 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:50.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:50.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:51.028 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:51.059 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:51.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:51.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:51.187 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:51.293 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:51.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:51.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:51.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:51.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:51.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:51.569 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:51 smithi094 bash[59677]: cluster 2023-10-09T09:20:50.238101+0000 mgr.y (mgr.25421) 1309 : cluster 0 pgmap v1442: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 2087/192324 objects degraded (1.085%); 90 KiB/s, 50 objects/s recovering 2023-10-09T09:20:51.584 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:51.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:51.591 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:51 smithi016 bash[75179]: cluster 2023-10-09T09:20:50.238101+0000 mgr.y (mgr.25421) 1309 : cluster 0 pgmap v1442: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 2087/192324 objects degraded (1.085%); 90 KiB/s, 50 objects/s recovering 2023-10-09T09:20:51.592 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:51 smithi016 bash[77892]: cluster 2023-10-09T09:20:50.238101+0000 mgr.y (mgr.25421) 1309 : cluster 0 pgmap v1442: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 2087/192324 objects degraded (1.085%); 90 KiB/s, 50 objects/s recovering 2023-10-09T09:20:51.691 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:51.721 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:51.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:51.852 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:51.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:51.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:51.983 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:51.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:52.086 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:52.117 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:52.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:52.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:52.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:52.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:52.367 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:52.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:52.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:52.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:52.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:52.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:52.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:52.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:52.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:52.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:52.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:52.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:52.787 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:52.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:52.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:52.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:52.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:53.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:53.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:53.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:53.187 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:53.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:20:53.387 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:53.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:53.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:53.559 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:53.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:53.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:53.680 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:53 smithi016 bash[75179]: cluster 2023-10-09T09:20:52.240913+0000 mgr.y (mgr.25421) 1310 : cluster 0 pgmap v1443: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 2087/184416 objects degraded (1.132%); 76 KiB/s, 52 objects/s recovering 2023-10-09T09:20:53.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:53 smithi016 bash[77892]: cluster 2023-10-09T09:20:52.240913+0000 mgr.y (mgr.25421) 1310 : cluster 0 pgmap v1443: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 2087/184416 objects degraded (1.132%); 76 KiB/s, 52 objects/s recovering 2023-10-09T09:20:53.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:53.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:53 smithi094 bash[59677]: cluster 2023-10-09T09:20:52.240913+0000 mgr.y (mgr.25421) 1310 : cluster 0 pgmap v1443: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 2087/184416 objects degraded (1.132%); 76 KiB/s, 52 objects/s recovering 2023-10-09T09:20:53.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:53.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:53.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:53.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:53.984 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:53.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:54.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:54.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:54.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:54.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:54.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:54.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:54.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:54.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:54.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:54.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:54.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:54.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:54.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:54.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:54.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:54.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:54.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:54.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:54.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:54.825 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:54.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:54.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:54.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:54.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:54.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:55.068 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:55.098 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:55.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:55.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:55.225 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:55.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:55.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:55.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:55.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:55.431 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:55 smithi094 bash[59677]: cluster 2023-10-09T09:20:54.241491+0000 mgr.y (mgr.25421) 1311 : cluster 0 pgmap v1444: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 2087/182451 objects degraded (1.144%); 0 B/s, 49 objects/s recovering 2023-10-09T09:20:55.431 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:55 smithi094 bash[59677]: cluster 2023-10-09T09:20:54.838664+0000 mon.a (mon.0) 1754 : cluster 3 Health check update: Degraded data redundancy: 2087/182451 objects degraded (1.144%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:55.469 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:55.496 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:55.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:55.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:55.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:55.632 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:55.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:55.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:55.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:55.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:55 smithi016 bash[77892]: cluster 2023-10-09T09:20:54.241491+0000 mgr.y (mgr.25421) 1311 : cluster 0 pgmap v1444: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 2087/182451 objects degraded (1.144%); 0 B/s, 49 objects/s recovering 2023-10-09T09:20:55.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:55 smithi016 bash[77892]: cluster 2023-10-09T09:20:54.838664+0000 mon.a (mon.0) 1754 : cluster 3 Health check update: Degraded data redundancy: 2087/182451 objects degraded (1.144%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:55.739 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:20:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:20:55] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.33.4" 2023-10-09T09:20:55.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:55 smithi016 bash[75179]: cluster 2023-10-09T09:20:54.241491+0000 mgr.y (mgr.25421) 1311 : cluster 0 pgmap v1444: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 2087/182451 objects degraded (1.144%); 0 B/s, 49 objects/s recovering 2023-10-09T09:20:55.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:55 smithi016 bash[75179]: cluster 2023-10-09T09:20:54.838664+0000 mon.a (mon.0) 1754 : cluster 3 Health check update: Degraded data redundancy: 2087/182451 objects degraded (1.144%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:20:55.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:55.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:55.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:55.894 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:55.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:55.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:56.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:56.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:56.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:56.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:56.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:56.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:56.276 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:20:56.276 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T09:20:56.276 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:20:56.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:20:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:56.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:20:56.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:56.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:56.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:56.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:56.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:56.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:56.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:56.620 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:20:56.665 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:20:56.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:56.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:56.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:56.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:56.755 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:20:56.819 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:20:56.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:56.858 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:20:56.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:56.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:56.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:56.907 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:20:56.953 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:20:56.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:56.999 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:20:57.009 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.009 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.009 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.010 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.010 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.010 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.010 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.010 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:20:57.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:57.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:57.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:57.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:57.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:57.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:57.291 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:57.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:57.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:57.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:57.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:57 smithi094 bash[59677]: cluster 2023-10-09T09:20:56.242160+0000 mgr.y (mgr.25421) 1312 : cluster 0 pgmap v1445: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2087/177630 objects degraded (1.175%); 0 B/s, 28 objects/s recovering 2023-10-09T09:20:57.437 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:57 smithi094 bash[59677]: audit 2023-10-09T09:20:56.568534+0000 mon.a (mon.0) 1755 : audit 0 from='client.? 172.21.15.16:0/3091696232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:57.437 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:57 smithi094 bash[59677]: audit 2023-10-09T09:20:57.029005+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.16:0/72883070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:20:57.451 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:20:57.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:57.568 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:57.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:57.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:57.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:57.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:57 smithi016 bash[75179]: cluster 2023-10-09T09:20:56.242160+0000 mgr.y (mgr.25421) 1312 : cluster 0 pgmap v1445: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2087/177630 objects degraded (1.175%); 0 B/s, 28 objects/s recovering 2023-10-09T09:20:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:57 smithi016 bash[75179]: audit 2023-10-09T09:20:56.568534+0000 mon.a (mon.0) 1755 : audit 0 from='client.? 172.21.15.16:0/3091696232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:57 smithi016 bash[75179]: audit 2023-10-09T09:20:57.029005+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.16:0/72883070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:20:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:57 smithi016 bash[77892]: cluster 2023-10-09T09:20:56.242160+0000 mgr.y (mgr.25421) 1312 : cluster 0 pgmap v1445: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2087/177630 objects degraded (1.175%); 0 B/s, 28 objects/s recovering 2023-10-09T09:20:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:57 smithi016 bash[77892]: audit 2023-10-09T09:20:56.568534+0000 mon.a (mon.0) 1755 : audit 0 from='client.? 172.21.15.16:0/3091696232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:20:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:57 smithi016 bash[77892]: audit 2023-10-09T09:20:57.029005+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.16:0/72883070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:20:57.809 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:57.831 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:57.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:57.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:57.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:58.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:58.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:20:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:58.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:20:58.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:58.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:58.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:58.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:58.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:58.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:58.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:58.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:58.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[75179]: cluster 2023-10-09T09:20:57.430781+0000 mon.a (mon.0) 1757 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[75179]: cluster 2023-10-09T09:20:57.441800+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[75179]: audit 2023-10-09T09:20:57.444373+0000 mon.a (mon.0) 1758 : audit 1 from='client.? 172.21.15.16:0/72883070' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[75179]: cluster 2023-10-09T09:20:57.444460+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[75179]: cluster 2023-10-09T09:20:57.456893+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[77892]: cluster 2023-10-09T09:20:57.430781+0000 mon.a (mon.0) 1757 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[77892]: cluster 2023-10-09T09:20:57.441800+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[77892]: audit 2023-10-09T09:20:57.444373+0000 mon.a (mon.0) 1758 : audit 1 from='client.? 172.21.15.16:0/72883070' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[77892]: cluster 2023-10-09T09:20:57.444460+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-10-09T09:20:58.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:58 smithi016 bash[77892]: cluster 2023-10-09T09:20:57.456893+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T09:20:58.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:58.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:58.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:58.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:58.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:58.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:58.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:58.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:58 smithi094 bash[59677]: cluster 2023-10-09T09:20:57.430781+0000 mon.a (mon.0) 1757 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:20:58.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:58 smithi094 bash[59677]: cluster 2023-10-09T09:20:57.441800+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T09:20:58.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:58 smithi094 bash[59677]: audit 2023-10-09T09:20:57.444373+0000 mon.a (mon.0) 1758 : audit 1 from='client.? 172.21.15.16:0/72883070' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:20:58.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:58 smithi094 bash[59677]: cluster 2023-10-09T09:20:57.444460+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-10-09T09:20:58.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:58 smithi094 bash[59677]: cluster 2023-10-09T09:20:57.456893+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T09:20:58.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:58.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:20:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:58.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:20:58.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:58.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:58.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:58.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:59.009 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:59.044 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:20:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:59.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:20:59.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:59.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:59.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:59.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:59.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:59.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:20:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:59.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:20:59.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:59.450 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:20:59 smithi094 bash[59677]: cluster 2023-10-09T09:20:58.242735+0000 mgr.y (mgr.25421) 1313 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 2087/173235 objects degraded (1.205%); 0 B/s, 22 objects/s recovering 2023-10-09T09:20:59.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:20:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:59.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:20:59.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:59.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:20:59.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:59.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:20:59.709 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:59.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:59.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:59.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:20:59 smithi016 bash[75179]: cluster 2023-10-09T09:20:58.242735+0000 mgr.y (mgr.25421) 1313 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 2087/173235 objects degraded (1.205%); 0 B/s, 22 objects/s recovering 2023-10-09T09:20:59.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:20:59 smithi016 bash[77892]: cluster 2023-10-09T09:20:58.242735+0000 mgr.y (mgr.25421) 1313 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 2087/173235 objects degraded (1.205%); 0 B/s, 22 objects/s recovering 2023-10-09T09:20:59.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:59.890 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:20:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:20:59.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:20:59.996 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:00.023 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:00.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:00.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:00.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:00.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:00.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:00.267 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:00.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:00.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:00.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:00.442 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:00.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:00.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:00.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:00.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:00.694 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:00.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:00.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:00.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:00.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:00.858 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:00.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:00.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:00.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:00.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:00 smithi094 bash[59677]: cluster 2023-10-09T09:20:59.839489+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: Degraded data redundancy: 2087/173235 objects degraded (1.205%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:00.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:00 smithi094 bash[59677]: cluster 2023-10-09T09:21:00.243391+0000 mgr.y (mgr.25421) 1314 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 2087/171954 objects degraded (1.214%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:01.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:01.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:01.132 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:00 smithi016 bash[75179]: cluster 2023-10-09T09:20:59.839489+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: Degraded data redundancy: 2087/173235 objects degraded (1.205%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:01.132 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:00 smithi016 bash[75179]: cluster 2023-10-09T09:21:00.243391+0000 mgr.y (mgr.25421) 1314 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 2087/171954 objects degraded (1.214%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:01.133 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:00 smithi016 bash[77892]: cluster 2023-10-09T09:20:59.839489+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: Degraded data redundancy: 2087/173235 objects degraded (1.205%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:01.133 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:00 smithi016 bash[77892]: cluster 2023-10-09T09:21:00.243391+0000 mgr.y (mgr.25421) 1314 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 2087/171954 objects degraded (1.214%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:01.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:01.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:01.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:01.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:01.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:01.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:01.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:01.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:21:01.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:01.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:01.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:01.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:01.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:01.858 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:01.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:01.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:01.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:01.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:01.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:02.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:02.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:02.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:02.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:02.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:02.264 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:02.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:02.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:02.371 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:02.377 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:21:02.377 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:21:02.377 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T09:21:02.378 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:21:02.378 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:21:02.378 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T09:21:02.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:21:02.394 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:02.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:02.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:02.498 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:02.532 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:02.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:02.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:02.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:02.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:02.784 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:02.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:02.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:02.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:02.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:02.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:02.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:03.064 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:03.087 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:03.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:03.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:03.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:03.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:03.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:03.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:03.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:03.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:03.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:03.487 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:03 smithi094 bash[59677]: cluster 2023-10-09T09:21:02.244108+0000 mgr.y (mgr.25421) 1315 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 2087/164256 objects degraded (1.271%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:03.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:03.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:03.525 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:03 smithi016 bash[75179]: cluster 2023-10-09T09:21:02.244108+0000 mgr.y (mgr.25421) 1315 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 2087/164256 objects degraded (1.271%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:03.525 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:03 smithi016 bash[77892]: cluster 2023-10-09T09:21:02.244108+0000 mgr.y (mgr.25421) 1315 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 2087/164256 objects degraded (1.271%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:03.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:03.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:03.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:03.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:03.741 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:03.768 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:03.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:03.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:03.873 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:03.916 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:03.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:04.022 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:04.048 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:04.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:04.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:04.156 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:04.178 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:04.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:04.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:04.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:04.315 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:04 smithi094 bash[59677]: audit 2023-10-09T09:21:04.259699+0000 mon.a (mon.0) 1761 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:04.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:04.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:04.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:04.425 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:04.448 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:04.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:04.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:04.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:04.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:04 smithi016 bash[75179]: audit 2023-10-09T09:21:04.259699+0000 mon.a (mon.0) 1761 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:04.560 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:04 smithi016 bash[77892]: audit 2023-10-09T09:21:04.259699+0000 mon.a (mon.0) 1761 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:04.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:04.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:04.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:04.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:04.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:04.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:04.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:04.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:04.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:04.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:04.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:05.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:05.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:05.126 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:05.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:05.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:05.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:05.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:05.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:05.400 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:05 smithi094 bash[59677]: cluster 2023-10-09T09:21:04.244577+0000 mgr.y (mgr.25421) 1316 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2087/162261 objects degraded (1.286%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:05.400 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:05 smithi094 bash[59677]: cluster 2023-10-09T09:21:04.841124+0000 mon.a (mon.0) 1762 : cluster 3 Health check update: Degraded data redundancy: 2087/162261 objects degraded (1.286%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.421 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:05.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:05.522 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:05.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:05.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:05.655 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:05.698 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:05.737 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:05 smithi016 bash[75179]: cluster 2023-10-09T09:21:04.244577+0000 mgr.y (mgr.25421) 1316 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2087/162261 objects degraded (1.286%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:05.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:05 smithi016 bash[75179]: cluster 2023-10-09T09:21:04.841124+0000 mon.a (mon.0) 1762 : cluster 3 Health check update: Degraded data redundancy: 2087/162261 objects degraded (1.286%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:05.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:05 smithi016 bash[77892]: cluster 2023-10-09T09:21:04.244577+0000 mgr.y (mgr.25421) 1316 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2087/162261 objects degraded (1.286%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:05.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:05 smithi016 bash[77892]: cluster 2023-10-09T09:21:04.841124+0000 mon.a (mon.0) 1762 : cluster 3 Health check update: Degraded data redundancy: 2087/162261 objects degraded (1.286%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:05.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:05.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:05.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:05.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:05.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:05.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:05.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:21:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:21:05] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T09:21:06.091 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:06.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:06.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:06.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:06.251 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:06.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:06.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:06.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:06.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:06.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:06.421 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:135696 11.e e314/294 11:74c6ef0e:::benchmark_data_smithi016_218422_object42558:head v 314'8522, mlcod=314'8520)", 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:06.413248+0000", 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0062000190000000002, 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0062327210000000001, 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.422 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413248+0000", 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413248+0000", 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:06.423 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413248+0000", 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413256+0000", 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1149999999999994e-06 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413266+0000", 2023-10-09T09:21:06.424 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.008e-05 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413273+0000", 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8020000000000003e-06 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413376+0000", 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000103058 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.425 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.413395+0000", 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9142000000000001e-05 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:135699 11.c 11:3ce0ac1f:::benchmark_data_smithi016_218422_object42561:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:06.416183+0000", 2023-10-09T09:21:06.426 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0032658969999999998, 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0033739719999999998, 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 135699 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:06.427 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416183+0000", 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416183+0000", 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416181+0000", 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:06.428 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416188+0000", 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7039999999999997e-06 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416202+0000", 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3991999999999999e-05 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:06.429 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416211+0000", 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0739999999999999e-06 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416463+0000", 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025267099999999999 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416931+0000", 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00046771699999999998 2023-10-09T09:21:06.430 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.417077+0000", 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014544300000000001 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:135693 11.6 e314/307 11:6757c792:::benchmark_data_smithi016_218422_object42555:head v 314'8500, mlcod=302'5824)", 2023-10-09T09:21:06.431 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:06.416555+0000", 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.002893465, 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0030822789999999998, 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416555+0000", 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.432 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416555+0000", 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416554+0000", 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416561+0000", 2023-10-09T09:21:06.433 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4150000000000004e-06 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416570+0000", 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3880000000000006e-06 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.416577+0000", 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4520000000000002e-06 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.434 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.417920+0000", 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001343445 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:06.417943+0000", 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3377999999999999e-05 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:06.435 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:06.436 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-10-09T09:21:06.436 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 3 2023-10-09T09:21:06.436 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:21:06.505 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:06.545 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:06.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:06.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:06.649 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:06.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:06.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:06.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:06.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:06.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:06.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:06.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:06.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:06.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:07.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:07.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:07.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:07.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:07.209 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:21:07.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:07.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:07.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:07.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:07.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:07.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:07.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:07.399 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:07 smithi016 bash[77892]: cluster 2023-10-09T09:21:06.245242+0000 mgr.y (mgr.25421) 1317 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 2087/157014 objects degraded (1.329%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:07.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:07.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:07 smithi094 bash[59677]: cluster 2023-10-09T09:21:06.245242+0000 mgr.y (mgr.25421) 1317 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 2087/157014 objects degraded (1.329%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:07.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:07.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:07.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:07.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:07.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:07.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:07.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:07.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:07 smithi016 bash[75179]: cluster 2023-10-09T09:21:06.245242+0000 mgr.y (mgr.25421) 1317 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 2087/157014 objects degraded (1.329%); 0 B/s, 22 objects/s recovering 2023-10-09T09:21:07.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:07.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:07.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:07.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:07.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:07.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:07.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:21:08.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:08.051 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:08.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:08.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:08.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:08.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:08.394 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:08.396 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:08.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:08.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:08.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:08.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:08.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:08.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:08.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:08.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:08.992 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:08.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:09.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:09.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:09.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:09.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:09.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:09.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:09.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:09.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:09.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:09.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:09.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:09.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[77892]: cluster 2023-10-09T09:21:08.246035+0000 mgr.y (mgr.25421) 1318 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 2087/152412 objects degraded (1.369%); 0 B/s, 21 objects/s recovering 2023-10-09T09:21:09.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[77892]: cluster 2023-10-09T09:21:08.333251+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[77892]: cluster 2023-10-09T09:21:08.336309+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-10-09T09:21:09.487 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[77892]: cluster 2023-10-09T09:21:08.337132+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.487 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[77892]: cluster 2023-10-09T09:21:08.340989+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[75179]: cluster 2023-10-09T09:21:08.246035+0000 mgr.y (mgr.25421) 1318 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 2087/152412 objects degraded (1.369%); 0 B/s, 21 objects/s recovering 2023-10-09T09:21:09.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[75179]: cluster 2023-10-09T09:21:08.333251+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[75179]: cluster 2023-10-09T09:21:08.336309+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-10-09T09:21:09.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[75179]: cluster 2023-10-09T09:21:08.337132+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.487 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:09 smithi016 bash[75179]: cluster 2023-10-09T09:21:08.340989+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:09.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:09.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:09.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:09.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:09.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:09.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:09 smithi094 bash[59677]: cluster 2023-10-09T09:21:08.246035+0000 mgr.y (mgr.25421) 1318 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 2087/152412 objects degraded (1.369%); 0 B/s, 21 objects/s recovering 2023-10-09T09:21:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:09 smithi094 bash[59677]: cluster 2023-10-09T09:21:08.333251+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:09 smithi094 bash[59677]: cluster 2023-10-09T09:21:08.336309+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-10-09T09:21:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:09 smithi094 bash[59677]: cluster 2023-10-09T09:21:08.337132+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:09 smithi094 bash[59677]: cluster 2023-10-09T09:21:08.340989+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T09:21:09.748 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:09.775 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:09.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:09.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:09.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:09.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:09.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:09.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:10.017 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:10.038 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:10.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:10.144 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.169 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:10.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:10.272 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:10.302 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:10.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:10.406 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.432 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:10.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:10.535 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:10 smithi094 bash[59677]: cluster 2023-10-09T09:21:09.341690+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T09:21:10.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:10 smithi094 bash[59677]: cluster 2023-10-09T09:21:09.343692+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-10-09T09:21:10.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:10 smithi094 bash[59677]: cluster 2023-10-09T09:21:09.344680+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T09:21:10.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:10 smithi094 bash[59677]: cluster 2023-10-09T09:21:09.846337+0000 mon.a (mon.0) 1765 : cluster 3 Health check update: Degraded data redundancy: 2087/152412 objects degraded (1.369%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:10.561 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:10.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:10.663 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.698 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:10.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:10.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[75179]: cluster 2023-10-09T09:21:09.341690+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T09:21:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[75179]: cluster 2023-10-09T09:21:09.343692+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-10-09T09:21:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[75179]: cluster 2023-10-09T09:21:09.344680+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T09:21:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[75179]: cluster 2023-10-09T09:21:09.846337+0000 mon.a (mon.0) 1765 : cluster 3 Health check update: Degraded data redundancy: 2087/152412 objects degraded (1.369%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[77892]: cluster 2023-10-09T09:21:09.341690+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T09:21:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[77892]: cluster 2023-10-09T09:21:09.343692+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-10-09T09:21:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[77892]: cluster 2023-10-09T09:21:09.344680+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T09:21:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:10 smithi016 bash[77892]: cluster 2023-10-09T09:21:09.846337+0000 mon.a (mon.0) 1765 : cluster 3 Health check update: Degraded data redundancy: 2087/152412 objects degraded (1.369%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:10.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:10.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:10.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:10.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:11.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:11.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:11.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:11.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:11.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:11.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:11.406 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:11 smithi016 bash[75179]: cluster 2023-10-09T09:21:10.247754+0000 mgr.y (mgr.25421) 1319 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 2087/151080 objects degraded (1.381%); 0 B/s, 14 objects/s recovering 2023-10-09T09:21:11.407 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:11 smithi016 bash[77892]: cluster 2023-10-09T09:21:10.247754+0000 mgr.y (mgr.25421) 1319 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 2087/151080 objects degraded (1.381%); 0 B/s, 14 objects/s recovering 2023-10-09T09:21:11.423 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:11.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:11.559 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:11.590 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:11.694 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:11.703 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:11 smithi094 bash[59677]: cluster 2023-10-09T09:21:10.247754+0000 mgr.y (mgr.25421) 1319 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 2087/151080 objects degraded (1.381%); 0 B/s, 14 objects/s recovering 2023-10-09T09:21:11.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:11.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:11.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:11.845 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:11.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:11.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:11.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:12.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:12.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:12.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:12.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:12.234 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:12.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:12.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:12.359 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:12.385 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:12.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:12.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:12.488 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:12.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:12.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:12.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:12.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:12.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:12.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:12.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:12.758 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:12.782 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:12.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:12.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:12.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:12.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:12.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:13.025 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:13.053 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:13.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:13.141 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:21:13.141 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T09:21:13.141 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:21:13.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:13.194 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:13.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:13.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:13.298 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:13.324 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:13.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:13.433 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:13.462 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:13.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:13.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:13.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:13.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:13.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:13.698 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:13.707 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:13 smithi094 bash[59677]: cluster 2023-10-09T09:21:12.248415+0000 mgr.y (mgr.25421) 1320 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 1343/144117 objects degraded (0.932%); 144 KiB/s, 59 objects/s recovering 2023-10-09T09:21:13.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:13.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:13.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:13 smithi016 bash[75179]: cluster 2023-10-09T09:21:12.248415+0000 mgr.y (mgr.25421) 1320 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 1343/144117 objects degraded (0.932%); 144 KiB/s, 59 objects/s recovering 2023-10-09T09:21:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:13 smithi016 bash[77892]: cluster 2023-10-09T09:21:12.248415+0000 mgr.y (mgr.25421) 1320 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 1343/144117 objects degraded (0.932%); 144 KiB/s, 59 objects/s recovering 2023-10-09T09:21:13.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:13.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:13.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:13.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:13.904 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:13.990 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:14.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:14.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:14.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:14.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:14.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:14.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:14.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:14.381 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:14.409 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:14.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:14.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:14.658 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:14.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:14.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:14.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:14.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:14.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:14.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:14.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:15.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:15.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:15.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:15.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:15.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:15.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:15.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:15.355 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:15.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:15.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:15.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:15.491 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:15.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:15 smithi016 bash[75179]: cluster 2023-10-09T09:21:14.249137+0000 mgr.y (mgr.25421) 1321 : cluster 0 pgmap v1457: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 1343/142587 objects degraded (0.942%); 144 KiB/s, 44 objects/s recovering 2023-10-09T09:21:15.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:15 smithi016 bash[75179]: cluster 2023-10-09T09:21:14.847383+0000 mon.a (mon.0) 1766 : cluster 3 Health check update: Degraded data redundancy: 1343/142587 objects degraded (0.942%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:15 smithi016 bash[77892]: cluster 2023-10-09T09:21:14.249137+0000 mgr.y (mgr.25421) 1321 : cluster 0 pgmap v1457: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 1343/142587 objects degraded (0.942%); 144 KiB/s, 44 objects/s recovering 2023-10-09T09:21:15.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:15 smithi016 bash[77892]: cluster 2023-10-09T09:21:14.847383+0000 mon.a (mon.0) 1766 : cluster 3 Health check update: Degraded data redundancy: 1343/142587 objects degraded (0.942%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:15.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:15.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:15.624 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:15.634 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:15 smithi094 bash[59677]: cluster 2023-10-09T09:21:14.249137+0000 mgr.y (mgr.25421) 1321 : cluster 0 pgmap v1457: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 1343/142587 objects degraded (0.942%); 144 KiB/s, 44 objects/s recovering 2023-10-09T09:21:15.634 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:15 smithi094 bash[59677]: cluster 2023-10-09T09:21:14.847383+0000 mon.a (mon.0) 1766 : cluster 3 Health check update: Degraded data redundancy: 1343/142587 objects degraded (0.942%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:15.650 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:15.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:15.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:15.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:15.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:15.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:15.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:15.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:15.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:15.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:16.004 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:21:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:21:15] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T09:21:16.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:16.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:16.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:16.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:16.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:16.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:16.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:16.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:16.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:16.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:16.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:16.464 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:16.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:16.568 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:16.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:16.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:16.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:16.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:16.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:16.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:16.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:16.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:16.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:16.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:16.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:17.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.029 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:17.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:17.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:17.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:17.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:17.289 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:21:17.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:17.356 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:21:17.409 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:21:17.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:17 smithi016 bash[75179]: cluster 2023-10-09T09:21:16.249835+0000 mgr.y (mgr.25421) 1322 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 1343/137592 objects degraded (0.976%); 144 KiB/s, 44 objects/s recovering 2023-10-09T09:21:17.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:17 smithi016 bash[75179]: audit 2023-10-09T09:21:16.365630+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:21:17.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:17 smithi016 bash[75179]: audit 2023-10-09T09:21:17.197816+0000 mon.a (mon.0) 1768 : audit 0 from='client.? 172.21.15.16:0/49976070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:21:17.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:17 smithi016 bash[77892]: cluster 2023-10-09T09:21:16.249835+0000 mgr.y (mgr.25421) 1322 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 1343/137592 objects degraded (0.976%); 144 KiB/s, 44 objects/s recovering 2023-10-09T09:21:17.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:17 smithi016 bash[77892]: audit 2023-10-09T09:21:16.365630+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:21:17.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:17 smithi016 bash[77892]: audit 2023-10-09T09:21:17.197816+0000 mon.a (mon.0) 1768 : audit 0 from='client.? 172.21.15.16:0/49976070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:21:17.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.441 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:17.455 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:21:17.498 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:21:17.538 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:21:17.547 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:17.582 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:21:17.618 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:21:17.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.622 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.623 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.623 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.623 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.623 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.623 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:21:17.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:17.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:17.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:17 smithi094 bash[59677]: cluster 2023-10-09T09:21:16.249835+0000 mgr.y (mgr.25421) 1322 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 1343/137592 objects degraded (0.976%); 144 KiB/s, 44 objects/s recovering 2023-10-09T09:21:17.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:17 smithi094 bash[59677]: audit 2023-10-09T09:21:16.365630+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:21:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:17 smithi094 bash[59677]: audit 2023-10-09T09:21:17.197816+0000 mon.a (mon.0) 1768 : audit 0 from='client.? 172.21.15.16:0/49976070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:21:17.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:17.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:17.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:17.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:17.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:17.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:18.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:18.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:18.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:18.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:18.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:18.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:18.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:18.360 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:18 smithi016 bash[75179]: audit 2023-10-09T09:21:17.629276+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.16:0/3960853573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:21:18.360 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:18 smithi016 bash[75179]: audit 2023-10-09T09:21:17.629991+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:21:18.382 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:21:18.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:18.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:18.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:18.503 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:18.532 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:18.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:18.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:18.648 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:18 smithi016 bash[77892]: audit 2023-10-09T09:21:17.629276+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.16:0/3960853573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:21:18.648 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:18 smithi016 bash[77892]: audit 2023-10-09T09:21:17.629991+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:21:18.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:18.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:18 smithi094 bash[59677]: audit 2023-10-09T09:21:17.629276+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.16:0/3960853573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:21:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:18 smithi094 bash[59677]: audit 2023-10-09T09:21:17.629991+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:21:18.773 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:18.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:18.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:18.908 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:18.931 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:18.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:19.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:19.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:19.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:19.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:19.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:19.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:19.245 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:21:19.245 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:21:19.246 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T09:21:19.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:21:19.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:19.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:19.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:19.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:19.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:19.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:19.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:19.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: cluster 2023-10-09T09:21:18.250506+0000 mgr.y (mgr.25421) 1323 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1343/133251 objects degraded (1.008%); 116 KiB/s, 57 objects/s recovering 2023-10-09T09:21:19.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: cluster 2023-10-09T09:21:18.357453+0000 mon.a (mon.0) 1770 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:21:19.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: cluster 2023-10-09T09:21:18.365215+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: cluster 2023-10-09T09:21:18.366037+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: audit 2023-10-09T09:21:18.369875+0000 mon.a (mon.0) 1771 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: cluster 2023-10-09T09:21:18.369943+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: audit 2023-10-09T09:21:19.272259+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[75179]: audit 2023-10-09T09:21:19.279192+0000 mon.a (mon.0) 1774 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: cluster 2023-10-09T09:21:18.250506+0000 mgr.y (mgr.25421) 1323 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1343/133251 objects degraded (1.008%); 116 KiB/s, 57 objects/s recovering 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: cluster 2023-10-09T09:21:18.357453+0000 mon.a (mon.0) 1770 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: cluster 2023-10-09T09:21:18.365215+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T09:21:19.603 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: cluster 2023-10-09T09:21:18.366037+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T09:21:19.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: audit 2023-10-09T09:21:18.369875+0000 mon.a (mon.0) 1771 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:21:19.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: cluster 2023-10-09T09:21:18.369943+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-10-09T09:21:19.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: audit 2023-10-09T09:21:19.272259+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:19.604 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:19 smithi016 bash[77892]: audit 2023-10-09T09:21:19.279192+0000 mon.a (mon.0) 1774 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:19.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:19.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:19.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:19.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:19.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: cluster 2023-10-09T09:21:18.250506+0000 mgr.y (mgr.25421) 1323 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1343/133251 objects degraded (1.008%); 116 KiB/s, 57 objects/s recovering 2023-10-09T09:21:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: cluster 2023-10-09T09:21:18.357453+0000 mon.a (mon.0) 1770 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:21:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: cluster 2023-10-09T09:21:18.365215+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T09:21:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: cluster 2023-10-09T09:21:18.366037+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T09:21:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: audit 2023-10-09T09:21:18.369875+0000 mon.a (mon.0) 1771 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:21:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: cluster 2023-10-09T09:21:18.369943+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-10-09T09:21:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: audit 2023-10-09T09:21:19.272259+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:19 smithi094 bash[59677]: audit 2023-10-09T09:21:19.279192+0000 mon.a (mon.0) 1774 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:19.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:19.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:19.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:19.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:19.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:19.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:20.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:20.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:20.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:20.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:20.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:20.205 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:20.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:20.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:20.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:20.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:20.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:20.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:20.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:20.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:20.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:20.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:20 smithi094 bash[59677]: cluster 2023-10-09T09:21:19.848409+0000 mon.a (mon.0) 1775 : cluster 3 Health check update: Degraded data redundancy: 1343/133251 objects degraded (1.008%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:20.570 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:20.595 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:20.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:20.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:20.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:20.707 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:20 smithi016 bash[75179]: cluster 2023-10-09T09:21:19.848409+0000 mon.a (mon.0) 1775 : cluster 3 Health check update: Degraded data redundancy: 1343/133251 objects degraded (1.008%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:20.707 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:20 smithi016 bash[77892]: cluster 2023-10-09T09:21:19.848409+0000 mon.a (mon.0) 1775 : cluster 3 Health check update: Degraded data redundancy: 1343/133251 objects degraded (1.008%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:21:20.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:20.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:20.837 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:20.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:20.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:20.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:20.964 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:21.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:21.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:21.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:21.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:21.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:21.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:21.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:21.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:21.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:21.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:21.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:21.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:21.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:21.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:21.567 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:21 smithi016 bash[75179]: cluster 2023-10-09T09:21:20.251135+0000 mgr.y (mgr.25421) 1324 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 1343/131979 objects degraded (1.018%); 115 KiB/s, 57 objects/s recovering 2023-10-09T09:21:21.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:21 smithi016 bash[77892]: cluster 2023-10-09T09:21:20.251135+0000 mgr.y (mgr.25421) 1324 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 1343/131979 objects degraded (1.018%); 115 KiB/s, 57 objects/s recovering 2023-10-09T09:21:21.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:21.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:21.693 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:21.712 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:21 smithi094 bash[59677]: cluster 2023-10-09T09:21:20.251135+0000 mgr.y (mgr.25421) 1324 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 1343/131979 objects degraded (1.018%); 115 KiB/s, 57 objects/s recovering 2023-10-09T09:21:21.740 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:21.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:21.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:21.845 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:21.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:21.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:21.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:22.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:22.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:22.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:22.175 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:22.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:22.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:22.283 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:22.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:22.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:22.422 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:22.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:22.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:22.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:22.606 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:22.658 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:22.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:22.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:22.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:22.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:22.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:22.911 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:22.935 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:22.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:23.039 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:23.068 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:23.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:23.171 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:23.195 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:23.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:23.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:23.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:23.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:23.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:23.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:23.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:23.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:23 smithi094 bash[59677]: cluster 2023-10-09T09:21:22.251846+0000 mgr.y (mgr.25421) 1325 : cluster 0 pgmap v1462: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1343/124326 objects degraded (1.080%); 0 B/s, 44 objects/s recovering 2023-10-09T09:21:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:23 smithi094 bash[59677]: cluster 2023-10-09T09:21:22.385229+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:23 smithi094 bash[59677]: cluster 2023-10-09T09:21:22.385820+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:23 smithi094 bash[59677]: cluster 2023-10-09T09:21:22.388996+0000 mon.a (mon.0) 1776 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-10-09T09:21:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:23 smithi094 bash[59677]: audit 2023-10-09T09:21:22.829966+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:23.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:23 smithi094 bash[59677]: audit 2023-10-09T09:21:22.837879+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:23.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:23.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[75179]: cluster 2023-10-09T09:21:22.251846+0000 mgr.y (mgr.25421) 1325 : cluster 0 pgmap v1462: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1343/124326 objects degraded (1.080%); 0 B/s, 44 objects/s recovering 2023-10-09T09:21:23.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[75179]: cluster 2023-10-09T09:21:22.385229+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:23.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[75179]: cluster 2023-10-09T09:21:22.385820+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:23.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[75179]: cluster 2023-10-09T09:21:22.388996+0000 mon.a (mon.0) 1776 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-10-09T09:21:23.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[75179]: audit 2023-10-09T09:21:22.829966+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:23.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[75179]: audit 2023-10-09T09:21:22.837879+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:23.571 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[77892]: cluster 2023-10-09T09:21:22.251846+0000 mgr.y (mgr.25421) 1325 : cluster 0 pgmap v1462: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 1343/124326 objects degraded (1.080%); 0 B/s, 44 objects/s recovering 2023-10-09T09:21:23.571 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[77892]: cluster 2023-10-09T09:21:22.385229+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:23.571 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[77892]: cluster 2023-10-09T09:21:22.385820+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:23.572 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[77892]: cluster 2023-10-09T09:21:22.388996+0000 mon.a (mon.0) 1776 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-10-09T09:21:23.572 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[77892]: audit 2023-10-09T09:21:22.829966+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:23.572 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:23 smithi016 bash[77892]: audit 2023-10-09T09:21:22.837879+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:23.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:23.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:23.630 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:21:23.630 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:21:23.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:23.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:21:23.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:23.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:23.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:23.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:23.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:23.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:24.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:24.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:24.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:21:24.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:24.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:24.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:24.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:24.421 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:24.449 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:24.552 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:24.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:24.688 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:24.696 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:24 smithi094 bash[59677]: cluster 2023-10-09T09:21:22.396814+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:24.697 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:24 smithi094 bash[59677]: cluster 2023-10-09T09:21:23.396946+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:24.697 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:24 smithi094 bash[59677]: cluster 2023-10-09T09:21:23.398552+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-10-09T09:21:24.697 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:24 smithi094 bash[59677]: cluster 2023-10-09T09:21:23.466786+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:24.711 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[75179]: cluster 2023-10-09T09:21:22.396814+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:24.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[75179]: cluster 2023-10-09T09:21:23.396946+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:24.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[75179]: cluster 2023-10-09T09:21:23.398552+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-10-09T09:21:24.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[75179]: cluster 2023-10-09T09:21:23.466786+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[77892]: cluster 2023-10-09T09:21:22.396814+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T09:21:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[77892]: cluster 2023-10-09T09:21:23.396946+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[77892]: cluster 2023-10-09T09:21:23.398552+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-10-09T09:21:24.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:24 smithi016 bash[77892]: cluster 2023-10-09T09:21:23.466786+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:24.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:24.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:24.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:24.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:24.966 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:24.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:25.072 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:25.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:25.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:25.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:25.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:25.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:25.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:21:25.365 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:25.398 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:25.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:25 smithi016 bash[75179]: cluster 2023-10-09T09:21:23.461817+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:25 smithi016 bash[75179]: cluster 2023-10-09T09:21:24.252386+0000 mgr.y (mgr.25421) 1326 : cluster 0 pgmap v1465: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 652/122418 objects degraded (0.533%); 107 KiB/s, 44 objects/s recovering 2023-10-09T09:21:25.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:25 smithi016 bash[75179]: cluster 2023-10-09T09:21:24.849570+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: Degraded data redundancy: 652/122418 objects degraded (0.533%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:25.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:25 smithi016 bash[77892]: cluster 2023-10-09T09:21:23.461817+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:25.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:25 smithi016 bash[77892]: cluster 2023-10-09T09:21:24.252386+0000 mgr.y (mgr.25421) 1326 : cluster 0 pgmap v1465: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 652/122418 objects degraded (0.533%); 107 KiB/s, 44 objects/s recovering 2023-10-09T09:21:25.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:25 smithi016 bash[77892]: cluster 2023-10-09T09:21:24.849570+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: Degraded data redundancy: 652/122418 objects degraded (0.533%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:25.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:25.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:25.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:25.633 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:25.640 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:25 smithi094 bash[59677]: cluster 2023-10-09T09:21:23.461817+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T09:21:25.640 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:25 smithi094 bash[59677]: cluster 2023-10-09T09:21:24.252386+0000 mgr.y (mgr.25421) 1326 : cluster 0 pgmap v1465: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 652/122418 objects degraded (0.533%); 107 KiB/s, 44 objects/s recovering 2023-10-09T09:21:25.640 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:25 smithi094 bash[59677]: cluster 2023-10-09T09:21:24.849570+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: Degraded data redundancy: 652/122418 objects degraded (0.533%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:25.658 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:25.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:25.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:25.779 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:21:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:21:25] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T09:21:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:25.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:25.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:25.934 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:25.959 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:25.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:25.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:26.067 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:26.094 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:26.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:26.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:26.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:26.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:26.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:26.346 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:26.371 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:26.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:26.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:26.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:26.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:26.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:26.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:26.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:26.751 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:26.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:26.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:26.881 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:26.906 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:26.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:27.011 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:27.039 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:27.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:27.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:27.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:27.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:27.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:27.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:27.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:27.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:27.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:27.452 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:27.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:27.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:27.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:27.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:27.593 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:27 smithi016 bash[75179]: cluster 2023-10-09T09:21:26.253149+0000 mgr.y (mgr.25421) 1327 : cluster 0 pgmap v1466: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 652/118218 objects degraded (0.552%); 81 KiB/s, 34 objects/s recovering 2023-10-09T09:21:27.593 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:27 smithi016 bash[77892]: cluster 2023-10-09T09:21:26.253149+0000 mgr.y (mgr.25421) 1327 : cluster 0 pgmap v1466: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 652/118218 objects degraded (0.552%); 81 KiB/s, 34 objects/s recovering 2023-10-09T09:21:27.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:27.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:27.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:27.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:27 smithi094 bash[59677]: cluster 2023-10-09T09:21:26.253149+0000 mgr.y (mgr.25421) 1327 : cluster 0 pgmap v1466: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 652/118218 objects degraded (0.552%); 81 KiB/s, 34 objects/s recovering 2023-10-09T09:21:27.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:27.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:27.804 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:27.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:27.910 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:27.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:27.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:28.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:28.039 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:28.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:28.177 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:28.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:28.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:28.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:28.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:28.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:28.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:28.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:28.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:28.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:28.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:28.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:28.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:28.951 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:28.973 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:28.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:29.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:29.107 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:29.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:29.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:29.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:29.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:29.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:29.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:29.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:29.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:29.478 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[75179]: cluster 2023-10-09T09:21:28.253870+0000 mgr.y (mgr.25421) 1328 : cluster 0 pgmap v1467: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 652/114534 objects degraded (0.569%); 80 KiB/s, 72 objects/s recovering 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[75179]: audit 2023-10-09T09:21:28.460992+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[75179]: audit 2023-10-09T09:21:28.480173+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[75179]: audit 2023-10-09T09:21:29.162760+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[75179]: audit 2023-10-09T09:21:29.163835+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[75179]: audit 2023-10-09T09:21:29.171676+0000 mon.a (mon.0) 1785 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[77892]: cluster 2023-10-09T09:21:28.253870+0000 mgr.y (mgr.25421) 1328 : cluster 0 pgmap v1467: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 652/114534 objects degraded (0.569%); 80 KiB/s, 72 objects/s recovering 2023-10-09T09:21:29.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[77892]: audit 2023-10-09T09:21:28.460992+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[77892]: audit 2023-10-09T09:21:28.480173+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[77892]: audit 2023-10-09T09:21:29.162760+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:21:29.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[77892]: audit 2023-10-09T09:21:29.163835+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:21:29.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:29 smithi016 bash[77892]: audit 2023-10-09T09:21:29.171676+0000 mon.a (mon.0) 1785 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.502 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:29.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:29.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:29.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:29.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:29.734 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:29 smithi094 bash[59677]: cluster 2023-10-09T09:21:28.253870+0000 mgr.y (mgr.25421) 1328 : cluster 0 pgmap v1467: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 652/114534 objects degraded (0.569%); 80 KiB/s, 72 objects/s recovering 2023-10-09T09:21:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:29 smithi094 bash[59677]: audit 2023-10-09T09:21:28.460992+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:29 smithi094 bash[59677]: audit 2023-10-09T09:21:28.480173+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:29 smithi094 bash[59677]: audit 2023-10-09T09:21:29.162760+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:21:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:29 smithi094 bash[59677]: audit 2023-10-09T09:21:29.163835+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:21:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:29 smithi094 bash[59677]: audit 2023-10-09T09:21:29.171676+0000 mon.a (mon.0) 1785 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:29.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:29.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:29.846 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:29.847 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:29.854 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:29.866 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:29.898 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:29.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:29.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:30.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:30.037 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:30.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:30.140 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:30.161 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:30.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:30.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:30.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:30.431 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:30.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:30.543 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:30 smithi016 bash[75179]: cluster 2023-10-09T09:21:29.850477+0000 mon.a (mon.0) 1786 : cluster 3 Health check update: Degraded data redundancy: 652/114534 objects degraded (0.569%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:30.544 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:30 smithi016 bash[77892]: cluster 2023-10-09T09:21:29.850477+0000 mon.a (mon.0) 1786 : cluster 3 Health check update: Degraded data redundancy: 652/114534 objects degraded (0.569%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:30.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:30.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:30.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:30.680 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:30 smithi094 bash[59677]: cluster 2023-10-09T09:21:29.850477+0000 mon.a (mon.0) 1786 : cluster 3 Health check update: Degraded data redundancy: 652/114534 objects degraded (0.569%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:30.697 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:30.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:30.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:30.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:30.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:30.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:31.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:31.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:31.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:31.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:31.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:31.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:31.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:31.326 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:31.354 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:31.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:31.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:31.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:31.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:31.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:31.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:31.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:31.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:31.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:31 smithi094 bash[59677]: cluster 2023-10-09T09:21:30.254392+0000 mgr.y (mgr.25421) 1329 : cluster 0 pgmap v1468: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 652/113367 objects degraded (0.575%); 80 KiB/s, 43 objects/s recovering 2023-10-09T09:21:31.748 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:31.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:31 smithi016 bash[75179]: cluster 2023-10-09T09:21:30.254392+0000 mgr.y (mgr.25421) 1329 : cluster 0 pgmap v1468: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 652/113367 objects degraded (0.575%); 80 KiB/s, 43 objects/s recovering 2023-10-09T09:21:31.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:31 smithi016 bash[77892]: cluster 2023-10-09T09:21:30.254392+0000 mgr.y (mgr.25421) 1329 : cluster 0 pgmap v1468: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 652/113367 objects degraded (0.575%); 80 KiB/s, 43 objects/s recovering 2023-10-09T09:21:31.781 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:31.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:31.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:31.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:31.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:31.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:32.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:32.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:32.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:32.160 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:32.188 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:32.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:32.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:32.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:32.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:32.554 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:32.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:32.684 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:32.712 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:32.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:32.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:32.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:32.965 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:32.989 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:32.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:33.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:33.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:33.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:33.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:33.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:33.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:33.359 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:33.391 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:33.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:33.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:33.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:33.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:33.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:33.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:33.632 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:33 smithi094 bash[59677]: cluster 2023-10-09T09:21:32.255041+0000 mgr.y (mgr.25421) 1330 : cluster 0 pgmap v1469: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 652/106083 objects degraded (0.615%); 65 KiB/s, 35 objects/s recovering 2023-10-09T09:21:33.652 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:33.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:33 smithi016 bash[75179]: cluster 2023-10-09T09:21:32.255041+0000 mgr.y (mgr.25421) 1330 : cluster 0 pgmap v1469: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 652/106083 objects degraded (0.615%); 65 KiB/s, 35 objects/s recovering 2023-10-09T09:21:33.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:33 smithi016 bash[77892]: cluster 2023-10-09T09:21:32.255041+0000 mgr.y (mgr.25421) 1330 : cluster 0 pgmap v1469: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 652/106083 objects degraded (0.615%); 65 KiB/s, 35 objects/s recovering 2023-10-09T09:21:33.757 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:33.802 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:33.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:33.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:34.129 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:34.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:34.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:34.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:34 smithi016 bash[75179]: audit 2023-10-09T09:21:34.261702+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:34.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:34 smithi016 bash[77892]: audit 2023-10-09T09:21:34.261702+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:34.750 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:34.753 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:34 smithi094 bash[59677]: audit 2023-10-09T09:21:34.261702+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:34.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:34.855 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:34.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:36.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[75179]: cluster 2023-10-09T09:21:34.255649+0000 mgr.y (mgr.25421) 1331 : cluster 0 pgmap v1470: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 652/104205 objects degraded (0.626%); 0 B/s, 54 objects/s recovering 2023-10-09T09:21:36.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[75179]: cluster 2023-10-09T09:21:34.496512+0000 mon.a (mon.0) 1788 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-10-09T09:21:36.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[75179]: cluster 2023-10-09T09:21:34.501868+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[75179]: cluster 2023-10-09T09:21:34.503757+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[75179]: cluster 2023-10-09T09:21:34.509254+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[75179]: cluster 2023-10-09T09:21:34.510895+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[75179]: cluster 2023-10-09T09:21:34.852194+0000 mon.a (mon.0) 1789 : cluster 3 Health check update: Degraded data redundancy: 652/104205 objects degraded (0.626%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:36.217 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:21:35] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.33.4" 2023-10-09T09:21:36.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[77892]: cluster 2023-10-09T09:21:34.255649+0000 mgr.y (mgr.25421) 1331 : cluster 0 pgmap v1470: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 652/104205 objects degraded (0.626%); 0 B/s, 54 objects/s recovering 2023-10-09T09:21:36.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[77892]: cluster 2023-10-09T09:21:34.496512+0000 mon.a (mon.0) 1788 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-10-09T09:21:36.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[77892]: cluster 2023-10-09T09:21:34.501868+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[77892]: cluster 2023-10-09T09:21:34.503757+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[77892]: cluster 2023-10-09T09:21:34.509254+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[77892]: cluster 2023-10-09T09:21:34.510895+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:35 smithi016 bash[77892]: cluster 2023-10-09T09:21:34.852194+0000 mon.a (mon.0) 1789 : cluster 3 Health check update: Degraded data redundancy: 652/104205 objects degraded (0.626%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:36.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:36.218 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:35 smithi094 bash[59677]: cluster 2023-10-09T09:21:34.255649+0000 mgr.y (mgr.25421) 1331 : cluster 0 pgmap v1470: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 652/104205 objects degraded (0.626%); 0 B/s, 54 objects/s recovering 2023-10-09T09:21:36.218 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:35 smithi094 bash[59677]: cluster 2023-10-09T09:21:34.496512+0000 mon.a (mon.0) 1788 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-10-09T09:21:36.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:35 smithi094 bash[59677]: cluster 2023-10-09T09:21:34.501868+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:35 smithi094 bash[59677]: cluster 2023-10-09T09:21:34.503757+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:35 smithi094 bash[59677]: cluster 2023-10-09T09:21:34.509254+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:35 smithi094 bash[59677]: cluster 2023-10-09T09:21:34.510895+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T09:21:36.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:35 smithi094 bash[59677]: cluster 2023-10-09T09:21:34.852194+0000 mon.a (mon.0) 1789 : cluster 3 Health check update: Degraded data redundancy: 652/104205 objects degraded (0.626%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:21:36.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:36.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:36.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:36.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:37.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[75179]: cluster 2023-10-09T09:21:35.493940+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[75179]: cluster 2023-10-09T09:21:35.494027+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[75179]: cluster 2023-10-09T09:21:35.495192+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-10-09T09:21:37.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[75179]: cluster 2023-10-09T09:21:35.537422+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[75179]: cluster 2023-10-09T09:21:35.551393+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[77892]: cluster 2023-10-09T09:21:35.493940+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[77892]: cluster 2023-10-09T09:21:35.494027+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[77892]: cluster 2023-10-09T09:21:35.495192+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[77892]: cluster 2023-10-09T09:21:35.537422+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:36 smithi016 bash[77892]: cluster 2023-10-09T09:21:35.551393+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.467 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:37.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:36 smithi094 bash[59677]: cluster 2023-10-09T09:21:35.493940+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:36 smithi094 bash[59677]: cluster 2023-10-09T09:21:35.494027+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.474 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:36 smithi094 bash[59677]: cluster 2023-10-09T09:21:35.495192+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-10-09T09:21:37.474 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:36 smithi094 bash[59677]: cluster 2023-10-09T09:21:35.537422+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.474 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:36 smithi094 bash[59677]: cluster 2023-10-09T09:21:35.551393+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T09:21:37.503 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:37.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:37.683 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:37 smithi016 bash[75179]: cluster 2023-10-09T09:21:36.256401+0000 mgr.y (mgr.25421) 1332 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 652/99594 objects degraded (0.655%); 0 B/s, 34 objects/s recovering 2023-10-09T09:21:37.754 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:37 smithi016 bash[77892]: cluster 2023-10-09T09:21:36.256401+0000 mgr.y (mgr.25421) 1332 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 652/99594 objects degraded (0.655%); 0 B/s, 34 objects/s recovering 2023-10-09T09:21:37.754 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:37.756 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:37 smithi094 bash[59677]: cluster 2023-10-09T09:21:36.256401+0000 mgr.y (mgr.25421) 1332 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 652/99594 objects degraded (0.655%); 0 B/s, 34 objects/s recovering 2023-10-09T09:21:37.797 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:38.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:38.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:38.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:38.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:38.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:38.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:38.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:38.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:38.494 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:38.524 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:38.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:38.627 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:38.655 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:38.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:38.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:38.759 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:38.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:38.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:38.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:38.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:38.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:38.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:39.016 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:39.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:39.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:39.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:39.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:39.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:39.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:39.285 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:39.312 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:39.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:39.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:39.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:39.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:39.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:39.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:39.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:39.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:39.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:39.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:39.721 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:39.729 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:39 smithi094 bash[59677]: cluster 2023-10-09T09:21:38.257022+0000 mgr.y (mgr.25421) 1333 : cluster 0 pgmap v1474: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 88 KiB/s, 80 objects/s recovering 2023-10-09T09:21:39.743 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:39.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:39.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:39.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:39 smithi016 bash[75179]: cluster 2023-10-09T09:21:38.257022+0000 mgr.y (mgr.25421) 1333 : cluster 0 pgmap v1474: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 88 KiB/s, 80 objects/s recovering 2023-10-09T09:21:39.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:39 smithi016 bash[77892]: cluster 2023-10-09T09:21:38.257022+0000 mgr.y (mgr.25421) 1333 : cluster 0 pgmap v1474: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 88 KiB/s, 80 objects/s recovering 2023-10-09T09:21:39.843 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:21:39.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:39.951 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:21:39.974 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:40.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:40.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:40.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:40.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:40.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:40.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:40 smithi016 bash[75179]: cluster 2023-10-09T09:21:39.853506+0000 mon.a (mon.0) 1791 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:21:40.854 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:40 smithi016 bash[77892]: cluster 2023-10-09T09:21:39.853506+0000 mon.a (mon.0) 1791 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:21:40.955 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:41.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:40 smithi094 bash[59677]: cluster 2023-10-09T09:21:39.853506+0000 mon.a (mon.0) 1791 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:21:41.048 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:41.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:41.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:41.203 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:41.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:41.303 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:41.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:41.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:41.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:41.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:41.763 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:41 smithi094 bash[59677]: audit 2023-10-09T09:21:39.816113+0000 mgr.y (mgr.25421) 1334 : audit 0 from='client.35952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:21:41.763 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:41 smithi094 bash[59677]: cluster 2023-10-09T09:21:40.257485+0000 mgr.y (mgr.25421) 1335 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 88 KiB/s, 80 objects/s recovering 2023-10-09T09:21:41.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:41.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:41 smithi016 bash[75179]: audit 2023-10-09T09:21:39.816113+0000 mgr.y (mgr.25421) 1334 : audit 0 from='client.35952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:21:41.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:41 smithi016 bash[75179]: cluster 2023-10-09T09:21:40.257485+0000 mgr.y (mgr.25421) 1335 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 88 KiB/s, 80 objects/s recovering 2023-10-09T09:21:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:41 smithi016 bash[77892]: audit 2023-10-09T09:21:39.816113+0000 mgr.y (mgr.25421) 1334 : audit 0 from='client.35952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:21:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:41 smithi016 bash[77892]: cluster 2023-10-09T09:21:40.257485+0000 mgr.y (mgr.25421) 1335 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 88 KiB/s, 80 objects/s recovering 2023-10-09T09:21:41.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:41.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:41.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:41.939 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:41.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:41.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:42.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:42.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:42.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:42.204 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:42.273 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:42.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:42.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:42.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:42.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:42.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:42.823 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:43.052 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:43.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:43.169 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:43.214 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:43.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:43.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:43.316 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:43.340 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:43.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:43.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:43.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:43.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:43.590 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:43.601 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:43 smithi094 bash[59677]: cluster 2023-10-09T09:21:42.258132+0000 mgr.y (mgr.25421) 1336 : cluster 0 pgmap v1476: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 88 KiB/s, 73 objects/s recovering 2023-10-09T09:21:43.619 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:43.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:43.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:43.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:43.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:43 smithi016 bash[77892]: cluster 2023-10-09T09:21:42.258132+0000 mgr.y (mgr.25421) 1336 : cluster 0 pgmap v1476: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 88 KiB/s, 73 objects/s recovering 2023-10-09T09:21:43.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:43 smithi016 bash[75179]: cluster 2023-10-09T09:21:42.258132+0000 mgr.y (mgr.25421) 1336 : cluster 0 pgmap v1476: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 88 KiB/s, 73 objects/s recovering 2023-10-09T09:21:43.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:43.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:43.835 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1474,"stamp":"2023-10-09T09:21:38.256609+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2086672312,"num_objects":31856,"num_object_clones":0,"num_object_copies":95568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31856,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":154700,"num_write_kb":5963086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6131,"num_bytes_recovered":159079586,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126134,"ondisk_log_size":126134,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13608544,"kb_used_data":6298464,"kb_used_omap":126,"kb_used_meta":7309761,"kb_avail":736352672,"statfs":{"total":767960285184,"available":754025136128,"internally_reserved":0,"allocated":6449627136,"data_stored":6442589677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":7485195899},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-388825088,"num_objects":-5933,"num_object_clones":0,"num_object_copies":-17799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-652,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5933,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":642,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002544"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":227,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.464896+0000","last_change":"2023-10-09T09:20:19.894893+0000","last_active":"2023-10-09T09:21:23.464896+0000","last_peered":"2023-10-09T09:21:23.464896+0000","last_clean":"2023-10-09T09:21:23.464896+0000","last_became_active":"2023-10-09T09:20:19.893953+0000","last_became_peered":"2023-10-09T09:20:19.893953+0000","last_unstale":"2023-10-09T09:21:23.464896+0000","last_undegraded":"2023-10-09T09:21:23.464896+0000","last_fullsized":"2023-10-09T09:21:23.464896+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503596+0000","last_change":"2023-10-09T09:14:48.316760+0000","last_active":"2023-10-09T09:21:34.503596+0000","last_peered":"2023-10-09T09:21:34.503596+0000","last_clean":"2023-10-09T09:21:34.503596+0000","last_became_active":"2023-10-09T09:14:48.316432+0000","last_became_peered":"2023-10-09T09:14:48.316432+0000","last_unstale":"2023-10-09T09:21:34.503596+0000","last_undegraded":"2023-10-09T09:21:34.503596+0000","last_fullsized":"2023-10-09T09:21:34.503596+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:41.630374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503101+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:21:34.503101+0000","last_peered":"2023-10-09T09:21:34.503101+0000","last_clean":"2023-10-09T09:21:34.503101+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:21:34.503101+0000","last_undegraded":"2023-10-09T09:21:34.503101+0000","last_fullsized":"2023-10-09T09:21:34.503101+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":358,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.500288+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:21:34.500288+0000","last_peered":"2023-10-09T09:21:34.500288+0000","last_clean":"2023-10-09T09:21:34.500288+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:21:34.500288+0000","last_undegraded":"2023-10-09T09:21:34.500288+0000","last_fullsized":"2023-10-09T09:21:34.500288+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":302,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.497859+0000","last_change":"2023-10-09T09:14:48.312713+0000","last_active":"2023-10-09T09:21:34.497859+0000","last_peered":"2023-10-09T09:21:34.497859+0000","last_clean":"2023-10-09T09:21:34.497859+0000","last_became_active":"2023-10-09T09:14:48.312429+0000","last_became_peered":"2023-10-09T09:14:48.312429+0000","last_unstale":"2023-10-09T09:21:34.497859+0000","last_undegraded":"2023-10-09T09:21:34.497859+0000","last_fullsized":"2023-10-09T09:21:34.497859+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:56.755105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":233,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507275+0000","last_change":"2023-10-09T09:19:42.872407+0000","last_active":"2023-10-09T09:21:34.507275+0000","last_peered":"2023-10-09T09:21:34.507275+0000","last_clean":"2023-10-09T09:21:34.507275+0000","last_became_active":"2023-10-09T09:19:42.871637+0000","last_became_peered":"2023-10-09T09:19:42.871637+0000","last_unstale":"2023-10-09T09:21:34.507275+0000","last_undegraded":"2023-10-09T09:21:34.507275+0000","last_fullsized":"2023-10-09T09:21:34.507275+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":310,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507247+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:21:34.507247+0000","last_peered":"2023-10-09T09:21:34.507247+0000","last_clean":"2023-10-09T09:21:34.507247+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:21:34.507247+0000","last_undegraded":"2023-10-09T09:21:34.507247+0000","last_fullsized":"2023-10-09T09:21:34.507247+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":69,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503564+0000","last_change":"2023-10-09T09:14:48.306040+0000","last_active":"2023-10-09T09:21:34.503564+0000","last_peered":"2023-10-09T09:21:34.503564+0000","last_clean":"2023-10-09T09:21:34.503564+0000","last_became_active":"2023-10-09T09:14:48.304137+0000","last_became_peered":"2023-10-09T09:14:48.304137+0000","last_unstale":"2023-10-09T09:21:34.503564+0000","last_undegraded":"2023-10-09T09:21:34.503564+0000","last_fullsized":"2023-10-09T09:21:34.503564+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:47.446761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":126,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.509622+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:21:34.509622+0000","last_peered":"2023-10-09T09:21:34.509622+0000","last_clean":"2023-10-09T09:21:34.509622+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:21:34.509622+0000","last_undegraded":"2023-10-09T09:21:34.509622+0000","last_fullsized":"2023-10-09T09:21:34.509622+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":284,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.513480+0000","last_change":"2023-10-09T09:13:48.190068+0000","last_active":"2023-10-09T09:21:34.513480+0000","last_peered":"2023-10-09T09:21:34.513480+0000","last_clean":"2023-10-09T09:21:34.513480+0000","last_became_active":"2023-10-09T09:13:48.189740+0000","last_became_peered":"2023-10-09T09:13:48.189740+0000","last_unstale":"2023-10-09T09:21:34.513480+0000","last_undegraded":"2023-10-09T09:21:34.513480+0000","last_fullsized":"2023-10-09T09:21:34.513480+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":107,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507573+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:21:34.507573+0000","last_peered":"2023-10-09T09:21:34.507573+0000","last_clean":"2023-10-09T09:21:34.507573+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:21:34.507573+0000","last_undegraded":"2023-10-09T09:21:34.507573+0000","last_fullsized":"2023-10-09T09:21:34.507573+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":96,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503428+0000","last_change":"2023-10-09T09:20:19.898526+0000","last_active":"2023-10-09T09:21:34.503428+0000","last_peered":"2023-10-09T09:21:34.503428+0000","last_clean":"2023-10-09T09:21:34.503428+0000","last_became_active":"2023-10-09T09:20:19.898197+0000","last_became_peered":"2023-10-09T09:20:19.898197+0000","last_unstale":"2023-10-09T09:21:34.503428+0000","last_undegraded":"2023-10-09T09:21:34.503428+0000","last_fullsized":"2023-10-09T09:21:34.503428+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:30.135339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":221,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465230+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:21:23.465230+0000","last_peered":"2023-10-09T09:21:23.465230+0000","last_clean":"2023-10-09T09:21:23.465230+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:21:23.465230+0000","last_undegraded":"2023-10-09T09:21:23.465230+0000","last_fullsized":"2023-10-09T09:21:23.465230+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":209,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465524+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:21:23.465524+0000","last_peered":"2023-10-09T09:21:23.465524+0000","last_clean":"2023-10-09T09:21:23.465524+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:21:23.465524+0000","last_undegraded":"2023-10-09T09:21:23.465524+0000","last_fullsized":"2023-10-09T09:21:23.465524+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":167,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507074+0000","last_change":"2023-10-09T09:20:19.876172+0000","last_active":"2023-10-09T09:21:34.507074+0000","last_peered":"2023-10-09T09:21:34.507074+0000","last_clean":"2023-10-09T09:21:34.507074+0000","last_became_active":"2023-10-09T09:20:19.765721+0000","last_became_peered":"2023-10-09T09:20:19.765721+0000","last_unstale":"2023-10-09T09:21:34.507074+0000","last_undegraded":"2023-10-09T09:21:34.507074+0000","last_fullsized":"2023-10-09T09:21:34.507074+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.523818+0000","last_change":"2023-10-09T09:14:48.319108+0000","last_active":"2023-10-09T09:21:34.523818+0000","last_peered":"2023-10-09T09:21:34.523818+0000","last_clean":"2023-10-09T09:21:34.523818+0000","last_became_active":"2023-10-09T09:14:48.318443+0000","last_became_peered":"2023-10-09T09:14:48.318443+0000","last_unstale":"2023-10-09T09:21:34.523818+0000","last_undegraded":"2023-10-09T09:21:34.523818+0000","last_fullsized":"2023-10-09T09:21:34.523818+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:31.939020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":338,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.499966+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:21:34.499966+0000","last_peered":"2023-10-09T09:21:34.499966+0000","last_clean":"2023-10-09T09:21:34.499966+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:21:34.499966+0000","last_undegraded":"2023-10-09T09:21:34.499966+0000","last_fullsized":"2023-10-09T09:21:34.499966+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":120,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507783+0000","last_change":"2023-10-09T09:15:23.415336+0000","last_active":"2023-10-09T09:21:34.507783+0000","last_peered":"2023-10-09T09:21:34.507783+0000","last_clean":"2023-10-09T09:21:34.507783+0000","last_became_active":"2023-10-09T09:15:23.414900+0000","last_became_peered":"2023-10-09T09:15:23.414900+0000","last_unstale":"2023-10-09T09:21:34.507783+0000","last_undegraded":"2023-10-09T09:21:34.507783+0000","last_fullsized":"2023-10-09T09:21:34.507783+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":163,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465205+0000","last_change":"2023-10-09T09:13:48.205269+0000","last_active":"2023-10-09T09:21:23.465205+0000","last_peered":"2023-10-09T09:21:23.465205+0000","last_clean":"2023-10-09T09:21:23.465205+0000","last_became_active":"2023-10-09T09:13:48.204894+0000","last_became_peered":"2023-10-09T09:13:48.204894+0000","last_unstale":"2023-10-09T09:21:23.465205+0000","last_undegraded":"2023-10-09T09:21:23.465205+0000","last_fullsized":"2023-10-09T09:21:23.465205+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":310,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.500363+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:21:34.500363+0000","last_peered":"2023-10-09T09:21:34.500363+0000","last_clean":"2023-10-09T09:21:34.500363+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:21:34.500363+0000","last_undegraded":"2023-10-09T09:21:34.500363+0000","last_fullsized":"2023-10-09T09:21:34.500363+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":237,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.504416+0000","last_change":"2023-10-09T09:15:23.420333+0000","last_active":"2023-10-09T09:21:34.504416+0000","last_peered":"2023-10-09T09:21:34.504416+0000","last_clean":"2023-10-09T09:21:34.504416+0000","last_became_active":"2023-10-09T09:15:23.412581+0000","last_became_peered":"2023-10-09T09:15:23.412581+0000","last_unstale":"2023-10-09T09:21:34.504416+0000","last_undegraded":"2023-10-09T09:21:34.504416+0000","last_fullsized":"2023-10-09T09:21:34.504416+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":28,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507600+0000","last_change":"2023-10-09T09:19:42.875063+0000","last_active":"2023-10-09T09:21:34.507600+0000","last_peered":"2023-10-09T09:21:34.507600+0000","last_clean":"2023-10-09T09:21:34.507600+0000","last_became_active":"2023-10-09T09:19:42.874414+0000","last_became_peered":"2023-10-09T09:19:42.874414+0000","last_unstale":"2023-10-09T09:21:34.507600+0000","last_undegraded":"2023-10-09T09:21:34.507600+0000","last_fullsized":"2023-10-09T09:21:34.507600+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:40.141249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.7","version":"321'9630","reported_seq":9667,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:37.162319+0000","last_change":"2023-10-09T09:18:10.373025+0000","last_active":"2023-10-09T09:21:37.162319+0000","last_peered":"2023-10-09T09:21:37.162319+0000","last_clean":"2023-10-09T09:21:37.162319+0000","last_became_active":"2023-10-09T09:18:10.372667+0000","last_became_peered":"2023-10-09T09:18:10.372667+0000","last_unstale":"2023-10-09T09:21:37.162319+0000","last_undegraded":"2023-10-09T09:21:37.162319+0000","last_fullsized":"2023-10-09T09:21:37.162319+0000","mapping_epoch":294,"log_start":"300'2800","ondisk_log_start":"300'2800","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:26.426396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123469824,"num_objects":1884,"num_object_clones":0,"num_object_copies":5652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9630,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":118,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507292+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:21:34.507292+0000","last_peered":"2023-10-09T09:21:34.507292+0000","last_clean":"2023-10-09T09:21:34.507292+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:21:34.507292+0000","last_undegraded":"2023-10-09T09:21:34.507292+0000","last_fullsized":"2023-10-09T09:21:34.507292+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":80,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.506597+0000","last_change":"2023-10-09T09:15:23.414649+0000","last_active":"2023-10-09T09:21:34.506597+0000","last_peered":"2023-10-09T09:21:34.506597+0000","last_clean":"2023-10-09T09:21:34.506597+0000","last_became_active":"2023-10-09T09:15:23.414272+0000","last_became_peered":"2023-10-09T09:15:23.414272+0000","last_unstale":"2023-10-09T09:21:34.506597+0000","last_undegraded":"2023-10-09T09:21:34.506597+0000","last_fullsized":"2023-10-09T09:21:34.506597+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"321'9739","reported_seq":12661,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:37.505444+0000","last_change":"2023-10-09T09:21:35.535431+0000","last_active":"2023-10-09T09:21:37.505444+0000","last_peered":"2023-10-09T09:21:37.505444+0000","last_clean":"2023-10-09T09:21:37.505444+0000","last_became_active":"2023-10-09T09:21:35.501036+0000","last_became_peered":"2023-10-09T09:21:35.501036+0000","last_unstale":"2023-10-09T09:21:37.505444+0000","last_undegraded":"2023-10-09T09:21:37.505444+0000","last_fullsized":"2023-10-09T09:21:37.505444+0000","mapping_epoch":320,"log_start":"298'1900","ondisk_log_start":"298'1900","created":294,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":7839,"log_dups_size":0,"ondisk_log_size":7839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:10.403219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126418944,"num_objects":1929,"num_object_clones":0,"num_object_copies":5787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9739,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1430,"num_bytes_recovered":48955392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":118,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507326+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:21:34.507326+0000","last_peered":"2023-10-09T09:21:34.507326+0000","last_clean":"2023-10-09T09:21:34.507326+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:21:34.507326+0000","last_undegraded":"2023-10-09T09:21:34.507326+0000","last_fullsized":"2023-10-09T09:21:34.507326+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":209,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.464801+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:21:23.464801+0000","last_peered":"2023-10-09T09:21:23.464801+0000","last_clean":"2023-10-09T09:21:23.464801+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:21:23.464801+0000","last_undegraded":"2023-10-09T09:21:23.464801+0000","last_fullsized":"2023-10-09T09:21:23.464801+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"321'9478","reported_seq":9513,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:35.692356+0000","last_change":"2023-10-09T09:18:10.916026+0000","last_active":"2023-10-09T09:21:35.692356+0000","last_peered":"2023-10-09T09:21:35.692356+0000","last_clean":"2023-10-09T09:21:35.692356+0000","last_became_active":"2023-10-09T09:18:10.915063+0000","last_became_peered":"2023-10-09T09:18:10.915063+0000","last_unstale":"2023-10-09T09:21:35.692356+0000","last_undegraded":"2023-10-09T09:21:35.692356+0000","last_fullsized":"2023-10-09T09:21:35.692356+0000","mapping_epoch":294,"log_start":"298'1100","ondisk_log_start":"298'1100","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8378,"log_dups_size":0,"ondisk_log_size":8378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126484480,"num_objects":1930,"num_object_clones":0,"num_object_copies":5790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9478,"num_write_kb":365056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":194,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507475+0000","last_change":"2023-10-09T09:15:23.437420+0000","last_active":"2023-10-09T09:21:34.507475+0000","last_peered":"2023-10-09T09:21:34.507475+0000","last_clean":"2023-10-09T09:21:34.507475+0000","last_became_active":"2023-10-09T09:15:23.429358+0000","last_became_peered":"2023-10-09T09:15:23.429358+0000","last_unstale":"2023-10-09T09:21:34.507475+0000","last_undegraded":"2023-10-09T09:21:34.507475+0000","last_fullsized":"2023-10-09T09:21:34.507475+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":224,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507052+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:21:34.507052+0000","last_peered":"2023-10-09T09:21:34.507052+0000","last_clean":"2023-10-09T09:21:34.507052+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:21:34.507052+0000","last_undegraded":"2023-10-09T09:21:34.507052+0000","last_fullsized":"2023-10-09T09:21:34.507052+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"321'9609","reported_seq":10992,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:35.555315+0000","last_change":"2023-10-09T09:21:09.433946+0000","last_active":"2023-10-09T09:21:35.555315+0000","last_peered":"2023-10-09T09:21:35.555315+0000","last_clean":"2023-10-09T09:21:35.555315+0000","last_became_active":"2023-10-09T09:21:09.351963+0000","last_became_peered":"2023-10-09T09:21:09.351963+0000","last_unstale":"2023-10-09T09:21:35.555315+0000","last_undegraded":"2023-10-09T09:21:35.555315+0000","last_fullsized":"2023-10-09T09:21:35.555315+0000","mapping_epoch":315,"log_start":"298'1200","ondisk_log_start":"298'1200","created":294,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8409,"log_dups_size":0,"ondisk_log_size":8409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137822208,"num_objects":2103,"num_object_clones":0,"num_object_copies":6309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9609,"num_write_kb":374784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":662,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":109,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.510048+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:21:34.510048+0000","last_peered":"2023-10-09T09:21:34.510048+0000","last_clean":"2023-10-09T09:21:34.510048+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:21:34.510048+0000","last_undegraded":"2023-10-09T09:21:34.510048+0000","last_fullsized":"2023-10-09T09:21:34.510048+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":338,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.498265+0000","last_change":"2023-10-09T09:18:51.869617+0000","last_active":"2023-10-09T09:21:34.498265+0000","last_peered":"2023-10-09T09:21:34.498265+0000","last_clean":"2023-10-09T09:21:34.498265+0000","last_became_active":"2023-10-09T09:18:51.869212+0000","last_became_peered":"2023-10-09T09:18:51.869212+0000","last_unstale":"2023-10-09T09:21:34.498265+0000","last_undegraded":"2023-10-09T09:21:34.498265+0000","last_fullsized":"2023-10-09T09:21:34.498265+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:36.753958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"321'9524","reported_seq":9559,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:35.663478+0000","last_change":"2023-10-09T09:18:10.372418+0000","last_active":"2023-10-09T09:21:35.663478+0000","last_peered":"2023-10-09T09:21:35.663478+0000","last_clean":"2023-10-09T09:21:35.663478+0000","last_became_active":"2023-10-09T09:18:10.371749+0000","last_became_peered":"2023-10-09T09:18:10.371749+0000","last_unstale":"2023-10-09T09:21:35.663478+0000","last_undegraded":"2023-10-09T09:21:35.663478+0000","last_fullsized":"2023-10-09T09:21:35.663478+0000","mapping_epoch":294,"log_start":"298'1100","ondisk_log_start":"298'1100","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8424,"log_dups_size":0,"ondisk_log_size":8424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9524,"num_write_kb":368832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":132,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.497988+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:21:34.497988+0000","last_peered":"2023-10-09T09:21:34.497988+0000","last_clean":"2023-10-09T09:21:34.497988+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:21:34.497988+0000","last_undegraded":"2023-10-09T09:21:34.497988+0000","last_fullsized":"2023-10-09T09:21:34.497988+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":182,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465397+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:21:23.465397+0000","last_peered":"2023-10-09T09:21:23.465397+0000","last_clean":"2023-10-09T09:21:23.465397+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:21:23.465397+0000","last_undegraded":"2023-10-09T09:21:23.465397+0000","last_fullsized":"2023-10-09T09:21:23.465397+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"321'9731","reported_seq":11072,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:37.631628+0000","last_change":"2023-10-09T09:20:32.882382+0000","last_active":"2023-10-09T09:21:37.631628+0000","last_peered":"2023-10-09T09:21:37.631628+0000","last_clean":"2023-10-09T09:21:37.631628+0000","last_became_active":"2023-10-09T09:20:32.867753+0000","last_became_peered":"2023-10-09T09:20:32.867753+0000","last_unstale":"2023-10-09T09:21:37.631628+0000","last_undegraded":"2023-10-09T09:21:37.631628+0000","last_fullsized":"2023-10-09T09:21:37.631628+0000","mapping_epoch":309,"log_start":"298'1900","ondisk_log_start":"298'1900","created":294,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":7831,"log_dups_size":0,"ondisk_log_size":7831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:56.801455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125108224,"num_objects":1909,"num_object_clones":0,"num_object_copies":5727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9731,"num_write_kb":372480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":639,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":188,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.506930+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:21:34.506930+0000","last_peered":"2023-10-09T09:21:34.506930+0000","last_clean":"2023-10-09T09:21:34.506930+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:21:34.506930+0000","last_undegraded":"2023-10-09T09:21:34.506930+0000","last_fullsized":"2023-10-09T09:21:34.506930+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":227,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465449+0000","last_change":"2023-10-09T09:20:19.935938+0000","last_active":"2023-10-09T09:21:23.465449+0000","last_peered":"2023-10-09T09:21:23.465449+0000","last_clean":"2023-10-09T09:21:23.465449+0000","last_became_active":"2023-10-09T09:20:19.935423+0000","last_became_peered":"2023-10-09T09:20:19.935423+0000","last_unstale":"2023-10-09T09:21:23.465449+0000","last_undegraded":"2023-10-09T09:21:23.465449+0000","last_fullsized":"2023-10-09T09:21:23.465449+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"321'9554","reported_seq":9588,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:35.764748+0000","last_change":"2023-10-09T09:18:10.916054+0000","last_active":"2023-10-09T09:21:35.764748+0000","last_peered":"2023-10-09T09:21:35.764748+0000","last_clean":"2023-10-09T09:21:35.764748+0000","last_became_active":"2023-10-09T09:18:10.915135+0000","last_became_peered":"2023-10-09T09:18:10.915135+0000","last_unstale":"2023-10-09T09:21:35.764748+0000","last_undegraded":"2023-10-09T09:21:35.764748+0000","last_fullsized":"2023-10-09T09:21:35.764748+0000","mapping_epoch":294,"log_start":"298'1200","ondisk_log_start":"298'1200","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8354,"log_dups_size":0,"ondisk_log_size":8354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134479872,"num_objects":2052,"num_object_clones":0,"num_object_copies":6156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9554,"num_write_kb":371392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":254,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465163+0000","last_change":"2023-10-09T09:20:19.755858+0000","last_active":"2023-10-09T09:21:23.465163+0000","last_peered":"2023-10-09T09:21:23.465163+0000","last_clean":"2023-10-09T09:21:23.465163+0000","last_became_active":"2023-10-09T09:20:19.755442+0000","last_became_peered":"2023-10-09T09:20:19.755442+0000","last_unstale":"2023-10-09T09:21:23.465163+0000","last_undegraded":"2023-10-09T09:21:23.465163+0000","last_fullsized":"2023-10-09T09:21:23.465163+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":200,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465251+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:21:23.465251+0000","last_peered":"2023-10-09T09:21:23.465251+0000","last_clean":"2023-10-09T09:21:23.465251+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:21:23.465251+0000","last_undegraded":"2023-10-09T09:21:23.465251+0000","last_fullsized":"2023-10-09T09:21:23.465251+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"321'9819","reported_seq":9856,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:37.184106+0000","last_change":"2023-10-09T09:18:10.917634+0000","last_active":"2023-10-09T09:21:37.184106+0000","last_peered":"2023-10-09T09:21:37.184106+0000","last_clean":"2023-10-09T09:21:37.184106+0000","last_became_active":"2023-10-09T09:18:10.914359+0000","last_became_peered":"2023-10-09T09:18:10.914359+0000","last_unstale":"2023-10-09T09:21:37.184106+0000","last_undegraded":"2023-10-09T09:21:37.184106+0000","last_fullsized":"2023-10-09T09:21:37.184106+0000","mapping_epoch":294,"log_start":"301'3000","ondisk_log_start":"301'3000","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:37.047604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129695744,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9819,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":70,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503828+0000","last_change":"2023-10-09T09:14:48.306639+0000","last_active":"2023-10-09T09:21:34.503828+0000","last_peered":"2023-10-09T09:21:34.503828+0000","last_clean":"2023-10-09T09:21:34.503828+0000","last_became_active":"2023-10-09T09:14:48.305729+0000","last_became_peered":"2023-10-09T09:14:48.305729+0000","last_unstale":"2023-10-09T09:21:34.503828+0000","last_undegraded":"2023-10-09T09:21:34.503828+0000","last_fullsized":"2023-10-09T09:21:34.503828+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:08.892989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":69,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507902+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:21:34.507902+0000","last_peered":"2023-10-09T09:21:34.507902+0000","last_clean":"2023-10-09T09:21:34.507902+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:21:34.507902+0000","last_undegraded":"2023-10-09T09:21:34.507902+0000","last_fullsized":"2023-10-09T09:21:34.507902+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"319'9374","reported_seq":11720,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:33.858505+0000","last_change":"2023-10-09T09:20:40.409450+0000","last_active":"2023-10-09T09:21:33.858505+0000","last_peered":"2023-10-09T09:21:33.858505+0000","last_clean":"2023-10-09T09:21:33.858505+0000","last_became_active":"2023-10-09T09:20:40.389612+0000","last_became_peered":"2023-10-09T09:20:40.389612+0000","last_unstale":"2023-10-09T09:21:33.858505+0000","last_undegraded":"2023-10-09T09:21:33.858505+0000","last_fullsized":"2023-10-09T09:21:33.858505+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":9374,"log_dups_size":0,"ondisk_log_size":9374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:24.706615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132055064,"num_objects":2016,"num_object_clones":0,"num_object_copies":6048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2016,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9374,"num_write_kb":364417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1140,"num_bytes_recovered":30867480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.523989+0000","last_change":"2023-10-09T09:14:48.299630+0000","last_active":"2023-10-09T09:21:34.523989+0000","last_peered":"2023-10-09T09:21:34.523989+0000","last_clean":"2023-10-09T09:21:34.523989+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:21:34.523989+0000","last_undegraded":"2023-10-09T09:21:34.523989+0000","last_fullsized":"2023-10-09T09:21:34.523989+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.510112+0000","last_change":"2023-10-09T09:20:19.757594+0000","last_active":"2023-10-09T09:21:34.510112+0000","last_peered":"2023-10-09T09:21:34.510112+0000","last_clean":"2023-10-09T09:21:34.510112+0000","last_became_active":"2023-10-09T09:20:19.757216+0000","last_became_peered":"2023-10-09T09:20:19.757216+0000","last_unstale":"2023-10-09T09:21:34.510112+0000","last_undegraded":"2023-10-09T09:21:34.510112+0000","last_fullsized":"2023-10-09T09:21:34.510112+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503347+0000","last_change":"2023-10-09T09:15:23.413972+0000","last_active":"2023-10-09T09:21:34.503347+0000","last_peered":"2023-10-09T09:21:34.503347+0000","last_clean":"2023-10-09T09:21:34.503347+0000","last_became_active":"2023-10-09T09:15:23.413531+0000","last_became_peered":"2023-10-09T09:15:23.413531+0000","last_unstale":"2023-10-09T09:21:34.503347+0000","last_undegraded":"2023-10-09T09:21:34.503347+0000","last_fullsized":"2023-10-09T09:21:34.503347+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:54.516595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"321'9510","reported_seq":9546,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:37.161256+0000","last_change":"2023-10-09T09:18:10.914938+0000","last_active":"2023-10-09T09:21:37.161256+0000","last_peered":"2023-10-09T09:21:37.161256+0000","last_clean":"2023-10-09T09:21:37.161256+0000","last_became_active":"2023-10-09T09:18:10.914146+0000","last_became_peered":"2023-10-09T09:18:10.914146+0000","last_unstale":"2023-10-09T09:21:37.161256+0000","last_undegraded":"2023-10-09T09:21:37.161256+0000","last_fullsized":"2023-10-09T09:21:37.161256+0000","mapping_epoch":294,"log_start":"298'2600","ondisk_log_start":"298'2600","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:15.765459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123469824,"num_objects":1884,"num_object_clones":0,"num_object_copies":5652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9510,"num_write_kb":364608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":118,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.509533+0000","last_change":"2023-10-09T09:14:48.319163+0000","last_active":"2023-10-09T09:21:34.509533+0000","last_peered":"2023-10-09T09:21:34.509533+0000","last_clean":"2023-10-09T09:21:34.509533+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:21:34.509533+0000","last_undegraded":"2023-10-09T09:21:34.509533+0000","last_fullsized":"2023-10-09T09:21:34.509533+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":182,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465848+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:21:23.465848+0000","last_peered":"2023-10-09T09:21:23.465848+0000","last_clean":"2023-10-09T09:21:23.465848+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:21:23.465848+0000","last_undegraded":"2023-10-09T09:21:23.465848+0000","last_fullsized":"2023-10-09T09:21:23.465848+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":97,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.523901+0000","last_change":"2023-10-09T09:20:19.932428+0000","last_active":"2023-10-09T09:21:34.523901+0000","last_peered":"2023-10-09T09:21:34.523901+0000","last_clean":"2023-10-09T09:21:34.523901+0000","last_became_active":"2023-10-09T09:20:19.931684+0000","last_became_peered":"2023-10-09T09:20:19.931684+0000","last_unstale":"2023-10-09T09:21:34.523901+0000","last_undegraded":"2023-10-09T09:21:34.523901+0000","last_fullsized":"2023-10-09T09:21:34.523901+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:40.774870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"321'9598","reported_seq":9633,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:35.682493+0000","last_change":"2023-10-09T09:18:10.915886+0000","last_active":"2023-10-09T09:21:35.682493+0000","last_peered":"2023-10-09T09:21:35.682493+0000","last_clean":"2023-10-09T09:21:35.682493+0000","last_became_active":"2023-10-09T09:18:10.915159+0000","last_became_peered":"2023-10-09T09:18:10.915159+0000","last_unstale":"2023-10-09T09:21:35.682493+0000","last_undegraded":"2023-10-09T09:21:35.682493+0000","last_fullsized":"2023-10-09T09:21:35.682493+0000","mapping_epoch":294,"log_start":"298'1200","ondisk_log_start":"298'1200","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8398,"log_dups_size":0,"ondisk_log_size":8398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130678784,"num_objects":1994,"num_object_clones":0,"num_object_copies":5982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9598,"num_write_kb":370944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":22,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T09:21:22.403250+0000","last_change":"2023-10-09T09:20:19.757802+0000","last_active":"2023-10-09T09:21:22.403250+0000","last_peered":"2023-10-09T09:21:22.403250+0000","last_clean":"2023-10-09T09:21:22.403250+0000","last_became_active":"2023-10-09T09:20:19.757450+0000","last_became_peered":"2023-10-09T09:20:19.757450+0000","last_unstale":"2023-10-09T09:21:22.403250+0000","last_undegraded":"2023-10-09T09:21:22.403250+0000","last_fullsized":"2023-10-09T09:21:22.403250+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:57.053230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":97,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503493+0000","last_change":"2023-10-09T09:20:19.902503+0000","last_active":"2023-10-09T09:21:34.503493+0000","last_peered":"2023-10-09T09:21:34.503493+0000","last_clean":"2023-10-09T09:21:34.503493+0000","last_became_active":"2023-10-09T09:20:19.902131+0000","last_became_peered":"2023-10-09T09:20:19.902131+0000","last_unstale":"2023-10-09T09:21:34.503493+0000","last_undegraded":"2023-10-09T09:21:34.503493+0000","last_fullsized":"2023-10-09T09:21:34.503493+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:43.897708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":70,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507836+0000","last_change":"2023-10-09T09:14:48.321029+0000","last_active":"2023-10-09T09:21:34.507836+0000","last_peered":"2023-10-09T09:21:34.507836+0000","last_clean":"2023-10-09T09:21:34.507836+0000","last_became_active":"2023-10-09T09:14:48.320435+0000","last_became_peered":"2023-10-09T09:14:48.320435+0000","last_unstale":"2023-10-09T09:21:34.507836+0000","last_undegraded":"2023-10-09T09:21:34.507836+0000","last_fullsized":"2023-10-09T09:21:34.507836+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:20.309472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"321'9730","reported_seq":9765,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:35.725254+0000","last_change":"2023-10-09T09:18:10.917223+0000","last_active":"2023-10-09T09:21:35.725254+0000","last_peered":"2023-10-09T09:21:35.725254+0000","last_clean":"2023-10-09T09:21:35.725254+0000","last_became_active":"2023-10-09T09:18:10.916265+0000","last_became_peered":"2023-10-09T09:18:10.916265+0000","last_unstale":"2023-10-09T09:21:35.725254+0000","last_undegraded":"2023-10-09T09:21:35.725254+0000","last_fullsized":"2023-10-09T09:21:35.725254+0000","mapping_epoch":294,"log_start":"298'1300","ondisk_log_start":"298'1300","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8430,"log_dups_size":0,"ondisk_log_size":8430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133169152,"num_objects":2032,"num_object_clones":0,"num_object_copies":6096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9730,"num_write_kb":376384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":154,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.509739+0000","last_change":"2023-10-09T09:20:19.759670+0000","last_active":"2023-10-09T09:21:34.509739+0000","last_peered":"2023-10-09T09:21:34.509739+0000","last_clean":"2023-10-09T09:21:34.509739+0000","last_became_active":"2023-10-09T09:20:19.759254+0000","last_became_peered":"2023-10-09T09:20:19.759254+0000","last_unstale":"2023-10-09T09:21:34.509739+0000","last_undegraded":"2023-10-09T09:21:34.509739+0000","last_fullsized":"2023-10-09T09:21:34.509739+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":559,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.506594+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:21:34.506594+0000","last_peered":"2023-10-09T09:21:34.506594+0000","last_clean":"2023-10-09T09:21:34.506594+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:21:34.506594+0000","last_undegraded":"2023-10-09T09:21:34.506594+0000","last_fullsized":"2023-10-09T09:21:34.506594+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":148,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.509783+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:21:34.509783+0000","last_peered":"2023-10-09T09:21:34.509783+0000","last_clean":"2023-10-09T09:21:34.509783+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:21:34.509783+0000","last_undegraded":"2023-10-09T09:21:34.509783+0000","last_fullsized":"2023-10-09T09:21:34.509783+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":179,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507565+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:21:34.507565+0000","last_peered":"2023-10-09T09:21:34.507565+0000","last_clean":"2023-10-09T09:21:34.507565+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:21:34.507565+0000","last_undegraded":"2023-10-09T09:21:34.507565+0000","last_fullsized":"2023-10-09T09:21:34.507565+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"321'9733","reported_seq":9768,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:35.764800+0000","last_change":"2023-10-09T09:18:10.915823+0000","last_active":"2023-10-09T09:21:35.764800+0000","last_peered":"2023-10-09T09:21:35.764800+0000","last_clean":"2023-10-09T09:21:35.764800+0000","last_became_active":"2023-10-09T09:18:10.915054+0000","last_became_peered":"2023-10-09T09:18:10.915054+0000","last_unstale":"2023-10-09T09:21:35.764800+0000","last_undegraded":"2023-10-09T09:21:35.764800+0000","last_fullsized":"2023-10-09T09:21:35.764800+0000","mapping_epoch":294,"log_start":"298'1300","ondisk_log_start":"298'1300","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8433,"log_dups_size":0,"ondisk_log_size":8433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9733,"num_write_kb":376256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":70,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.503635+0000","last_change":"2023-10-09T09:14:48.301697+0000","last_active":"2023-10-09T09:21:34.503635+0000","last_peered":"2023-10-09T09:21:34.503635+0000","last_clean":"2023-10-09T09:21:34.503635+0000","last_became_active":"2023-10-09T09:14:48.301379+0000","last_became_peered":"2023-10-09T09:14:48.301379+0000","last_unstale":"2023-10-09T09:21:34.503635+0000","last_undegraded":"2023-10-09T09:21:34.503635+0000","last_fullsized":"2023-10-09T09:21:34.503635+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:11.963433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":358,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.497703+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:21:34.497703+0000","last_peered":"2023-10-09T09:21:34.497703+0000","last_clean":"2023-10-09T09:21:34.497703+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:21:34.497703+0000","last_undegraded":"2023-10-09T09:21:34.497703+0000","last_fullsized":"2023-10-09T09:21:34.497703+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":381,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.500106+0000","last_change":"2023-10-09T09:20:19.890699+0000","last_active":"2023-10-09T09:21:34.500106+0000","last_peered":"2023-10-09T09:21:34.500106+0000","last_clean":"2023-10-09T09:21:34.500106+0000","last_became_active":"2023-10-09T09:20:19.889949+0000","last_became_peered":"2023-10-09T09:20:19.889949+0000","last_unstale":"2023-10-09T09:21:34.500106+0000","last_undegraded":"2023-10-09T09:21:34.500106+0000","last_fullsized":"2023-10-09T09:21:34.500106+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:59.920953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"321'9680","reported_seq":10337,"reported_epoch":321,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T09:21:37.184173+0000","last_change":"2023-10-09T09:21:33.853554+0000","last_active":"2023-10-09T09:21:37.184173+0000","last_peered":"2023-10-09T09:21:37.184173+0000","last_clean":"2023-10-09T09:19:41.829162+0000","last_became_active":"2023-10-09T09:20:20.107684+0000","last_became_peered":"2023-10-09T09:20:20.107684+0000","last_unstale":"2023-10-09T09:21:37.184173+0000","last_undegraded":"2023-10-09T09:21:37.184173+0000","last_fullsized":"2023-10-09T09:20:19.748480+0000","mapping_epoch":307,"log_start":"300'2800","ondisk_log_start":"300'2800","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6880,"log_dups_size":0,"ondisk_log_size":6880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:27.199008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128319488,"num_objects":1958,"num_object_clones":0,"num_object_copies":5874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9680,"num_write_kb":372416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":303,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1958}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":101,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507716+0000","last_change":"2023-10-09T09:09:44.439090+0000","last_active":"2023-10-09T09:21:34.507716+0000","last_peered":"2023-10-09T09:21:34.507716+0000","last_clean":"2023-10-09T09:21:34.507716+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:21:34.507716+0000","last_undegraded":"2023-10-09T09:21:34.507716+0000","last_fullsized":"2023-10-09T09:21:34.507716+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":97,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.523683+0000","last_change":"2023-10-09T09:20:19.912177+0000","last_active":"2023-10-09T09:21:34.523683+0000","last_peered":"2023-10-09T09:21:34.523683+0000","last_clean":"2023-10-09T09:21:34.523683+0000","last_became_active":"2023-10-09T09:20:19.908966+0000","last_became_peered":"2023-10-09T09:20:19.908966+0000","last_unstale":"2023-10-09T09:21:34.523683+0000","last_undegraded":"2023-10-09T09:21:34.523683+0000","last_fullsized":"2023-10-09T09:21:34.523683+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:54.778943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":334,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.498149+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:21:34.498149+0000","last_peered":"2023-10-09T09:21:34.498149+0000","last_clean":"2023-10-09T09:21:34.498149+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:21:34.498149+0000","last_undegraded":"2023-10-09T09:21:34.498149+0000","last_fullsized":"2023-10-09T09:21:34.498149+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"321'10012","reported_seq":13774,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-10-09T09:21:37.185178+0000","last_change":"2023-10-09T09:21:23.439146+0000","last_active":"2023-10-09T09:21:37.185178+0000","last_peered":"2023-10-09T09:21:37.185178+0000","last_clean":"2023-10-09T09:21:37.185178+0000","last_became_active":"2023-10-09T09:21:23.402629+0000","last_became_peered":"2023-10-09T09:21:23.402629+0000","last_unstale":"2023-10-09T09:21:37.185178+0000","last_undegraded":"2023-10-09T09:21:37.185178+0000","last_fullsized":"2023-10-09T09:21:37.185178+0000","mapping_epoch":318,"log_start":"301'3100","ondisk_log_start":"301'3100","created":294,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6912,"log_dups_size":0,"ondisk_log_size":6912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:17.924211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134348800,"num_objects":2050,"num_object_clones":0,"num_object_copies":6150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10012,"num_write_kb":385984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1850,"num_bytes_recovered":73662464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":90,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507513+0000","last_change":"2023-10-09T09:14:19.749169+0000","last_active":"2023-10-09T09:21:34.507513+0000","last_peered":"2023-10-09T09:21:34.507513+0000","last_clean":"2023-10-09T09:21:34.507513+0000","last_became_active":"2023-10-09T09:14:19.740888+0000","last_became_peered":"2023-10-09T09:14:19.740888+0000","last_unstale":"2023-10-09T09:21:34.507513+0000","last_undegraded":"2023-10-09T09:21:34.507513+0000","last_fullsized":"2023-10-09T09:21:34.507513+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:01.361061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":70,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.505858+0000","last_change":"2023-10-09T09:14:48.317228+0000","last_active":"2023-10-09T09:21:34.505858+0000","last_peered":"2023-10-09T09:21:34.505858+0000","last_clean":"2023-10-09T09:21:34.505858+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:21:34.505858+0000","last_undegraded":"2023-10-09T09:21:34.505858+0000","last_fullsized":"2023-10-09T09:21:34.505858+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":101,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.506447+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:21:34.506447+0000","last_peered":"2023-10-09T09:21:34.506447+0000","last_clean":"2023-10-09T09:21:34.506447+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:21:34.506447+0000","last_undegraded":"2023-10-09T09:21:34.506447+0000","last_fullsized":"2023-10-09T09:21:34.506447+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"320'9719","reported_seq":9753,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.556570+0000","last_change":"2023-10-09T09:18:10.373277+0000","last_active":"2023-10-09T09:21:34.556570+0000","last_peered":"2023-10-09T09:21:34.556570+0000","last_clean":"2023-10-09T09:21:34.556570+0000","last_became_active":"2023-10-09T09:18:10.372954+0000","last_became_peered":"2023-10-09T09:18:10.372954+0000","last_unstale":"2023-10-09T09:21:34.556570+0000","last_undegraded":"2023-10-09T09:21:34.556570+0000","last_fullsized":"2023-10-09T09:21:34.556570+0000","mapping_epoch":294,"log_start":"298'2000","ondisk_log_start":"298'2000","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136642560,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9719,"num_write_kb":377728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":215,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507381+0000","last_change":"2023-10-09T09:20:19.758027+0000","last_active":"2023-10-09T09:21:34.507381+0000","last_peered":"2023-10-09T09:21:34.507381+0000","last_clean":"2023-10-09T09:21:34.507381+0000","last_became_active":"2023-10-09T09:20:19.757552+0000","last_became_peered":"2023-10-09T09:20:19.757552+0000","last_unstale":"2023-10-09T09:21:34.507381+0000","last_undegraded":"2023-10-09T09:21:34.507381+0000","last_fullsized":"2023-10-09T09:21:34.507381+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":376,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.500209+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:21:34.500209+0000","last_peered":"2023-10-09T09:21:34.500209+0000","last_clean":"2023-10-09T09:21:34.500209+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:21:34.500209+0000","last_undegraded":"2023-10-09T09:21:34.500209+0000","last_fullsized":"2023-10-09T09:21:34.500209+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":200,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.464999+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:21:23.464999+0000","last_peered":"2023-10-09T09:21:23.464999+0000","last_clean":"2023-10-09T09:21:23.464999+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:21:23.464999+0000","last_undegraded":"2023-10-09T09:21:23.464999+0000","last_fullsized":"2023-10-09T09:21:23.464999+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":344,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.498076+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:21:34.498076+0000","last_peered":"2023-10-09T09:21:34.498076+0000","last_clean":"2023-10-09T09:21:34.498076+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:21:34.498076+0000","last_undegraded":"2023-10-09T09:21:34.498076+0000","last_fullsized":"2023-10-09T09:21:34.498076+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T09:21:22.405745+0000","last_change":"2023-10-09T09:20:19.916151+0000","last_active":"2023-10-09T09:21:22.405745+0000","last_peered":"2023-10-09T09:21:22.405745+0000","last_clean":"2023-10-09T09:21:22.405745+0000","last_became_active":"2023-10-09T09:20:19.915607+0000","last_became_peered":"2023-10-09T09:20:19.915607+0000","last_unstale":"2023-10-09T09:21:22.405745+0000","last_undegraded":"2023-10-09T09:21:22.405745+0000","last_fullsized":"2023-10-09T09:21:22.405745+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:57:51.697949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":126,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.509909+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:21:34.509909+0000","last_peered":"2023-10-09T09:21:34.509909+0000","last_clean":"2023-10-09T09:21:34.509909+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:21:34.509909+0000","last_undegraded":"2023-10-09T09:21:34.509909+0000","last_fullsized":"2023-10-09T09:21:34.509909+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":200,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.465639+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:21:23.465639+0000","last_peered":"2023-10-09T09:21:23.465639+0000","last_clean":"2023-10-09T09:21:23.465639+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:21:23.465639+0000","last_undegraded":"2023-10-09T09:21:23.465639+0000","last_fullsized":"2023-10-09T09:21:23.465639+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":113,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.510029+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:21:34.510029+0000","last_peered":"2023-10-09T09:21:34.510029+0000","last_clean":"2023-10-09T09:21:34.510029+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:21:34.510029+0000","last_undegraded":"2023-10-09T09:21:34.510029+0000","last_fullsized":"2023-10-09T09:21:34.510029+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":182,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.464802+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:21:23.464802+0000","last_peered":"2023-10-09T09:21:23.464802+0000","last_clean":"2023-10-09T09:21:23.464802+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:21:23.464802+0000","last_undegraded":"2023-10-09T09:21:23.464802+0000","last_fullsized":"2023-10-09T09:21:23.464802+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":302,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507154+0000","last_change":"2023-10-09T09:20:19.760410+0000","last_active":"2023-10-09T09:21:34.507154+0000","last_peered":"2023-10-09T09:21:34.507154+0000","last_clean":"2023-10-09T09:21:34.507154+0000","last_became_active":"2023-10-09T09:20:19.759916+0000","last_became_peered":"2023-10-09T09:20:19.759916+0000","last_unstale":"2023-10-09T09:21:34.507154+0000","last_undegraded":"2023-10-09T09:21:34.507154+0000","last_fullsized":"2023-10-09T09:21:34.507154+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":22,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T09:21:22.403113+0000","last_change":"2023-10-09T09:20:19.759403+0000","last_active":"2023-10-09T09:21:22.403113+0000","last_peered":"2023-10-09T09:21:22.403113+0000","last_clean":"2023-10-09T09:21:22.403113+0000","last_became_active":"2023-10-09T09:20:19.759030+0000","last_became_peered":"2023-10-09T09:20:19.759030+0000","last_unstale":"2023-10-09T09:21:22.403113+0000","last_undegraded":"2023-10-09T09:21:22.403113+0000","last_fullsized":"2023-10-09T09:21:22.403113+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:10.903233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":464,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-09T09:21:34.507337+0000","last_change":"2023-10-09T09:20:19.788084+0000","last_active":"2023-10-09T09:21:34.507337+0000","last_peered":"2023-10-09T09:21:34.507337+0000","last_clean":"2023-10-09T09:21:34.507337+0000","last_became_active":"2023-10-09T09:20:19.762112+0000","last_became_peered":"2023-10-09T09:20:19.762112+0000","last_unstale":"2023-10-09T09:21:34.507337+0000","last_undegraded":"2023-10-09T09:21:34.507337+0000","last_fullsized":"2023-10-09T09:21:34.507337+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:47.406781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":209,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T09:21:23.464621+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:21:23.464621+0000","last_peered":"2023-10-09T09:21:23.464621+0000","last_clean":"2023-10-09T09:21:23.464621+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:21:23.464621+0000","last_undegraded":"2023-10-09T09:21:23.464621+0000","last_fullsized":"2023-10-09T09:21:23.464621+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2086076440,"num_objects":31832,"num_object_clones":0,"num_object_copies":95496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31832,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":154440,"num_write_kb":5960641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6024,"num_bytes_recovered":156106776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6369132544,"data_stored":6369116256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125940,"ondisk_log_size":125940,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":97,"num_bytes_recovered":20970,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430813,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1698328,"kb_used_data":790928,"kb_used_omap":0,"kb_used_meta":907392,"kb_avail":92046824,"statfs":{"total":95995035648,"available":94255947776,"internally_reserved":0,"allocated":809910272,"data_stored":809312241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929169408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.927,"15min":2.077},"min":{"1min":0.248,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":15.178000000000001,"5min":50.515999999999998,"15min":51.328000000000003},"last":0.92700000000000005},{"interface":"front","average":{"1min":1.982,"5min":2.153,"15min":2.0510000000000002},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28799999999999998},"max":{"1min":15.199,"5min":50.621000000000002,"15min":51.429000000000002},"last":0.89400000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:21:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.4139999999999999,"15min":1.3600000000000001},"min":{"1min":0.34200000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.4290000000000003,"5min":7.9740000000000002,"15min":21.800999999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.665,"15min":1.2589999999999999},"min":{"1min":0.36499999999999999,"5min":0.35499999999999998,"15min":0.316},"max":{"1min":4.4370000000000003,"5min":15.564,"15min":21.734000000000002},"last":0.79800000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.4630000000000001,"15min":1.659},"min":{"1min":0.371,"5min":0.32800000000000001,"15min":0.31},"max":{"1min":4.577,"5min":11.186,"15min":45.345999999999997},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.1339999999999999,"15min":1.6240000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.30599999999999999},"max":{"1min":5.6669999999999998,"5min":8.0709999999999997,"15min":46.228000000000002},"last":0.72099999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:21:26 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":1.764,"15min":2.1059999999999999},"min":{"1min":0.41999999999999998,"5min":0.38300000000000001,"15min":0.316},"max":{"1min":6.0099999999999998,"5min":18.178999999999998,"15min":62.060000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.268,"5min":1.448,"15min":2.1259999999999999},"min":{"1min":0.38600000000000001,"5min":0.34300000000000003,"15min":0.318},"max":{"1min":4.2789999999999999,"5min":17.931999999999999,"15min":63.268999999999998},"last":1.0780000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:21:13 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.179,"15min":1.4870000000000001},"min":{"1min":0.38400000000000001,"5min":0.32000000000000001,"15min":0.28699999999999998},"max":{"1min":4.4180000000000001,"5min":8.0809999999999995,"15min":13.895},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.361,"5min":1.139,"15min":1.5009999999999999},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.6669999999999998,"5min":8.0969999999999995,"15min":16.433},"last":1.1419999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:21:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.3819999999999999,"15min":1.339},"min":{"1min":0.42199999999999999,"5min":0.39800000000000002,"15min":0.28199999999999997},"max":{"1min":3.7069999999999999,"5min":15.616,"15min":15.616},"last":1.018},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.464,"15min":1.667},"min":{"1min":0.41099999999999998,"5min":0.38,"15min":0.34499999999999997},"max":{"1min":6.8029999999999999,"5min":11.818,"15min":18.501000000000001},"last":1.1220000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:21:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":1.6659999999999999,"15min":1.673},"min":{"1min":0.42999999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.898,"5min":12.898,"15min":21.145},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.51,"5min":1.4510000000000001,"15min":1.7090000000000001},"min":{"1min":0.50600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.0779999999999994,"5min":12.269,"15min":19.068999999999999},"last":1.1930000000000001}]}]},{"osd":6,"up_from":42,"seq":180388627046,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2231120,"kb_used_data":1039304,"kb_used_omap":0,"kb_used_meta":1191744,"kb_avail":91514032,"statfs":{"total":95995035648,"available":93710368768,"internally_reserved":0,"allocated":1064247296,"data_stored":1063657285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1220345856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":2.5,"15min":2.819},"min":{"1min":0.31900000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.282,"5min":120.812,"15min":120.812},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.262,"15min":2.7290000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24099999999999999},"max":{"1min":5.1520000000000001,"5min":120.857,"15min":120.857},"last":0.437}]},{"osd":1,"last update":"Mon Oct 9 09:21:21 2023","interfaces":[{"interface":"back","average":{"1min":10.323,"5min":3.4329999999999998,"15min":2.1760000000000002},"min":{"1min":0.37,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":142.31100000000001,"5min":142.31100000000001,"15min":142.31100000000001},"last":5.0570000000000004},{"interface":"front","average":{"1min":9.5890000000000004,"5min":3.4329999999999998,"15min":2.0489999999999999},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.25700000000000001},"max":{"1min":132.48099999999999,"5min":132.48099999999999,"15min":132.48099999999999},"last":7.6260000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":2.1560000000000001,"15min":1.923},"min":{"1min":0.33600000000000002,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":7.431,"5min":82.170000000000002,"15min":82.170000000000002},"last":3.2970000000000002},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.921,"15min":1.857},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8490000000000002,"5min":82.352000000000004,"15min":82.352000000000004},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:21:21 2023","interfaces":[{"interface":"back","average":{"1min":4.2370000000000001,"5min":1.948,"15min":1.8149999999999999},"min":{"1min":0.53100000000000003,"5min":0.371,"15min":0.28999999999999998},"max":{"1min":14.525,"5min":14.525,"15min":16.977},"last":1.474},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.339,"15min":1.599},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":6.1440000000000001,"5min":6.1440000000000001,"15min":20.573},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:21:07 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":1.3720000000000001,"15min":1.6479999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.645,"5min":15.818,"15min":29.585999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.298,"15min":1.6479999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":6.0220000000000002,"5min":7.2949999999999999,"15min":20.318000000000001},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 09:21:07 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.2629999999999999,"15min":1.7},"min":{"1min":0.312,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":2.6120000000000001,"5min":11.567,"15min":18.103000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.7829999999999999,"5min":1.282,"15min":1.639},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":6.1509999999999998,"5min":6.1509999999999998,"15min":13.609},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.3100000000000001,"15min":1.7629999999999999},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.33700000000000002},"max":{"1min":3.875,"5min":11.597,"15min":29.530000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.504,"15min":1.5780000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.36199999999999999},"max":{"1min":7.9489999999999998,"5min":11.477,"15min":13.523999999999999},"last":0.46700000000000003}]}]},{"osd":1,"up_from":83,"seq":356482285994,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1632128,"kb_used_data":1018232,"kb_used_omap":9,"kb_used_meta":613878,"kb_avail":92113024,"statfs":{"total":95995035648,"available":94323736576,"internally_reserved":0,"allocated":1042669568,"data_stored":1041516142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":628612073},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":4.0659999999999998,"15min":2.516},"min":{"1min":0.47499999999999998,"5min":0.215,"15min":0.189},"max":{"1min":4.0090000000000003,"5min":133.76499999999999,"15min":133.76499999999999},"last":0.76100000000000001},{"interface":"front","average":{"1min":2.5960000000000001,"5min":2.7909999999999999,"15min":2.3359999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.23300000000000001},"max":{"1min":10.225,"5min":127.395,"15min":127.395},"last":0.40699999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:20:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":3.4460000000000002,"15min":1.7809999999999999},"min":{"1min":0.45000000000000001,"5min":0.28100000000000003,"15min":0.23799999999999999},"max":{"1min":5.0899999999999999,"5min":236.29599999999999,"15min":236.29599999999999},"last":4.4219999999999997},{"interface":"front","average":{"1min":2.5710000000000002,"5min":3.355,"15min":1.593},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.252},"max":{"1min":11.423,"5min":236.21000000000001,"15min":236.21000000000001},"last":14.638999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":3.0750000000000002,"15min":2.165},"min":{"1min":0.308,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":8.2810000000000006,"5min":152.637,"15min":152.637},"last":14.673999999999999},{"interface":"front","average":{"1min":2.3799999999999999,"5min":4.9509999999999996,"15min":2.7850000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.22600000000000001},"max":{"1min":7.6870000000000003,"5min":152.374,"15min":152.374},"last":0.86399999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":1.468,"15min":1.3839999999999999},"min":{"1min":0.40300000000000002,"5min":0.38,"15min":0.376},"max":{"1min":8.1470000000000002,"5min":8.3059999999999992,"15min":8.3059999999999992},"last":14.56},{"interface":"front","average":{"1min":1.829,"5min":3.0670000000000002,"15min":1.837},"min":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.2190000000000003,"5min":133.702,"15min":133.702},"last":0.79900000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":1.3500000000000001,"15min":1.282},"min":{"1min":0.441,"5min":0.432,"15min":0.33800000000000002},"max":{"1min":6.0060000000000002,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.3440000000000001,"15min":1.2130000000000001},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.2100000000000009,"5min":8.2100000000000009,"15min":8.2100000000000009},"last":14.464}]},{"osd":6,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":3.1779999999999999,"15min":2.0209999999999999},"min":{"1min":0.42999999999999999,"5min":0.40100000000000002,"15min":0.33200000000000002},"max":{"1min":3.6859999999999999,"5min":133.739,"15min":133.739},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.319,"15min":1.4399999999999999},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.377},"max":{"1min":3.9689999999999999,"5min":7.3040000000000003,"15min":8.9079999999999995},"last":1.0269999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.633,"15min":1.3919999999999999},"min":{"1min":0.46899999999999997,"5min":0.41499999999999998,"15min":0.38500000000000001},"max":{"1min":8.2469999999999999,"5min":19.861999999999998,"15min":19.861999999999998},"last":14.529},{"interface":"front","average":{"1min":1.7889999999999999,"5min":3.1789999999999998,"15min":1.95},"min":{"1min":0.47599999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":3.7040000000000002,"5min":133.66499999999999,"15min":133.66499999999999},"last":0.84199999999999997}]}]},{"osd":0,"up_from":78,"seq":335007449520,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1736096,"kb_used_data":773784,"kb_used_omap":28,"kb_used_meta":962275,"kb_avail":92009056,"statfs":{"total":95995035648,"available":94217273344,"internally_reserved":0,"allocated":792354816,"data_stored":791172311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":985370448},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.323,"15min":1.129},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.193},"max":{"1min":4.8049999999999997,"5min":6.0519999999999996,"15min":9.7620000000000005},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.5939999999999999,"5min":1.2370000000000001,"15min":1.0569999999999999},"min":{"1min":0.23000000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":11.456,"5min":11.456,"15min":11.456},"last":0.438}]},{"osd":2,"last update":"Mon Oct 9 09:21:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.321,"15min":1.8740000000000001},"min":{"1min":0.33600000000000002,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":9.3219999999999992,"5min":9.3219999999999992,"15min":74.200999999999993},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.206,"5min":1.254,"15min":1.9490000000000001},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.8930000000000007,"5min":8.8930000000000007,"15min":73.983999999999995},"last":0.39800000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:21:15 2023","interfaces":[{"interface":"back","average":{"1min":2.4790000000000001,"5min":1.534,"15min":1.7270000000000001},"min":{"1min":0.436,"5min":0.32400000000000001,"15min":0.29299999999999998},"max":{"1min":7.25,"5min":9.2569999999999997,"15min":20.364000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":3.0510000000000002,"5min":1.335,"15min":1.958},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.29599999999999999},"max":{"1min":9.9269999999999996,"5min":9.9269999999999996,"15min":137.94999999999999},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.347,"15min":1.631},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":8.8040000000000003,"5min":8.8040000000000003,"15min":14.163},"last":0.73999999999999999},{"interface":"front","average":{"1min":2.2149999999999999,"5min":1.736,"15min":1.8640000000000001},"min":{"1min":0.316,"5min":0.314,"15min":0.314},"max":{"1min":9.5359999999999996,"5min":11.346,"15min":11.346},"last":0.91400000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.3620000000000001,"15min":1.579},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":4.7389999999999999,"5min":9.6739999999999995,"15min":9.9770000000000003},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.823,"5min":1.325,"15min":1.5149999999999999},"min":{"1min":0.44,"5min":0.35499999999999998,"15min":0.35299999999999998},"max":{"1min":8.8279999999999994,"5min":8.8279999999999994,"15min":16.364000000000001},"last":0.62}]},{"osd":6,"last update":"Mon Oct 9 09:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":1.532,"15min":2.0329999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.39000000000000001},"max":{"1min":9.2840000000000007,"5min":11.314,"15min":98.864000000000004},"last":0.99299999999999999},{"interface":"front","average":{"1min":2.6619999999999999,"5min":1.5569999999999999,"15min":2.1179999999999999},"min":{"1min":0.502,"5min":0.39400000000000002,"15min":0.38300000000000001},"max":{"1min":16.911000000000001,"5min":16.911000000000001,"15min":98.878},"last":0.96699999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:21:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":1.5129999999999999,"15min":1.587},"min":{"1min":0.54800000000000004,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":8.8640000000000008,"5min":8.8640000000000008,"15min":9.6780000000000008},"last":0.871},{"interface":"front","average":{"1min":2.637,"5min":1.776,"15min":1.879},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":9.5850000000000009,"5min":9.6859999999999999,"15min":10.734},"last":0.88200000000000001}]}]},{"osd":2,"up_from":88,"seq":377957122467,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1756572,"kb_used_data":857940,"kb_used_omap":81,"kb_used_meta":898478,"kb_avail":91988580,"statfs":{"total":95995035648,"available":94196305920,"internally_reserved":0,"allocated":878530560,"data_stored":877336171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":920041690},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":1.2090000000000001,"15min":1.694},"min":{"1min":0.27800000000000002,"5min":0.23899999999999999,"15min":0.188},"max":{"1min":8.6199999999999992,"5min":8.6199999999999992,"15min":85.091999999999999},"last":1.887},{"interface":"front","average":{"1min":1.736,"5min":1.1100000000000001,"15min":1.6699999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22700000000000001},"max":{"1min":5.6870000000000003,"5min":5.6870000000000003,"15min":84.373999999999995},"last":0.32100000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:21:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0310000000000001,"5min":1.5109999999999999,"15min":1.7150000000000001},"min":{"1min":0.28199999999999997,"5min":0.245,"15min":0.214},"max":{"1min":16.241,"5min":16.241,"15min":122.367},"last":4.2089999999999996},{"interface":"front","average":{"1min":3.5019999999999998,"5min":1.5940000000000001,"15min":1.6850000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":16.948,"5min":16.948,"15min":122.777},"last":1.3779999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:21:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":2.6040000000000001,"15min":2.0859999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.5640000000000001,"5min":152.19999999999999,"15min":152.19999999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":4.0430000000000001,"5min":3.246,"15min":2.27},"min":{"1min":0.311,"5min":0.311,"15min":0.20100000000000001},"max":{"1min":16.931000000000001,"5min":153.70400000000001,"15min":153.70400000000001},"last":2.9390000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:21:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2890000000000001,"5min":1.276,"15min":1.472},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.1719999999999997,"5min":7.1719999999999997,"15min":14.343999999999999},"last":2.976},{"interface":"front","average":{"1min":1.593,"5min":1.2330000000000001,"15min":1.516},"min":{"1min":0.38300000000000001,"5min":0.35799999999999998,"15min":0.29699999999999999},"max":{"1min":6.8600000000000003,"5min":6.8600000000000003,"15min":12.351000000000001},"last":1.784}]},{"osd":5,"last update":"Mon Oct 9 09:21:11 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":1.21,"15min":1.5429999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35799999999999998},"max":{"1min":7.8079999999999998,"5min":7.8079999999999998,"15min":72.311000000000007},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.355,"15min":1.7789999999999999},"min":{"1min":0.41099999999999998,"5min":0.39700000000000002,"15min":0.36899999999999999},"max":{"1min":7.2080000000000002,"5min":7.2080000000000002,"15min":73.042000000000002},"last":3.0289999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:21:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":1.5640000000000001,"15min":1.635},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.34300000000000003},"max":{"1min":8.5730000000000004,"5min":9.3399999999999999,"15min":14.714},"last":1.8180000000000001},{"interface":"front","average":{"1min":1.984,"5min":1.804,"15min":1.681},"min":{"1min":0.44900000000000001,"5min":0.432,"15min":0.40999999999999998},"max":{"1min":7.7389999999999999,"5min":8.5670000000000002,"15min":14.571999999999999},"last":0.495}]},{"osd":7,"last update":"Mon Oct 9 09:21:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.635,"15min":1.5529999999999999},"min":{"1min":0.46300000000000002,"5min":0.41599999999999998,"15min":0.40200000000000002},"max":{"1min":6.9059999999999997,"5min":29.513999999999999,"15min":29.513999999999999},"last":1.855},{"interface":"front","average":{"1min":1.577,"5min":1.6910000000000001,"15min":1.526},"min":{"1min":0.45200000000000001,"5min":0.44400000000000001,"15min":0.40200000000000002},"max":{"1min":7.8460000000000001,"5min":30.216999999999999,"15min":30.216999999999999},"last":0.42999999999999999}]}]},{"osd":3,"up_from":93,"seq":399431958942,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1961388,"kb_used_data":1033060,"kb_used_omap":7,"kb_used_meta":928312,"kb_avail":91783764,"statfs":{"total":95995035648,"available":93986574336,"internally_reserved":0,"allocated":1057853440,"data_stored":1056710447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":950592104},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.337,"15min":1.9410000000000001},"min":{"1min":0.22900000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.6230000000000002,"5min":10.022,"15min":130.45500000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.718,"5min":1.6599999999999999,"15min":2.262},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.6769999999999996,"5min":8.6769999999999996,"15min":129.74799999999999},"last":0.24199999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":1.629,"15min":1.3009999999999999},"min":{"1min":0.309,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.0259999999999998,"5min":9.7029999999999994,"15min":32.520000000000003},"last":0.85199999999999998},{"interface":"front","average":{"1min":2.2749999999999999,"5min":1.3720000000000001,"15min":1.1679999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.22900000000000001},"max":{"1min":9.6310000000000002,"5min":9.6310000000000002,"15min":32.585999999999999},"last":0.58499999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.724,"15min":1.8839999999999999},"min":{"1min":0.377,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.4640000000000004,"5min":37.997,"15min":129.67699999999999},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.8380000000000001,"5min":1.748,"15min":1.9610000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":5.8129999999999997,"5min":37.499000000000002,"15min":130.40000000000001},"last":0.90700000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.292,"15min":1.359},"min":{"1min":0.35399999999999998,"5min":0.34599999999999997,"15min":0.32200000000000001},"max":{"1min":5.2990000000000004,"5min":7.5519999999999996,"15min":7.5519999999999996},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.827,"5min":1.5620000000000001,"15min":1.73},"min":{"1min":0.35599999999999998,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":8.4510000000000005,"5min":8.4510000000000005,"15min":12.617000000000001},"last":0.48599999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.607,"15min":1.716},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.5540000000000003,"5min":8.5540000000000003,"15min":16.132999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.274,"15min":1.5580000000000001},"min":{"1min":0.442,"5min":0.39800000000000002,"15min":0.36899999999999999},"max":{"1min":4.7889999999999997,"5min":5.4370000000000003,"15min":14.361000000000001},"last":2.9609999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":1.71,"15min":1.909},"min":{"1min":0.39600000000000002,"5min":0.38500000000000001,"15min":0.37},"max":{"1min":8.5920000000000005,"5min":8.5920000000000005,"15min":12.105},"last":5.5350000000000001},{"interface":"front","average":{"1min":1.9970000000000001,"5min":1.655,"15min":1.681},"min":{"1min":0.47099999999999997,"5min":0.40899999999999997,"15min":0.38500000000000001},"max":{"1min":5.7789999999999999,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":5.5060000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":1.6970000000000001,"15min":1.589},"min":{"1min":0.49399999999999999,"5min":0.434,"15min":0.41499999999999998},"max":{"1min":4.7130000000000001,"5min":9.6669999999999998,"15min":9.6669999999999998},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.456,"5min":1.5189999999999999,"15min":1.4790000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.2789999999999999,"5min":7.9320000000000004,"15min":8.8930000000000007},"last":0.61199999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019509,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1425872,"kb_used_data":523592,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":92319280,"statfs":{"total":95995035648,"available":94534942720,"internally_reserved":0,"allocated":536158208,"data_stored":535560234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:20:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":1.7150000000000001,"15min":2.1859999999999999},"min":{"1min":0.28199999999999997,"5min":0.247,"15min":0.247},"max":{"1min":6.8609999999999998,"5min":16.170999999999999,"15min":116.59999999999999},"last":2.4769999999999999},{"interface":"front","average":{"1min":1.137,"5min":1.429,"15min":2.109},"min":{"1min":0.33800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.1230000000000002,"5min":16.143999999999998,"15min":116.843},"last":1.2849999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:21:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":1.4670000000000001,"15min":1.2989999999999999},"min":{"1min":0.33400000000000002,"5min":0.317,"15min":0.308},"max":{"1min":11.895,"5min":11.895,"15min":13.853999999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.99,"5min":1.716,"15min":1.3779999999999999},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.9350000000000005,"5min":9.9350000000000005,"15min":11.391999999999999},"last":0.51500000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:20:47 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.4990000000000001,"15min":1.8640000000000001},"min":{"1min":0.44700000000000001,"5min":0.34699999999999998,"15min":0.29299999999999998},"max":{"1min":3.161,"5min":9.4819999999999993,"15min":116.7},"last":1.3580000000000001},{"interface":"front","average":{"1min":1.206,"5min":1.2390000000000001,"15min":1.9159999999999999},"min":{"1min":0.40899999999999997,"5min":0.34699999999999998,"15min":0.315},"max":{"1min":5.0739999999999998,"5min":10.817,"15min":116.63800000000001},"last":1.1879999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:21:17 2023","interfaces":[{"interface":"back","average":{"1min":4.149,"5min":2.2669999999999999,"15min":4.0759999999999996},"min":{"1min":0.40400000000000003,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":27.282,"5min":55.299999999999997,"15min":165.995},"last":1.5880000000000001},{"interface":"front","average":{"1min":2.8809999999999998,"5min":2.1469999999999998,"15min":4.2220000000000004},"min":{"1min":0.36099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":27.620999999999999,"5min":55.622,"15min":165.53200000000001},"last":0.48299999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.5900000000000001,"15min":1.6120000000000001},"min":{"1min":0.36599999999999999,"5min":0.33600000000000002,"15min":0.30099999999999999},"max":{"1min":5.1669999999999998,"5min":9.3989999999999991,"15min":13.907},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.3660000000000001,"15min":1.3779999999999999},"min":{"1min":0.39000000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.1840000000000002,"5min":10.794,"15min":11.042999999999999},"last":0.53200000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:21:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0529999999999999,"5min":1.4359999999999999,"15min":1.8520000000000001},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":12.074,"5min":18.103999999999999,"15min":74.700999999999993},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.407,"5min":1.758,"15min":1.923},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33000000000000002},"max":{"1min":6.4020000000000001,"5min":11.958,"15min":73.572999999999993},"last":0.59999999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:20:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.7529999999999999,"15min":1.6479999999999999},"min":{"1min":0.44400000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.2220000000000004,"5min":15.484999999999999,"15min":15.484999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.339,"5min":1.3899999999999999,"15min":1.5680000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.3230000000000004,"5min":11.090999999999999,"15min":13.916},"last":0.58599999999999997}]}]},{"osd":5,"up_from":36,"seq":154618823280,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1167040,"kb_used_data":261624,"kb_used_omap":0,"kb_used_meta":905408,"kb_avail":92578112,"statfs":{"total":95995035648,"available":94799986688,"internally_reserved":0,"allocated":267902976,"data_stored":267324846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927137792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.5630000000000002,"15min":2.8359999999999999},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.1240000000000001,"5min":139.81700000000001,"15min":139.81700000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.607,"5min":2.835,"15min":2.8879999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.7149999999999999,"5min":141.66900000000001,"15min":141.66900000000001},"last":0.65600000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":1.5149999999999999,"15min":1.512},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.1900000000000004,"5min":25.116,"15min":25.116},"last":0.99099999999999999},{"interface":"front","average":{"1min":2.024,"5min":1.603,"15min":1.484},"min":{"1min":0.33600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.346,"5min":25.282,"15min":25.282},"last":0.71599999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.089,"15min":1.9830000000000001},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":3.5950000000000002,"5min":10.163,"15min":128.77199999999999},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.647,"5min":1.3720000000000001,"15min":1.8999999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":7.3399999999999999,"5min":7.7519999999999998,"15min":128.72999999999999},"last":1.111}]},{"osd":3,"last update":"Mon Oct 9 09:21:05 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.506,"15min":1.754},"min":{"1min":0.34499999999999997,"5min":0.33000000000000002,"15min":0.32000000000000001},"max":{"1min":8.8219999999999992,"5min":11.427,"15min":57.963000000000001},"last":0.88},{"interface":"front","average":{"1min":1.667,"5min":1.3540000000000001,"15min":1.927},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.31},"max":{"1min":4.9199999999999999,"5min":10.630000000000001,"15min":57.603000000000002},"last":0.83999999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.0640000000000001,"15min":1.76},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26600000000000001},"max":{"1min":1.206,"5min":10.201000000000001,"15min":13.904},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.125,"15min":1.629},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":1.9790000000000001,"5min":11.478,"15min":22.228000000000002},"last":0.96099999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.792,"15min":1.831},"min":{"1min":0.36199999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.2890000000000001,"5min":53.840000000000003,"15min":53.840000000000003},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.796,"5min":1.984,"15min":1.883},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.7219999999999995,"5min":50.404000000000003,"15min":50.404000000000003},"last":1.056}]},{"osd":7,"last update":"Mon Oct 9 09:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.266,"15min":1.516},"min":{"1min":0.39200000000000002,"5min":0.36799999999999999,"15min":0.34799999999999998},"max":{"1min":3.4289999999999998,"5min":5.9699999999999998,"15min":12.159000000000001},"last":1.1559999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.421,"15min":1.599},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.33400000000000002},"max":{"1min":1.1830000000000001,"5min":10.598000000000001,"15min":11.090999999999999},"last":1.018}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":770703360,"data_stored":770703360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1026494464,"data_stored":1026490392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":860229632,"data_stored":860225560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1054343168,"data_stored":1054343168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532287488,"data_stored":532283416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":264896512,"data_stored":264896512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1054085120,"data_stored":1054081048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":806092800,"data_stored":806092800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:21:43.842 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:21:43.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-10-09T09:21:43.850 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:43.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:43.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:43.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:44.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:44.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:44.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:44.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:44.307 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:44.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:44.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:44.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:44.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:44.576 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:44.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:44.709 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:44.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:44.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:44.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:44.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:44.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:44.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:45.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:45.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:45.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:45.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:45.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:45.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:45.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:45.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:45.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:45.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:45.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:45.479 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:45.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:45.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:45.582 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:45.612 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:45.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:45.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:45.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[75179]: cluster 2023-10-09T09:21:44.258685+0000 mgr.y (mgr.25421) 1337 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 72 KiB/s, 60 objects/s recovering 2023-10-09T09:21:45.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[75179]: cluster 2023-10-09T09:21:44.537987+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[75179]: cluster 2023-10-09T09:21:44.547739+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[75179]: cluster 2023-10-09T09:21:44.551768+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-10-09T09:21:45.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[75179]: cluster 2023-10-09T09:21:44.560991+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[77892]: cluster 2023-10-09T09:21:44.258685+0000 mgr.y (mgr.25421) 1337 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 72 KiB/s, 60 objects/s recovering 2023-10-09T09:21:45.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[77892]: cluster 2023-10-09T09:21:44.537987+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[77892]: cluster 2023-10-09T09:21:44.547739+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[77892]: cluster 2023-10-09T09:21:44.551768+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-10-09T09:21:45.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[77892]: cluster 2023-10-09T09:21:44.560991+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:45 smithi094 bash[59677]: cluster 2023-10-09T09:21:44.258685+0000 mgr.y (mgr.25421) 1337 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 72 KiB/s, 60 objects/s recovering 2023-10-09T09:21:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:45 smithi094 bash[59677]: cluster 2023-10-09T09:21:44.537987+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:45 smithi094 bash[59677]: cluster 2023-10-09T09:21:44.547739+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:45 smithi094 bash[59677]: cluster 2023-10-09T09:21:44.551768+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-10-09T09:21:45.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:45 smithi094 bash[59677]: cluster 2023-10-09T09:21:44.560991+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T09:21:45.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:45.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:45.850 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:45.872 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:45.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:45.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:45.991 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:21:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:21:45] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.33.4" 2023-10-09T09:21:46.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:46.119 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:46.161 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:46.266 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:46.292 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:46.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:46.399 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:46.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:46.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:46.554 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:21:46.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:46.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:46.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:46 smithi094 bash[59677]: cluster 2023-10-09T09:21:45.547127+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:46 smithi094 bash[59677]: cluster 2023-10-09T09:21:45.548597+0000 mon.a (mon.0) 1793 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-10-09T09:21:46.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:46 smithi094 bash[59677]: cluster 2023-10-09T09:21:45.549999+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:46 smithi094 bash[59677]: cluster 2023-10-09T09:21:45.579448+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.668 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:46 smithi094 bash[59677]: audit 2023-10-09T09:21:46.496871+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.16:0/1131657677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:21:46.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:46.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:46.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[75179]: cluster 2023-10-09T09:21:45.547127+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[75179]: cluster 2023-10-09T09:21:45.548597+0000 mon.a (mon.0) 1793 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[75179]: cluster 2023-10-09T09:21:45.549999+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[75179]: cluster 2023-10-09T09:21:45.579448+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[75179]: audit 2023-10-09T09:21:46.496871+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.16:0/1131657677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[77892]: cluster 2023-10-09T09:21:45.547127+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[77892]: cluster 2023-10-09T09:21:45.548597+0000 mon.a (mon.0) 1793 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-10-09T09:21:46.795 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[77892]: cluster 2023-10-09T09:21:45.549999+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.796 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[77892]: cluster 2023-10-09T09:21:45.579448+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T09:21:46.796 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:46 smithi016 bash[77892]: audit 2023-10-09T09:21:46.496871+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.16:0/1131657677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:21:46.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:46.917 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:46.946 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:46.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:47.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:47.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:47.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:47.177 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:47.199 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:47.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:47.307 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:47.332 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:47.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:47.437 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:47.465 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:47.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:47.572 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:47.601 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:47.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:47.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:47.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:47.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:47.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:47.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:47 smithi094 bash[59677]: cluster 2023-10-09T09:21:46.259307+0000 mgr.y (mgr.25421) 1338 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 0 B/s, 28 objects/s recovering 2023-10-09T09:21:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:47 smithi094 bash[59677]: cluster 2023-10-09T09:21:46.545835+0000 mon.a (mon.0) 1795 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:21:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:47 smithi094 bash[59677]: cluster 2023-10-09T09:21:46.549138+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T09:21:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:47 smithi094 bash[59677]: audit 2023-10-09T09:21:46.549887+0000 mon.a (mon.0) 1796 : audit 1 from='client.? 172.21.15.16:0/1131657677' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:21:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:47 smithi094 bash[59677]: cluster 2023-10-09T09:21:46.549924+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-10-09T09:21:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:47 smithi094 bash[59677]: cluster 2023-10-09T09:21:46.597067+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T09:21:47.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:47.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[75179]: cluster 2023-10-09T09:21:46.259307+0000 mgr.y (mgr.25421) 1338 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 0 B/s, 28 objects/s recovering 2023-10-09T09:21:47.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[75179]: cluster 2023-10-09T09:21:46.545835+0000 mon.a (mon.0) 1795 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:21:47.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[75179]: cluster 2023-10-09T09:21:46.549138+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T09:21:47.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[75179]: audit 2023-10-09T09:21:46.549887+0000 mon.a (mon.0) 1796 : audit 1 from='client.? 172.21.15.16:0/1131657677' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:21:47.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[75179]: cluster 2023-10-09T09:21:46.549924+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-10-09T09:21:47.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[75179]: cluster 2023-10-09T09:21:46.597067+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T09:21:47.842 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[77892]: cluster 2023-10-09T09:21:46.259307+0000 mgr.y (mgr.25421) 1338 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 0 B/s, 28 objects/s recovering 2023-10-09T09:21:47.842 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[77892]: cluster 2023-10-09T09:21:46.545835+0000 mon.a (mon.0) 1795 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:21:47.842 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[77892]: cluster 2023-10-09T09:21:46.549138+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T09:21:47.842 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[77892]: audit 2023-10-09T09:21:46.549887+0000 mon.a (mon.0) 1796 : audit 1 from='client.? 172.21.15.16:0/1131657677' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:21:47.842 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[77892]: cluster 2023-10-09T09:21:46.549924+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-10-09T09:21:47.842 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:47 smithi016 bash[77892]: cluster 2023-10-09T09:21:46.597067+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T09:21:47.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:47.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:47.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:47.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:47.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:48.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:48.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:48.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:48.243 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:48.261 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:21:48.271 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:48.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:48.325 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:21:48.375 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:21:48.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:48.414 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:48.432 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:21:48.471 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:21:48.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:21:48.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:48.541 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:21:48.544 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:48.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:48.580 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:21:48.588 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.588 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.588 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.589 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.589 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.589 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.589 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.589 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:21:48.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:48.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:48.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:48.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:48.795 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:48 smithi094 bash[59677]: audit 2023-10-09T09:21:48.199574+0000 mon.a (mon.0) 1798 : audit 0 from='client.? 172.21.15.16:0/3619578521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:21:48.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:48.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:48.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:48.930 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:48 smithi016 bash[75179]: audit 2023-10-09T09:21:48.199574+0000 mon.a (mon.0) 1798 : audit 0 from='client.? 172.21.15.16:0/3619578521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:21:48.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:48 smithi016 bash[77892]: audit 2023-10-09T09:21:48.199574+0000 mon.a (mon.0) 1798 : audit 0 from='client.? 172.21.15.16:0/3619578521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:21:48.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:48.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:49.055 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:49.086 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:49.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:49.190 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:49.216 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:49.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:49.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:49.321 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:49.363 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:49.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:49.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:49.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:49.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:49.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:49.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:49.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:49.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:49.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:49 smithi094 bash[59677]: cluster 2023-10-09T09:21:48.259918+0000 mgr.y (mgr.25421) 1339 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:49 smithi094 bash[59677]: cluster 2023-10-09T09:21:48.555625+0000 mon.a (mon.0) 1799 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:21:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:49 smithi094 bash[59677]: audit 2023-10-09T09:21:49.266628+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:49 smithi094 bash[59677]: audit 2023-10-09T09:21:49.267870+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:49.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[75179]: cluster 2023-10-09T09:21:48.259918+0000 mgr.y (mgr.25421) 1339 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:49.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[75179]: cluster 2023-10-09T09:21:48.555625+0000 mon.a (mon.0) 1799 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:21:49.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[75179]: audit 2023-10-09T09:21:49.266628+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[75179]: audit 2023-10-09T09:21:49.267870+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[77892]: cluster 2023-10-09T09:21:48.259918+0000 mgr.y (mgr.25421) 1339 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[77892]: cluster 2023-10-09T09:21:48.555625+0000 mon.a (mon.0) 1799 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:21:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[77892]: audit 2023-10-09T09:21:49.266628+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:49 smithi016 bash[77892]: audit 2023-10-09T09:21:49.267870+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:49.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:49.766 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:21:49.766 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35619.0:143743 10.2 10.45b2de22 (undecoded) ondisk+write+known_if_redirected e264)", 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:47.336416+0000", 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 422.42079383700002, 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.95504108200000004, 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.767 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35619", 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143743 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336416+0000", 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.768 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336416+0000", 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336414+0000", 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336421+0000", 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4970000000000004e-06 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336434+0000", 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2531000000000001e-05 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336451+0000", 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7153e-05 2023-10-09T09:21:49.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336522+0000", 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0551999999999998e-05 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.336535+0000", 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3535e-05 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291443+0000", 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.954907477 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291457+0000", 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4214e-05 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.772 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35619.0:143746 10.2 10.b5e1b82 (undecoded) ondisk+write+known_if_redirected e264)", 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:47.339315+0000", 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 422.41789456800001, 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.95217537299999999, 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35619", 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143746 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.773 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339315+0000", 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339315+0000", 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339314+0000", 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339326+0000", 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2346e-05 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339341+0000", 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5221999999999999e-05 2023-10-09T09:21:49.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339354+0000", 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2336e-05 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339408+0000", 2023-10-09T09:21:49.776 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4886000000000003e-05 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.339419+0000", 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0361e-05 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291482+0000", 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.95206352699999997 2023-10-09T09:21:49.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291490+0000", 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1620000000000008e-06 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35619.0:143748 10.2 10.292bf352 (undecoded) ondisk+write+known_if_redirected e264)", 2023-10-09T09:21:49.778 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:47.342273+0000", 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 422.41493694600001, 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.94924558699999995, 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35619", 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143748 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.779 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342273+0000", 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342273+0000", 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342271+0000", 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342278+0000", 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6919999999999996e-06 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342296+0000", 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7459999999999999e-05 2023-10-09T09:21:49.781 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342303+0000", 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5310000000000004e-06 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342350+0000", 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6879000000000003e-05 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.782 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.342368+0000", 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8207e-05 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291511+0000", 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.94914231599999999 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.783 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291518+0000", 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8040000000000002e-06 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35619.0:143788 10.2 10.7167ee72 (undecoded) ondisk+write+known_if_redirected e264)", 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:14:47.429377+0000", 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 422.32783278699998, 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.86216932400000001, 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:21:49.784 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35619", 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1717224770", 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 143788 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429377+0000", 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429377+0000", 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429375+0000", 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429386+0000", 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1306e-05 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429411+0000", 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4657000000000001e-05 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429424+0000", 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3138e-05 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429589+0000", 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000165382 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:47.429600+0000", 2023-10-09T09:21:49.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0711e-05 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291538+0000", 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.86193865999999997 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:14:48.291546+0000", 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7919999999999993e-06 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.789 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:81037 11.c 11:3595492c:::benchmark_data_smithi016_218422_object81036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.125123+0000", 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 141.63208679499999, 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34122242699999999, 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.790 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81037 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125123+0000", 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125123+0000", 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125122+0000", 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125145+0000", 2023-10-09T09:21:49.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3411999999999999e-05 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125172+0000", 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7112000000000001e-05 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125180+0000", 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8830000000000006e-06 2023-10-09T09:21:49.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125230+0000", 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9447000000000003e-05 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125380+0000", 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000150388 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.125562+0000", 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018211900000000001 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.128806+0000", 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0032438929999999999 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131277+0000", 2023-10-09T09:21:49.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024709440000000001 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466236+0000", 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33495906399999997 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466293+0000", 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7071999999999998e-05 2023-10-09T09:21:49.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466345+0000", 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2048e-05 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:81042 11.c 11:39b275bd:::benchmark_data_smithi016_218422_object81041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T09:21:49.797 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.131231+0000", 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 141.62597859900001, 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33525664799999999, 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81042 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.798 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.799 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131231+0000", 2023-10-09T09:21:49.799 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.799 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.799 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131231+0000", 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131230+0000", 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131253+0000", 2023-10-09T09:21:49.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3479e-05 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131262+0000", 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1390000000000001e-06 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131269+0000", 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.613e-06 2023-10-09T09:21:49.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131306+0000", 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7369000000000001e-05 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131428+0000", 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012154000000000001 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:21:49.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.131571+0000", 2023-10-09T09:21:49.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014326700000000001 2023-10-09T09:21:49.803 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.803 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.134049+0000", 2023-10-09T09:21:49.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024776070000000002 2023-10-09T09:21:49.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.804 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135388+0000", 2023-10-09T09:21:49.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0013394909999999999 2023-10-09T09:21:49.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466389+0000", 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33100046300000002 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466443+0000", 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4141000000000002e-05 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466488+0000", 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5045e-05 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:81048 11.c 11:38ef0765:::benchmark_data_smithi016_218422_object81047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.135400+0000", 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 141.62180952700001, 2023-10-09T09:21:49.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.331205904, 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81048 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135400+0000", 2023-10-09T09:21:49.807 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135400+0000", 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135399+0000", 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135440+0000", 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0376000000000001e-05 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135451+0000", 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1195e-05 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.809 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135458+0000", 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5000000000000002e-06 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135499+0000", 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0694e-05 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135593+0000", 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3506999999999998e-05 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.135686+0000", 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3086999999999998e-05 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.140536+0000", 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048500720000000004 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.142229+0000", 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0016937230000000001 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466530+0000", 2023-10-09T09:21:49.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.324300062 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466575+0000", 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5772000000000002e-05 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.466606+0000", 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0743e-05 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.813 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:103921 11.c 11:3b855f96:::benchmark_data_smithi016_218422_object10783:head [delete] snapc 0=[] ondisk+write+known_if_redirected e306)", 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:20:18.710950+0000", 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 91.046260124, 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4175558910000001, 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.814 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 103921 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.710950+0000", 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.710950+0000", 2023-10-09T09:21:49.815 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.710948+0000", 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.710955+0000", 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.207e-06 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.816 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.711987+0000", 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001032343 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.711996+0000", 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0589999999999992e-06 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.714573+0000", 2023-10-09T09:21:49.817 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025773889999999998 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.715002+0000", 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042902400000000002 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.818 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.715470+0000", 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000467188 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.732007+0000", 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016537374000000001 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.819 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.743297+0000", 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011289741000000001 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.778612+0000", 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.035315173999999998 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.778624+0000", 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2480000000000001e-05 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.115819+0000", 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3371945890000001 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for missing object", 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.115922+0000", 2023-10-09T09:21:49.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010265899999999999 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.128431+0000", 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012508806000000001 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.128505+0000", 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4957999999999996e-05 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.822 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:103927 11.c 11:30bc29a5:::benchmark_data_smithi016_218422_object10789:head [delete] snapc 0=[] ondisk+write+known_if_redirected e306)", 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:20:18.720175+0000", 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 91.037034927999997, 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.41528147, 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.823 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 103927 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.720175+0000", 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.720175+0000", 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.720173+0000", 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.720180+0000", 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7020000000000004e-06 2023-10-09T09:21:49.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.720194+0000", 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3903999999999999e-05 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.720202+0000", 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4030000000000007e-06 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.732033+0000", 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011831024000000001 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.732197+0000", 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000163649 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.732353+0000", 2023-10-09T09:21:49.827 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000156181 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.743272+0000", 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010918495 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.743306+0000", 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4443000000000002e-05 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.780820+0000", 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.037513817999999997 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.780830+0000", 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0434e-05 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.829 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.116272+0000", 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3354415340000001 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for missing object", 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.116420+0000", 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014786399999999999 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.135388+0000", 2023-10-09T09:21:49.830 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018967754999999999 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.135456+0000", 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8714999999999995e-05 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.831 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:103928 11.c 11:3a122e7a:::benchmark_data_smithi016_218422_object10790:head [delete] snapc 0=[] ondisk+write+known_if_redirected e306)", 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:20:18.724636+0000", 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 91.032573417999998, 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.410722268, 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.832 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 103928 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.724636+0000", 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.833 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.724636+0000", 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.724635+0000", 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.724642+0000", 2023-10-09T09:21:49.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8630000000000002e-06 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.724671+0000", 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9042000000000001e-05 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.724680+0000", 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1679999999999993e-06 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.736678+0000", 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011998497 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.737883+0000", 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0012049249999999999 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.738368+0000", 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00048472700000000001 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.743821+0000", 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0054529770000000003 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.746390+0000", 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025690000000000001 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.780856+0000", 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.034465576999999997 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.780863+0000", 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7870000000000002e-06 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.116469+0000", 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.335605471 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for missing object", 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.116593+0000", 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000123693 2023-10-09T09:21:49.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.135275+0000", 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018682521000000001 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.135359+0000", 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3498999999999995e-05 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.840 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:103946 11.f 11:fa40b23f:::benchmark_data_smithi016_218422_object10808:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:20:18.760678+0000", 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 90.996531837000006, 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3926138290000001, 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.841 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 103946 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.760678+0000", 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.760678+0000", 2023-10-09T09:21:49.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.760677+0000", 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.760682+0000", 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4349999999999996e-06 2023-10-09T09:21:49.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.760693+0000", 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0716e-05 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.760701+0000", 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4940000000000003e-06 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.784955+0000", 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.024253539000000001 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:18.784967+0000", 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1875999999999999e-05 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:19.972662+0000", 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.187695419 2023-10-09T09:21:49.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:19.972817+0000", 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000155274 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:19.972973+0000", 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015586500000000001 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.152809+0000", 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17983584999999999 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.152879+0000", 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0137999999999998e-05 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.152995+0000", 2023-10-09T09:21:49.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000115451 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.153060+0000", 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5529000000000002e-05 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:20:20.153292+0000", 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000231473 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.848 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139271 11.f 11:f841b58f:::benchmark_data_smithi016_218422_object46133:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.124009+0000", 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.633200463000001, 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.40106777399999999, 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.849 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139271 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124009+0000", 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124009+0000", 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124008+0000", 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124015+0000", 2023-10-09T09:21:49.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.939e-06 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124029+0000", 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4363e-05 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124040+0000", 2023-10-09T09:21:49.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0643e-05 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124224+0000", 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018460500000000001 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124561+0000", 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00033716399999999999 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.124994+0000", 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00043251200000000002 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.140908+0000", 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015914581000000001 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.155811+0000", 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014902230000000001 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.524836+0000", 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.36902531100000002 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.524892+0000", 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5954e-05 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525077+0000", 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000185081 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.856 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139273 11.f 11:f38a9e12:::benchmark_data_smithi016_218422_object46135:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.128434+0000", 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.628775929, 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.39689893799999998, 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139273 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.857 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.128434+0000", 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.128434+0000", 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.128433+0000", 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.128436+0000", 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8540000000000001e-06 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.128442+0000", 2023-10-09T09:21:49.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2099999999999998e-06 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.128447+0000", 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0420000000000002e-06 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.136908+0000", 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0084605979999999997 2023-10-09T09:21:49.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.137053+0000", 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000144716 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.137184+0000", 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000131605 2023-10-09T09:21:49.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.155853+0000", 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018668998999999999 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.155862+0000", 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3620000000000006e-06 2023-10-09T09:21:49.862 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525147+0000", 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.36928495900000002 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525207+0000", 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0322000000000003e-05 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525333+0000", 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012564999999999999 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139277 11.f 11:f2870225:::benchmark_data_smithi016_218422_object46139:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.130091+0000", 2023-10-09T09:21:49.864 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.627118363999998, 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.39553352200000003, 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139277 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.865 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.130091+0000", 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.130091+0000", 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.130090+0000", 2023-10-09T09:21:49.866 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.130097+0000", 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3189999999999997e-06 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.130113+0000", 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5794000000000002e-05 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.130121+0000", 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1240000000000005e-06 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.142659+0000", 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012538173 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.868 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.142779+0000", 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119307 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.142905+0000", 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012642099999999999 2023-10-09T09:21:49.869 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.870 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.870 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.155867+0000", 2023-10-09T09:21:49.870 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.01296225 2023-10-09T09:21:49.870 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.155900+0000", 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2316e-05 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525464+0000", 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.36956407800000002 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525507+0000", 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3272000000000002e-05 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525625+0000", 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011776400000000001 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139285 11.f 11:fa7a968c:::benchmark_data_smithi016_218422_object46147:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.141581+0000", 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.615628977, 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.38436537199999998, 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139285 2023-10-09T09:21:49.873 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.141581+0000", 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.141581+0000", 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.874 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.141580+0000", 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.141584+0000", 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3460000000000004e-06 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.875 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.141593+0000", 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3110000000000006e-06 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.141599+0000", 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4010000000000002e-06 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.876 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.146875+0000", 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00527619 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.147017+0000", 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000141698 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:21:49.877 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.147164+0000", 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014699299999999999 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.155931+0000", 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0087671539999999992 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.157867+0000", 2023-10-09T09:21:49.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001935799 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525787+0000", 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.36791960600000001 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525829+0000", 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2874999999999999e-05 2023-10-09T09:21:49.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525946+0000", 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000116704 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139301 11.f 11:f3bd3c99:::benchmark_data_smithi016_218422_object46163:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.880 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.162792+0000", 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.594417962999998, 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.363348958, 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139301 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.162792+0000", 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.162792+0000", 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.162790+0000", 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.162796+0000", 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1070000000000001e-06 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.162808+0000", 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1258e-05 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.162816+0000", 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9699999999999999e-06 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.173722+0000", 2023-10-09T09:21:49.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010906333000000001 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.173854+0000", 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013186099999999999 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.173990+0000", 2023-10-09T09:21:49.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013595100000000001 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181190+0000", 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0071997780000000004 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525751+0000", 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34456149899999999 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525991+0000", 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000239514 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.526031+0000", 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0065999999999999e-05 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.526141+0000", 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000110102 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139307 11.c 11:3cd4597e:::benchmark_data_smithi016_218422_object46169:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.171767+0000", 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.585442786000002, 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35017479099999999, 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.888 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139307 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.171767+0000", 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.171767+0000", 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.171766+0000", 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.171773+0000", 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0960000000000003e-06 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.171784+0000", 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1243000000000001e-05 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.171793+0000", 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9460000000000004e-06 2023-10-09T09:21:49.891 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.171840+0000", 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7389000000000003e-05 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.172005+0000", 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016494099999999999 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.172155+0000", 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014986399999999999 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.176270+0000", 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0041150520000000001 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181482+0000", 2023-10-09T09:21:49.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0052114989999999996 2023-10-09T09:21:49.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.521630+0000", 2023-10-09T09:21:49.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34014834700000002 2023-10-09T09:21:49.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.521699+0000", 2023-10-09T09:21:49.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9245999999999994e-05 2023-10-09T09:21:49.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.521942+0000", 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000242568 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139314 11.c 11:3b81f61c:::benchmark_data_smithi016_218422_object46176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.896 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.181696+0000", 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.575514159999997, 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35240918399999999, 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139314 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181696+0000", 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181696+0000", 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181695+0000", 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181699+0000", 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3139999999999997e-06 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181706+0000", 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.277e-06 2023-10-09T09:21:49.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181711+0000", 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1830000000000002e-06 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.513751+0000", 2023-10-09T09:21:49.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33203990100000003 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.514167+0000", 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00041551199999999999 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.514565+0000", 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00039775599999999998 2023-10-09T09:21:49.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.533446+0000", 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018881704999999999 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.533525+0000", 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8949000000000006e-05 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.533545+0000", 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0088000000000001e-05 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.533899+0000", 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00035334899999999998 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.534105+0000", 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000206076 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.35859.0:139271 11.f e316/307 ondisk, result = 0)", 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.181775+0000", 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.575434233000003, 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34331245599999999, 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:21:49.904 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181775+0000", 2023-10-09T09:21:49.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181775+0000", 2023-10-09T09:21:49.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.906 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181775+0000", 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181780+0000", 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3279999999999996e-06 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.513667+0000", 2023-10-09T09:21:49.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33188710100000002 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.513679+0000", 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1678000000000001e-05 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.524818+0000", 2023-10-09T09:21:49.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011138646 2023-10-09T09:21:49.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.909 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.909 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.524828+0000", 2023-10-09T09:21:49.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0522e-05 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.525088+0000", 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025978900000000002 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.910 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:139316 11.b 11:daa9d672:::benchmark_data_smithi016_218422_object46178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:12.181819+0000", 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 37.575390521999999, 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35279461299999998, 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:21:49.911 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 139316 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181819+0000", 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181819+0000", 2023-10-09T09:21:49.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181819+0000", 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.181822+0000", 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7349999999999998e-06 2023-10-09T09:21:49.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.513655+0000", 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33183305299999999 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.513679+0000", 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3618999999999999e-05 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:21:49.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.513869+0000", 2023-10-09T09:21:49.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019028 2023-10-09T09:21:49.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:21:49.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.514076+0000", 2023-10-09T09:21:49.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000206601 2023-10-09T09:21:49.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.916 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:21:49.916 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.514454+0000", 2023-10-09T09:21:49.916 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00037852200000000001 2023-10-09T09:21:49.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.524417+0000", 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0099621499999999995 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.534520+0000", 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010103521000000001 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.534529+0000", 2023-10-09T09:21:49.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2450000000000004e-06 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.534574+0000", 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5064000000000001e-05 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:12.534614+0000", 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9443000000000002e-05 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.918 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.919 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.919 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:21:49.919 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:21:49.919 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:21:49.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:49.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:49.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:50.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:50.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:50.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:50.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:50.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:50.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:50.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:50.435 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:50.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:50.563 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:50.589 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:50.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:50.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:50.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:50.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:50.833 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:50.860 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:50.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:50.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:50.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:50 smithi016 bash[75179]: cluster 2023-10-09T09:21:50.260485+0000 mgr.y (mgr.25421) 1340 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:50.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:50 smithi016 bash[75179]: audit 2023-10-09T09:21:50.842832+0000 mon.a (mon.0) 1802 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T09:21:50.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:50 smithi016 bash[75179]: audit 2023-10-09T09:21:50.843097+0000 mon.a (mon.0) 1803 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:21:50.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:50 smithi016 bash[77892]: cluster 2023-10-09T09:21:50.260485+0000 mgr.y (mgr.25421) 1340 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:50.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:50 smithi016 bash[77892]: audit 2023-10-09T09:21:50.842832+0000 mon.a (mon.0) 1802 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T09:21:50.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:50 smithi016 bash[77892]: audit 2023-10-09T09:21:50.843097+0000 mon.a (mon.0) 1803 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:21:50.993 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:50.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:51.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:51.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:51.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:51.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:51.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:50 smithi094 bash[59677]: cluster 2023-10-09T09:21:50.260485+0000 mgr.y (mgr.25421) 1340 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:51.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:50 smithi094 bash[59677]: audit 2023-10-09T09:21:50.842832+0000 mon.a (mon.0) 1802 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T09:21:51.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:50 smithi094 bash[59677]: audit 2023-10-09T09:21:50.843097+0000 mon.a (mon.0) 1803 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:21:51.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:51.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:51.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:51.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:51.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:51.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:51.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:51.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:51.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:51.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:51.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:51.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:51.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:51.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:51.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:51.883 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[77892]: cluster 2023-10-09T09:21:50.872311+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T09:21:51.883 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[77892]: audit 2023-10-09T09:21:50.874417+0000 mon.a (mon.0) 1804 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T09:21:51.883 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[77892]: audit 2023-10-09T09:21:50.874615+0000 mon.a (mon.0) 1805 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:21:51.883 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[77892]: cluster 2023-10-09T09:21:50.874787+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2023-10-09T09:21:51.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[77892]: cluster 2023-10-09T09:21:50.886564+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T09:21:51.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:21:51.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:51.940 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:51 smithi094 bash[59677]: cluster 2023-10-09T09:21:50.872311+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T09:21:51.941 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:51 smithi094 bash[59677]: audit 2023-10-09T09:21:50.874417+0000 mon.a (mon.0) 1804 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T09:21:51.941 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:51 smithi094 bash[59677]: audit 2023-10-09T09:21:50.874615+0000 mon.a (mon.0) 1805 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:21:51.941 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:51 smithi094 bash[59677]: cluster 2023-10-09T09:21:50.874787+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2023-10-09T09:21:51.941 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:51 smithi094 bash[59677]: cluster 2023-10-09T09:21:50.886564+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T09:21:51.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:51.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:51.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:52.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:52.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:52.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:52.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[75179]: cluster 2023-10-09T09:21:50.872311+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T09:21:52.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[75179]: audit 2023-10-09T09:21:50.874417+0000 mon.a (mon.0) 1804 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T09:21:52.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[75179]: audit 2023-10-09T09:21:50.874615+0000 mon.a (mon.0) 1805 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:21:52.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[75179]: cluster 2023-10-09T09:21:50.874787+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2023-10-09T09:21:52.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:51 smithi016 bash[75179]: cluster 2023-10-09T09:21:50.886564+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T09:21:52.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:21:52.209 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:52.236 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:52.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:52.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:52.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:52.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:52.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:52.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:52.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:52.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:52.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:52.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:52.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:52.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:52.667 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:52.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:52.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:52.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:52.900 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:52.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:52.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:53.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:53.054 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:53.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:53.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:53.161 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:53.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:52 smithi094 bash[59677]: cluster 2023-10-09T09:21:51.875653+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2023-10-09T09:21:53.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:52 smithi094 bash[59677]: cluster 2023-10-09T09:21:51.884327+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:52 smithi094 bash[59677]: cluster 2023-10-09T09:21:51.888954+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:52 smithi094 bash[59677]: cluster 2023-10-09T09:21:51.890336+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:52 smithi094 bash[59677]: cluster 2023-10-09T09:21:52.260923+0000 mgr.y (mgr.25421) 1341 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:53.195 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:53.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:53.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[75179]: cluster 2023-10-09T09:21:51.875653+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2023-10-09T09:21:53.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[75179]: cluster 2023-10-09T09:21:51.884327+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[75179]: cluster 2023-10-09T09:21:51.888954+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[75179]: cluster 2023-10-09T09:21:51.890336+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[75179]: cluster 2023-10-09T09:21:52.260923+0000 mgr.y (mgr.25421) 1341 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:53.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[77892]: cluster 2023-10-09T09:21:51.875653+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2023-10-09T09:21:53.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[77892]: cluster 2023-10-09T09:21:51.884327+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[77892]: cluster 2023-10-09T09:21:51.888954+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[77892]: cluster 2023-10-09T09:21:51.890336+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T09:21:53.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:52 smithi016 bash[77892]: cluster 2023-10-09T09:21:52.260923+0000 mgr.y (mgr.25421) 1341 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 0 B/s, 20 objects/s recovering 2023-10-09T09:21:53.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:53.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:53.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:53.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:53.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:21:53.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:53.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:53.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:53.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:53.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:53.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:53.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:53.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:53.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:53.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:53.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:53.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:53.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:53.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:53.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:54.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:54.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:54.127 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:54.151 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:54.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:54.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:54.288 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:54.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:54.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:54.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:54.418 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:54.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:54.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:54.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:54.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:54.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:54.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:54.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:54.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:54.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:54.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:54.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:54.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:55.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:55.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:55.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:55.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:55.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:55.237 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:55.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:55.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:55.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:55.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:55 smithi016 bash[75179]: cluster 2023-10-09T09:21:54.261827+0000 mgr.y (mgr.25421) 1342 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 0 B/s, 15 objects/s recovering 2023-10-09T09:21:55.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:55.851 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:55 smithi094 bash[59677]: cluster 2023-10-09T09:21:54.261827+0000 mgr.y (mgr.25421) 1342 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 0 B/s, 15 objects/s recovering 2023-10-09T09:21:55.910 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:55 smithi016 bash[77892]: cluster 2023-10-09T09:21:54.261827+0000 mgr.y (mgr.25421) 1342 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 0 B/s, 15 objects/s recovering 2023-10-09T09:21:55.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:55.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:56.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:21:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:21:55] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T09:21:56.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:56.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:56.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:56.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:56.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:56.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:56.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:56.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:56.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:56.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:56.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:56.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:56.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:56.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:56.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:56.631 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:56.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:56.641 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:56.736 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:56.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:56 smithi094 bash[59677]: audit 2023-10-09T09:21:55.386923+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:56.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:56 smithi016 bash[75179]: audit 2023-10-09T09:21:55.386923+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:56.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:56 smithi016 bash[77892]: audit 2023-10-09T09:21:55.386923+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:21:56.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:56.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:56.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:56.884 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:56.906 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:56.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:56.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:56.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:57.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:57.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:57.152 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:57.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:57.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:57.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:57.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:57 smithi016 bash[75179]: cluster 2023-10-09T09:21:56.262426+0000 mgr.y (mgr.25421) 1343 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T09:21:57.399 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:57 smithi016 bash[77892]: cluster 2023-10-09T09:21:56.262426+0000 mgr.y (mgr.25421) 1343 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T09:21:57.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:57.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:57.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:57.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:57 smithi094 bash[59677]: cluster 2023-10-09T09:21:56.262426+0000 mgr.y (mgr.25421) 1343 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T09:21:57.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:57.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:21:57.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:21:57.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:57.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:57.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:57.864 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:57.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:57.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:57.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:58.100 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:58.135 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:58.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:58.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:58.262 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:21:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:58.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:21:58.366 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:58.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:58.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:58.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:58.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:58.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:58.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:58.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:58.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:21:58.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:58.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:21:58.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:58.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:58.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:58.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:58.935 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:21:58.937 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:21:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:58.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:21:59.044 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:59.065 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:59.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:21:59.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:59.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:59.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:59.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:59.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:59.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:59.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:59.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:59.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:21:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:59.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:21:59.528 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:21:59 smithi016 bash[75179]: cluster 2023-10-09T09:21:58.264628+0000 mgr.y (mgr.25421) 1344 : cluster 0 pgmap v1489: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T09:21:59.528 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:21:59 smithi016 bash[77892]: cluster 2023-10-09T09:21:58.264628+0000 mgr.y (mgr.25421) 1344 : cluster 0 pgmap v1489: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T09:21:59.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:59.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:21:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:59.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:21:59.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:21:59 smithi094 bash[59677]: cluster 2023-10-09T09:21:58.264628+0000 mgr.y (mgr.25421) 1344 : cluster 0 pgmap v1489: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T09:21:59.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:21:59.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:59.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:21:59.867 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:59.897 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:21:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:21:59.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:00.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:00.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:00.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:00.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:00.263 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:00.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:00.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:00.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:00.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:00.524 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:00.558 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:00.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:00.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:00.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:00.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:00.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:00.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:00.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:00 smithi094 bash[59677]: cluster 2023-10-09T09:22:00.265149+0000 mgr.y (mgr.25421) 1345 : cluster 0 pgmap v1490: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T09:22:00.935 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:00.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:00 smithi016 bash[75179]: cluster 2023-10-09T09:22:00.265149+0000 mgr.y (mgr.25421) 1345 : cluster 0 pgmap v1490: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T09:22:00.958 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:00.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:01.065 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:01.103 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:01.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:01.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:01.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:01.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:01.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:01.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:01.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:01.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:01.456 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:01.503 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:01.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:01.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:01.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:01.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:00 smithi016 bash[77892]: cluster 2023-10-09T09:22:00.265149+0000 mgr.y (mgr.25421) 1345 : cluster 0 pgmap v1490: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T09:22:01.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:01.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:01.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:01.742 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:01.771 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:01.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:01.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:01.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:01.906 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 11 pg_num to 26 2023-10-09T09:22:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:01.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:01.924 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:01 smithi016 bash[77892]: audit 2023-10-09T09:22:00.891701+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.16:0/2296051980' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:22:01.924 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:01 smithi016 bash[77892]: audit 2023-10-09T09:22:00.892781+0000 mon.a (mon.0) 1809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:22:01.924 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:01 smithi016 bash[75179]: audit 2023-10-09T09:22:00.891701+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.16:0/2296051980' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:22:01.924 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:01 smithi016 bash[75179]: audit 2023-10-09T09:22:00.892781+0000 mon.a (mon.0) 1809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:22:02.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:02.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:02.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:02.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:02.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:02.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:02.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:01 smithi094 bash[59677]: audit 2023-10-09T09:22:00.891701+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.16:0/2296051980' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:22:02.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:01 smithi094 bash[59677]: audit 2023-10-09T09:22:00.892781+0000 mon.a (mon.0) 1809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:22:02.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:02.303 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:02.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:02.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:02.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:02.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:02.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:02.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:02.587 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:02.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:02.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:02.690 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:02.715 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:02.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:02.819 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:02.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:02.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:02.967 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:03.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[77892]: audit 2023-10-09T09:22:01.885380+0000 mon.a (mon.0) 1810 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:22:03.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[77892]: cluster 2023-10-09T09:22:01.885519+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2023-10-09T09:22:03.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[77892]: cluster 2023-10-09T09:22:01.889082+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T09:22:03.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[77892]: cluster 2023-10-09T09:22:01.895865+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T09:22:03.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[77892]: cluster 2023-10-09T09:22:02.265638+0000 mgr.y (mgr.25421) 1346 : cluster 0 pgmap v1492: 89 pgs: 89 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T09:22:03.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[77892]: audit 2023-10-09T09:22:02.267086+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:22:03.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[75179]: audit 2023-10-09T09:22:01.885380+0000 mon.a (mon.0) 1810 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:22:03.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[75179]: cluster 2023-10-09T09:22:01.885519+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2023-10-09T09:22:03.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[75179]: cluster 2023-10-09T09:22:01.889082+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T09:22:03.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[75179]: cluster 2023-10-09T09:22:01.895865+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T09:22:03.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[75179]: cluster 2023-10-09T09:22:02.265638+0000 mgr.y (mgr.25421) 1346 : cluster 0 pgmap v1492: 89 pgs: 89 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T09:22:03.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:02 smithi016 bash[75179]: audit 2023-10-09T09:22:02.267086+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:22:03.016 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:03.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:03.133 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:02 smithi094 bash[59677]: audit 2023-10-09T09:22:01.885380+0000 mon.a (mon.0) 1810 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:22:03.133 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:02 smithi094 bash[59677]: cluster 2023-10-09T09:22:01.885519+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2023-10-09T09:22:03.133 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:02 smithi094 bash[59677]: cluster 2023-10-09T09:22:01.889082+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T09:22:03.133 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:02 smithi094 bash[59677]: cluster 2023-10-09T09:22:01.895865+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T09:22:03.134 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:02 smithi094 bash[59677]: cluster 2023-10-09T09:22:02.265638+0000 mgr.y (mgr.25421) 1346 : cluster 0 pgmap v1492: 89 pgs: 89 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T09:22:03.134 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:02 smithi094 bash[59677]: audit 2023-10-09T09:22:02.267086+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:22:03.161 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:03.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:03.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:03.414 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:03.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:03.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:03.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:03.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:03.674 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:03.706 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:03.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:03.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:03.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:03.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:03.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:03 smithi094 bash[59677]: audit 2023-10-09T09:22:02.915500+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:22:03.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:03 smithi094 bash[59677]: cluster 2023-10-09T09:22:02.915722+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2023-10-09T09:22:03.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:03 smithi094 bash[59677]: cluster 2023-10-09T09:22:02.921259+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:03.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:03 smithi094 bash[59677]: cluster 2023-10-09T09:22:02.921825+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:03.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:03 smithi094 bash[59677]: cluster 2023-10-09T09:22:02.922700+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:03.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:03.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:04.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:04.099 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:04.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:04.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:04.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[75179]: audit 2023-10-09T09:22:02.915500+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[75179]: cluster 2023-10-09T09:22:02.915722+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[75179]: cluster 2023-10-09T09:22:02.921259+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[75179]: cluster 2023-10-09T09:22:02.921825+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[75179]: cluster 2023-10-09T09:22:02.922700+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[77892]: audit 2023-10-09T09:22:02.915500+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[77892]: cluster 2023-10-09T09:22:02.915722+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[77892]: cluster 2023-10-09T09:22:02.921259+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:04.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[77892]: cluster 2023-10-09T09:22:02.921825+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:04.146 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:03 smithi016 bash[77892]: cluster 2023-10-09T09:22:02.922700+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T09:22:04.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:04.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:04.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:04.336 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:04.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:04.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:04.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:04.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:04.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:04.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:04.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:04.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:04.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:04.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:04.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:04.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:04.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:04.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:04.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:04.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:05.019 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:05.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:05.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:05.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:05.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.908705+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.910828+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.911287+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.911361+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.912888+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.912933+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.914217+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:03.915020+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: audit 2023-10-09T09:22:04.262794+0000 mon.a (mon.0) 1816 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:05.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:04 smithi094 bash[59677]: cluster 2023-10-09T09:22:04.270248+0000 mgr.y (mgr.25421) 1347 : cluster 0 pgmap v1495: 99 pgs: 10 unknown, 89 active+clean; 975 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-10-09T09:22:05.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:05.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:05.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.908705+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.910828+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.911287+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.911361+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.912888+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.912933+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.914217+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:03.915020+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: audit 2023-10-09T09:22:04.262794+0000 mon.a (mon.0) 1816 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:05.258 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[77892]: cluster 2023-10-09T09:22:04.270248+0000 mgr.y (mgr.25421) 1347 : cluster 0 pgmap v1495: 99 pgs: 10 unknown, 89 active+clean; 975 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.908705+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.910828+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.911287+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.911361+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.912888+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.912933+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.914217+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.259 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:03.915020+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T09:22:05.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: audit 2023-10-09T09:22:04.262794+0000 mon.a (mon.0) 1816 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:05.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:04 smithi016 bash[75179]: cluster 2023-10-09T09:22:04.270248+0000 mgr.y (mgr.25421) 1347 : cluster 0 pgmap v1495: 99 pgs: 10 unknown, 89 active+clean; 975 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-10-09T09:22:05.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:05.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:05.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:05.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:05.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:05.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:05.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:05.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:05.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:05.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:05.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:05.729 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:05.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:05.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:05.841 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:22:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:22:05] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T09:22:05.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:05.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:05.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:05.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:06.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:06.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:06.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:06.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:06.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:06.241 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:06.266 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:06.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:06.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:06.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:06.502 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:06.525 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:06.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:06.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:06.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:06.755 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:06.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:06.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:06.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:06.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:06.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:07.020 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:07.045 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:07.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:07.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:07.284 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:07.315 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:07.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:07.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:07.428 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:07 smithi016 bash[75179]: cluster 2023-10-09T09:22:06.272661+0000 mgr.y (mgr.25421) 1348 : cluster 0 pgmap v1496: 99 pgs: 4 unknown, 95 active+clean; 879 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T09:22:07.430 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:07 smithi016 bash[77892]: cluster 2023-10-09T09:22:06.272661+0000 mgr.y (mgr.25421) 1348 : cluster 0 pgmap v1496: 99 pgs: 4 unknown, 95 active+clean; 879 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T09:22:07.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:07.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:07.558 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:07.567 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:07 smithi094 bash[59677]: cluster 2023-10-09T09:22:06.272661+0000 mgr.y (mgr.25421) 1348 : cluster 0 pgmap v1496: 99 pgs: 4 unknown, 95 active+clean; 879 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T09:22:07.586 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:07.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:07.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:07.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:07.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:07.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:07.992 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:07.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:08.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:08.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:08.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:08.228 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:08.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:08.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:08.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:08.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:08 smithi016 bash[75179]: audit 2023-10-09T09:22:08.274461+0000 mon.a (mon.0) 1817 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:22:08.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:08 smithi016 bash[77892]: audit 2023-10-09T09:22:08.274461+0000 mon.a (mon.0) 1817 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:22:08.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:08.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:08.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:08.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:08 smithi094 bash[59677]: audit 2023-10-09T09:22:08.274461+0000 mon.a (mon.0) 1817 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:22:08.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:08.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:08.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:08.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:08.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:08.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:08.743 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:08.770 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:08.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:08.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:08.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:08.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:08.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:08.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:09.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:09.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:09.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:09.256 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:09.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:09.389 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:09.423 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[75179]: cluster 2023-10-09T09:22:08.273369+0000 mgr.y (mgr.25421) 1349 : cluster 0 pgmap v1497: 99 pgs: 99 active+clean; 782 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T09:22:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[75179]: audit 2023-10-09T09:22:08.338943+0000 mon.a (mon.0) 1818 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:22:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[75179]: cluster 2023-10-09T09:22:08.341287+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2023-10-09T09:22:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[75179]: cluster 2023-10-09T09:22:08.346345+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[75179]: cluster 2023-10-09T09:22:08.348410+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[75179]: cluster 2023-10-09T09:22:08.348528+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[77892]: cluster 2023-10-09T09:22:08.273369+0000 mgr.y (mgr.25421) 1349 : cluster 0 pgmap v1497: 99 pgs: 99 active+clean; 782 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T09:22:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[77892]: audit 2023-10-09T09:22:08.338943+0000 mon.a (mon.0) 1818 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:22:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[77892]: cluster 2023-10-09T09:22:08.341287+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2023-10-09T09:22:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[77892]: cluster 2023-10-09T09:22:08.346345+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[77892]: cluster 2023-10-09T09:22:08.348410+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:09 smithi016 bash[77892]: cluster 2023-10-09T09:22:08.348528+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:09.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:09.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:09.653 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:09.666 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:09 smithi094 bash[59677]: cluster 2023-10-09T09:22:08.273369+0000 mgr.y (mgr.25421) 1349 : cluster 0 pgmap v1497: 99 pgs: 99 active+clean; 782 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T09:22:09.666 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:09 smithi094 bash[59677]: audit 2023-10-09T09:22:08.338943+0000 mon.a (mon.0) 1818 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:22:09.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:09 smithi094 bash[59677]: cluster 2023-10-09T09:22:08.341287+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2023-10-09T09:22:09.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:09 smithi094 bash[59677]: cluster 2023-10-09T09:22:08.346345+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:09 smithi094 bash[59677]: cluster 2023-10-09T09:22:08.348410+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.667 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:09 smithi094 bash[59677]: cluster 2023-10-09T09:22:08.348528+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T09:22:09.686 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:09.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:09.791 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:09.821 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:09.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:09.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:09.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:10.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:10.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:10.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:10.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:10.178 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:10.207 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:10.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:10.311 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:10.343 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:10.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:10.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:10.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:10.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:10.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:10.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:10 smithi094 bash[59677]: cluster 2023-10-09T09:22:09.345526+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-10-09T09:22:10.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:10 smithi094 bash[59677]: cluster 2023-10-09T09:22:09.348270+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:10 smithi094 bash[59677]: cluster 2023-10-09T09:22:09.349523+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:10 smithi094 bash[59677]: cluster 2023-10-09T09:22:09.350289+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:10 smithi094 bash[59677]: cluster 2023-10-09T09:22:09.351331+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:10 smithi094 bash[59677]: cluster 2023-10-09T09:22:09.353122+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:10 smithi094 bash[59677]: audit 2023-10-09T09:22:10.275537+0000 mon.a (mon.0) 1821 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:22:10.586 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:10.611 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:10.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:10.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:10.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[75179]: cluster 2023-10-09T09:22:09.345526+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-10-09T09:22:10.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[75179]: cluster 2023-10-09T09:22:09.348270+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[75179]: cluster 2023-10-09T09:22:09.349523+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[75179]: cluster 2023-10-09T09:22:09.350289+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[75179]: cluster 2023-10-09T09:22:09.351331+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[75179]: cluster 2023-10-09T09:22:09.353122+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.728 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[75179]: audit 2023-10-09T09:22:10.275537+0000 mon.a (mon.0) 1821 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:22:10.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[77892]: cluster 2023-10-09T09:22:09.345526+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-10-09T09:22:10.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[77892]: cluster 2023-10-09T09:22:09.348270+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[77892]: cluster 2023-10-09T09:22:09.349523+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[77892]: cluster 2023-10-09T09:22:09.350289+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[77892]: cluster 2023-10-09T09:22:09.351331+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[77892]: cluster 2023-10-09T09:22:09.353122+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T09:22:10.729 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:10 smithi016 bash[77892]: audit 2023-10-09T09:22:10.275537+0000 mon.a (mon.0) 1821 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:22:10.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:10.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:10.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:11.071 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:11.100 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:11.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:11.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:11.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:11.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:11.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:11.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:11.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:11.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:11.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:11.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:11.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:11.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.273936+0000 mgr.y (mgr.25421) 1350 : cluster 0 pgmap v1500: 99 pgs: 99 active+clean; 749 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.349094+0000 osd.1 (osd.1) 8 : cluster 0 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: audit 2023-10-09T09:22:10.351442+0000 mon.a (mon.0) 1822 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.351753+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.355594+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.357498+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.362482+0000 osd.1 (osd.1) 9 : cluster 0 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.369792+0000 osd.1 (osd.1) 10 : cluster 0 11.10 starting backfill to osd.6 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.926 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.273936+0000 mgr.y (mgr.25421) 1350 : cluster 0 pgmap v1500: 99 pgs: 99 active+clean; 749 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T09:22:12.926 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.349094+0000 osd.1 (osd.1) 8 : cluster 0 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: audit 2023-10-09T09:22:10.351442+0000 mon.a (mon.0) 1822 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.351753+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.355594+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.357498+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.362482+0000 osd.1 (osd.1) 9 : cluster 0 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:11 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.369792+0000 osd.1 (osd.1) 10 : cluster 0 11.10 starting backfill to osd.6 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.935 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:22:12.935 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:22:12.935 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-10-09T09:22:12.935 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:22:12.936 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:22:12.936 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.937 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.938 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.938 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.938 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.938 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.938 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:22:12.938 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:22:12.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: cluster 2023-10-09T09:22:10.475746+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: cluster 2023-10-09T09:22:11.368256+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: cluster 2023-10-09T09:22:11.370215+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-10-09T09:22:12.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: cluster 2023-10-09T09:22:11.378080+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: cluster 2023-10-09T09:22:11.378342+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: cluster 2023-10-09T09:22:11.380621+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: audit 2023-10-09T09:22:11.825717+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.16:0/2540123243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: audit 2023-10-09T09:22:11.966600+0000 mon.a (mon.0) 1825 : audit 1 from='client.? 172.21.15.16:0/1272827492' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[75179]: audit 2023-10-09T09:22:12.276276+0000 mon.a (mon.0) 1826 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: cluster 2023-10-09T09:22:10.475746+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: cluster 2023-10-09T09:22:11.368256+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: cluster 2023-10-09T09:22:11.370215+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: cluster 2023-10-09T09:22:11.378080+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: cluster 2023-10-09T09:22:11.378342+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.940 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: cluster 2023-10-09T09:22:11.380621+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.941 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: audit 2023-10-09T09:22:11.825717+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.16:0/2540123243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:22:12.941 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: audit 2023-10-09T09:22:11.966600+0000 mon.a (mon.0) 1825 : audit 1 from='client.? 172.21.15.16:0/1272827492' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:22:12.941 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:12 smithi016 bash[77892]: audit 2023-10-09T09:22:12.276276+0000 mon.a (mon.0) 1826 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:22:12.941 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:22:12.941 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:22:12.941 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T09:22:12.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:22:12.942 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.273936+0000 mgr.y (mgr.25421) 1350 : cluster 0 pgmap v1500: 99 pgs: 99 active+clean; 749 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T09:22:12.942 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.349094+0000 osd.1 (osd.1) 8 : cluster 0 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: audit 2023-10-09T09:22:10.351442+0000 mon.a (mon.0) 1822 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.351753+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.355594+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.357498+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.362482+0000 osd.1 (osd.1) 9 : cluster 0 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:11 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.369792+0000 osd.1 (osd.1) 10 : cluster 0 11.10 starting backfill to osd.6 from (0'0,0'0] MAX to 329'11021 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: cluster 2023-10-09T09:22:10.475746+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: cluster 2023-10-09T09:22:11.368256+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: cluster 2023-10-09T09:22:11.370215+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: cluster 2023-10-09T09:22:11.378080+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: cluster 2023-10-09T09:22:11.378342+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.944 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: cluster 2023-10-09T09:22:11.380621+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T09:22:12.944 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: audit 2023-10-09T09:22:11.825717+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.16:0/2540123243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:22:12.944 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: audit 2023-10-09T09:22:11.966600+0000 mon.a (mon.0) 1825 : audit 1 from='client.? 172.21.15.16:0/1272827492' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:22:12.944 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:12 smithi094 bash[59677]: audit 2023-10-09T09:22:12.276276+0000 mon.a (mon.0) 1826 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:22:12.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:13.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:13.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:13.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:13.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:13.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:13.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:13.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:13.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.274615+0000 mgr.y (mgr.25421) 1351 : cluster 0 pgmap v1503: 99 pgs: 99 active+clean; 670 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 447 op/s 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.367168+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: audit 2023-10-09T09:22:12.379963+0000 mon.a (mon.0) 1828 : audit 1 from='client.? 172.21.15.16:0/1272827492' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: audit 2023-10-09T09:22:12.380086+0000 mon.a (mon.0) 1829 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.380285+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.383546+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.384138+0000 osd.0 (osd.0) 29 : cluster 0 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 330'10824 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.386423+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:13.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.396728+0000 osd.0 (osd.0) 30 : cluster 0 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 330'10824 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.274615+0000 mgr.y (mgr.25421) 1351 : cluster 0 pgmap v1503: 99 pgs: 99 active+clean; 670 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 447 op/s 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.367168+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: audit 2023-10-09T09:22:12.379963+0000 mon.a (mon.0) 1828 : audit 1 from='client.? 172.21.15.16:0/1272827492' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: audit 2023-10-09T09:22:12.380086+0000 mon.a (mon.0) 1829 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.380285+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.383546+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.384138+0000 osd.0 (osd.0) 29 : cluster 0 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 330'10824 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.386423+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:13.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:13 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.396728+0000 osd.0 (osd.0) 30 : cluster 0 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 330'10824 2023-10-09T09:22:13.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:13.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:13.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:13.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:13.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:13.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:13.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.274615+0000 mgr.y (mgr.25421) 1351 : cluster 0 pgmap v1503: 99 pgs: 99 active+clean; 670 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 447 op/s 2023-10-09T09:22:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.367168+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:22:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: audit 2023-10-09T09:22:12.379963+0000 mon.a (mon.0) 1828 : audit 1 from='client.? 172.21.15.16:0/1272827492' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:22:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: audit 2023-10-09T09:22:12.380086+0000 mon.a (mon.0) 1829 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:22:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.380285+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2023-10-09T09:22:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.383546+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.384138+0000 osd.0 (osd.0) 29 : cluster 0 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 330'10824 2023-10-09T09:22:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.386423+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:13 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.396728+0000 osd.0 (osd.0) 30 : cluster 0 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 330'10824 2023-10-09T09:22:13.830 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:13.885 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:13.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:13.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:13.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:14.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:14.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:14.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:14.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:14.523 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:14.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:14.626 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:14.657 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:14.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:14.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:14.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:14 smithi094 bash[59677]: cluster 2023-10-09T09:22:12.512766+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:14.660 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:14 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.386906+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2023-10-09T09:22:14.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:14 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.388066+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:14 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.388399+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:14 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.393048+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:14 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.393102+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:14 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.395550+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[75179]: cluster 2023-10-09T09:22:12.512766+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.386906+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.388066+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.388399+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.393048+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.393102+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.395550+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[77892]: cluster 2023-10-09T09:22:12.512766+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.386906+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2023-10-09T09:22:14.663 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.388066+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.388399+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.393048+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.393102+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.664 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:14 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.395550+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:14.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:14.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:14.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:14.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:14.902 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:15.180 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:15.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:15.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:15.608 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.447535+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.608 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.449677+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:13.452501+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.609 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:14.275514+0000 mgr.y (mgr.25421) 1352 : cluster 0 pgmap v1506: 99 pgs: 2 peering, 97 active+clean; 602 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 9.2 MiB/s, 146 objects/s recovering 2023-10-09T09:22:15.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:14.393007+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:14.395797+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2023-10-09T09:22:15.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:14.396681+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:14.397120+0000 osd.0 (osd.0) 31 : cluster 0 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 333'10875 2023-10-09T09:22:15.610 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:15 smithi094 bash[59677]: cluster 2023-10-09T09:22:14.441672+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.638 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:15.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.447535+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.449677+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:13.452501+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:14.275514+0000 mgr.y (mgr.25421) 1352 : cluster 0 pgmap v1506: 99 pgs: 2 peering, 97 active+clean; 602 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 9.2 MiB/s, 146 objects/s recovering 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:14.393007+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:14.395797+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:14.396681+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:14.397120+0000 osd.0 (osd.0) 31 : cluster 0 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 333'10875 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[77892]: cluster 2023-10-09T09:22:14.441672+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:15.769 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:22:15] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.447535+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.449677+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:13.452501+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:14.275514+0000 mgr.y (mgr.25421) 1352 : cluster 0 pgmap v1506: 99 pgs: 2 peering, 97 active+clean; 602 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 9.2 MiB/s, 146 objects/s recovering 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:14.393007+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:14.395797+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:14.396681+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:14.397120+0000 osd.0 (osd.0) 31 : cluster 0 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 333'10875 2023-10-09T09:22:15.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:15 smithi016 bash[75179]: cluster 2023-10-09T09:22:14.441672+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:15.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:15.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:15.899 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:16.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:16.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:16.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:16.420 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:16 smithi094 bash[59677]: cluster 2023-10-09T09:22:14.448832+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:16.629 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:16 smithi094 bash[59677]: audit 2023-10-09T09:22:15.415858+0000 mon.a (mon.0) 1833 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:16.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:16.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:16 smithi016 bash[75179]: cluster 2023-10-09T09:22:14.448832+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:16.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:16 smithi016 bash[75179]: audit 2023-10-09T09:22:15.415858+0000 mon.a (mon.0) 1833 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:16.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:16 smithi016 bash[77892]: cluster 2023-10-09T09:22:14.448832+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T09:22:16.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:16 smithi016 bash[77892]: audit 2023-10-09T09:22:15.415858+0000 mon.a (mon.0) 1833 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:16.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:16.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:16.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:16.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:16.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:16.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:17.035 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:17.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:17.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:17.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:17.180 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:17.229 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:17.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:17.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:17.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:17.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:17.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:17.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:17.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:17.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:17.619 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:17.629 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:17 smithi094 bash[59677]: cluster 2023-10-09T09:22:16.276302+0000 mgr.y (mgr.25421) 1353 : cluster 0 pgmap v1508: 99 pgs: 2 peering, 97 active+clean; 551 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:22:17.646 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:17.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:17.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:17 smithi016 bash[75179]: cluster 2023-10-09T09:22:16.276302+0000 mgr.y (mgr.25421) 1353 : cluster 0 pgmap v1508: 99 pgs: 2 peering, 97 active+clean; 551 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:22:17.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:17 smithi016 bash[77892]: cluster 2023-10-09T09:22:16.276302+0000 mgr.y (mgr.25421) 1353 : cluster 0 pgmap v1508: 99 pgs: 2 peering, 97 active+clean; 551 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:22:17.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:17.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:17.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:22:17.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:17.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:22:17.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:17.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:17.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:18.033 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:18.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:18.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:18.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:18.199 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:18.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:18.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:18.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:18.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:18.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:18.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:18.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:18.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:18.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:18.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:18.594 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:18.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:18.700 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:18.708 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:18 smithi094 bash[59677]: cluster 2023-10-09T09:22:17.422944+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:18 smithi094 bash[59677]: cluster 2023-10-09T09:22:17.423550+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:18 smithi094 bash[59677]: cluster 2023-10-09T09:22:17.425976+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2023-10-09T09:22:18.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:18 smithi094 bash[59677]: cluster 2023-10-09T09:22:17.438053+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:18 smithi094 bash[59677]: cluster 2023-10-09T09:22:17.439860+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:18.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:18.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[75179]: cluster 2023-10-09T09:22:17.422944+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[75179]: cluster 2023-10-09T09:22:17.423550+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[75179]: cluster 2023-10-09T09:22:17.425976+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2023-10-09T09:22:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[75179]: cluster 2023-10-09T09:22:17.438053+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[75179]: cluster 2023-10-09T09:22:17.439860+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[77892]: cluster 2023-10-09T09:22:17.422944+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[77892]: cluster 2023-10-09T09:22:17.423550+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[77892]: cluster 2023-10-09T09:22:17.425976+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2023-10-09T09:22:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[77892]: cluster 2023-10-09T09:22:17.438053+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:18 smithi016 bash[77892]: cluster 2023-10-09T09:22:17.439860+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T09:22:18.756 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:18.762 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:18.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:18.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:18.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:18.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:18.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:19.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:19.103 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:19.140 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:19.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:19.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:19.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:19.400 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:19.438 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:19.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:19.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:19.551 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: cluster 2023-10-09T09:22:18.277113+0000 mgr.y (mgr.25421) 1354 : cluster 0 pgmap v1510: 99 pgs: 1 peering, 98 active+clean; 476 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: cluster 2023-10-09T09:22:18.434644+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: cluster 2023-10-09T09:22:18.436486+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: cluster 2023-10-09T09:22:18.436565+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: cluster 2023-10-09T09:22:18.439108+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: cluster 2023-10-09T09:22:18.473843+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: cluster 2023-10-09T09:22:18.499330+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[75179]: audit 2023-10-09T09:22:19.263665+0000 mon.a (mon.0) 1836 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: cluster 2023-10-09T09:22:18.277113+0000 mgr.y (mgr.25421) 1354 : cluster 0 pgmap v1510: 99 pgs: 1 peering, 98 active+clean; 476 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: cluster 2023-10-09T09:22:18.434644+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.552 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: cluster 2023-10-09T09:22:18.436486+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2023-10-09T09:22:19.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: cluster 2023-10-09T09:22:18.436565+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: cluster 2023-10-09T09:22:18.439108+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: cluster 2023-10-09T09:22:18.473843+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: cluster 2023-10-09T09:22:18.499330+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.553 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:19 smithi016 bash[77892]: audit 2023-10-09T09:22:19.263665+0000 mon.a (mon.0) 1836 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:19.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:19.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:19.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:19.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: cluster 2023-10-09T09:22:18.277113+0000 mgr.y (mgr.25421) 1354 : cluster 0 pgmap v1510: 99 pgs: 1 peering, 98 active+clean; 476 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:22:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: cluster 2023-10-09T09:22:18.434644+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: cluster 2023-10-09T09:22:18.436486+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2023-10-09T09:22:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: cluster 2023-10-09T09:22:18.436565+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: cluster 2023-10-09T09:22:18.439108+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: cluster 2023-10-09T09:22:18.473843+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: cluster 2023-10-09T09:22:18.499330+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T09:22:19.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:19 smithi094 bash[59677]: audit 2023-10-09T09:22:19.263665+0000 mon.a (mon.0) 1836 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:19.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:19.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:19.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:19.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:19.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:19.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:20.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:20.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:20.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:20.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:20.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:20.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:20.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:20.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:20.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:20.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:20.476 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:20.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:20.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:20.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:20.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:20.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:20.750 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:20.783 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:20.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:20.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:20.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:20.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:20 smithi016 bash[75179]: cluster 2023-10-09T09:22:20.278033+0000 mgr.y (mgr.25421) 1355 : cluster 0 pgmap v1512: 99 pgs: 99 active+clean; 438 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 438 op/s 2023-10-09T09:22:20.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:20 smithi016 bash[75179]: audit 2023-10-09T09:22:20.279155+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T09:22:20.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:20 smithi016 bash[77892]: cluster 2023-10-09T09:22:20.278033+0000 mgr.y (mgr.25421) 1355 : cluster 0 pgmap v1512: 99 pgs: 99 active+clean; 438 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 438 op/s 2023-10-09T09:22:20.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:20 smithi016 bash[77892]: audit 2023-10-09T09:22:20.279155+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T09:22:20.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:20.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:21.034 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:21.043 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:20 smithi094 bash[59677]: cluster 2023-10-09T09:22:20.278033+0000 mgr.y (mgr.25421) 1355 : cluster 0 pgmap v1512: 99 pgs: 99 active+clean; 438 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 438 op/s 2023-10-09T09:22:21.043 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:20 smithi094 bash[59677]: audit 2023-10-09T09:22:20.279155+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T09:22:21.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:21.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:21.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:21.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:21.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:21.299 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:21.325 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:21.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:21.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:21.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:21.567 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:21.596 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:21.699 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:21.730 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:21.809 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:21.809 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:21.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:21.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:21.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:21.959 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:21.991 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:21.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:22.095 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:22.126 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:22.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:22.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:22.230 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:22.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:21 smithi094 bash[59677]: audit 2023-10-09T09:22:20.883730+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T09:22:22.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:21 smithi094 bash[59677]: cluster 2023-10-09T09:22:20.883834+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2023-10-09T09:22:22.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:21 smithi094 bash[59677]: cluster 2023-10-09T09:22:20.891381+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:21 smithi094 bash[59677]: cluster 2023-10-09T09:22:20.892436+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:21 smithi094 bash[59677]: cluster 2023-10-09T09:22:20.896923+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:21 smithi094 bash[59677]: cluster 2023-10-09T09:22:20.899727+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:21 smithi094 bash[59677]: cluster 2023-10-09T09:22:20.902203+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[75179]: audit 2023-10-09T09:22:20.883730+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T09:22:22.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[75179]: cluster 2023-10-09T09:22:20.883834+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2023-10-09T09:22:22.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[75179]: cluster 2023-10-09T09:22:20.891381+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[75179]: cluster 2023-10-09T09:22:20.892436+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[75179]: cluster 2023-10-09T09:22:20.896923+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[75179]: cluster 2023-10-09T09:22:20.899727+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[75179]: cluster 2023-10-09T09:22:20.902203+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[77892]: audit 2023-10-09T09:22:20.883730+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T09:22:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[77892]: cluster 2023-10-09T09:22:20.883834+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2023-10-09T09:22:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[77892]: cluster 2023-10-09T09:22:20.891381+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[77892]: cluster 2023-10-09T09:22:20.892436+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[77892]: cluster 2023-10-09T09:22:20.896923+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[77892]: cluster 2023-10-09T09:22:20.899727+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:21 smithi016 bash[77892]: cluster 2023-10-09T09:22:20.902203+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T09:22:22.265 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:22.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:22.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:22.369 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:22.400 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:22.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:22.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:22.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:22.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:22.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:22.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:22.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:22.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:22.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:22.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:22.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:22.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:22.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:22.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:22.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:23.028 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:23.060 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:23.105 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[75179]: cluster 2023-10-09T09:22:21.891298+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2023-10-09T09:22:23.105 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[75179]: cluster 2023-10-09T09:22:21.892584+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T09:22:23.105 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[75179]: cluster 2023-10-09T09:22:22.278981+0000 mgr.y (mgr.25421) 1356 : cluster 0 pgmap v1515: 99 pgs: 99 active+clean; 329 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:22:23.105 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[75179]: audit 2023-10-09T09:22:22.280725+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T09:22:23.106 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[77892]: cluster 2023-10-09T09:22:21.891298+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2023-10-09T09:22:23.106 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[77892]: cluster 2023-10-09T09:22:21.892584+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T09:22:23.106 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[77892]: cluster 2023-10-09T09:22:22.278981+0000 mgr.y (mgr.25421) 1356 : cluster 0 pgmap v1515: 99 pgs: 99 active+clean; 329 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:22:23.106 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:22 smithi016 bash[77892]: audit 2023-10-09T09:22:22.280725+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T09:22:23.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:23.175 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:22 smithi094 bash[59677]: cluster 2023-10-09T09:22:21.891298+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2023-10-09T09:22:23.175 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:22 smithi094 bash[59677]: cluster 2023-10-09T09:22:21.892584+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T09:22:23.175 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:22 smithi094 bash[59677]: cluster 2023-10-09T09:22:22.278981+0000 mgr.y (mgr.25421) 1356 : cluster 0 pgmap v1515: 99 pgs: 99 active+clean; 329 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:22:23.175 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:22 smithi094 bash[59677]: audit 2023-10-09T09:22:22.280725+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T09:22:23.193 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:23.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:23.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:23.428 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:23.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:23.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:23.564 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:23.594 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:23.697 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:23.727 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:23.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:23.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:23.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:23.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:23.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:23 smithi094 bash[59677]: cluster 2023-10-09T09:22:22.897170+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:23.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:23 smithi094 bash[59677]: cluster 2023-10-09T09:22:22.898453+0000 osd.1 (osd.1) 11 : cluster 0 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 338'10932 2023-10-09T09:22:23.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:23 smithi094 bash[59677]: audit 2023-10-09T09:22:22.900271+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T09:22:23.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:23 smithi094 bash[59677]: cluster 2023-10-09T09:22:22.900360+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2023-10-09T09:22:23.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:23 smithi094 bash[59677]: cluster 2023-10-09T09:22:22.908546+0000 osd.1 (osd.1) 12 : cluster 0 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 338'10932 2023-10-09T09:22:23.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:23 smithi094 bash[59677]: cluster 2023-10-09T09:22:22.919978+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:23.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:23 smithi094 bash[59677]: cluster 2023-10-09T09:22:22.919979+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:23.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:23.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:24.094 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:24.121 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:24.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:24.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:24.188 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[77892]: cluster 2023-10-09T09:22:22.897170+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:24.188 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[77892]: cluster 2023-10-09T09:22:22.898453+0000 osd.1 (osd.1) 11 : cluster 0 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 338'10932 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[77892]: audit 2023-10-09T09:22:22.900271+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[77892]: cluster 2023-10-09T09:22:22.900360+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[77892]: cluster 2023-10-09T09:22:22.908546+0000 osd.1 (osd.1) 12 : cluster 0 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 338'10932 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[77892]: cluster 2023-10-09T09:22:22.919978+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[77892]: cluster 2023-10-09T09:22:22.919979+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[75179]: cluster 2023-10-09T09:22:22.897170+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[75179]: cluster 2023-10-09T09:22:22.898453+0000 osd.1 (osd.1) 11 : cluster 0 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 338'10932 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[75179]: audit 2023-10-09T09:22:22.900271+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T09:22:24.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[75179]: cluster 2023-10-09T09:22:22.900360+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2023-10-09T09:22:24.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[75179]: cluster 2023-10-09T09:22:22.908546+0000 osd.1 (osd.1) 12 : cluster 0 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 338'10932 2023-10-09T09:22:24.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[75179]: cluster 2023-10-09T09:22:22.919978+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:24.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:23 smithi016 bash[75179]: cluster 2023-10-09T09:22:22.919979+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T09:22:24.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:24.251 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:24.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:24.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:24.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:24.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:24.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:24.490 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:24.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:24.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:24.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:24.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:24.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:24.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:24.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:24.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:24.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:24.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:24.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:24.900 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:24.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:25.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:25.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:25.139 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:25.152 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:24 smithi094 bash[59677]: cluster 2023-10-09T09:22:23.904153+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T09:22:25.152 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:24 smithi094 bash[59677]: cluster 2023-10-09T09:22:23.905384+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2023-10-09T09:22:25.152 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:24 smithi094 bash[59677]: cluster 2023-10-09T09:22:23.908417+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T09:22:25.152 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:24 smithi094 bash[59677]: cluster 2023-10-09T09:22:24.279578+0000 mgr.y (mgr.25421) 1357 : cluster 0 pgmap v1518: 99 pgs: 1 remapped+peering, 98 active+clean; 247 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:22:25.152 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:24 smithi094 bash[59677]: cluster 2023-10-09T09:22:24.871176+0000 mon.a (mon.0) 1845 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2023-10-09T09:22:25.171 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:25.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:25.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:25.312 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:25.353 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:23.904153+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T09:22:25.353 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:23.905384+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2023-10-09T09:22:25.353 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:23.908417+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T09:22:25.353 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:24.279578+0000 mgr.y (mgr.25421) 1357 : cluster 0 pgmap v1518: 99 pgs: 1 remapped+peering, 98 active+clean; 247 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:22:25.354 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:24.871176+0000 mon.a (mon.0) 1845 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2023-10-09T09:22:25.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[77892]: cluster 2023-10-09T09:22:23.904153+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T09:22:25.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[77892]: cluster 2023-10-09T09:22:23.905384+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2023-10-09T09:22:25.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[77892]: cluster 2023-10-09T09:22:23.908417+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T09:22:25.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[77892]: cluster 2023-10-09T09:22:24.279578+0000 mgr.y (mgr.25421) 1357 : cluster 0 pgmap v1518: 99 pgs: 1 remapped+peering, 98 active+clean; 247 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 3.9 MiB/s, 62 objects/s recovering 2023-10-09T09:22:25.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[77892]: cluster 2023-10-09T09:22:24.871176+0000 mon.a (mon.0) 1845 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2023-10-09T09:22:25.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:25.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:25.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:25.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:25.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:25.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:25.680 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:25.705 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:25.808 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:25.835 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:25.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:25.911 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:22:25] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-09T09:22:25.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:25.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:25 smithi094 bash[59677]: cluster 2023-10-09T09:22:24.868964+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:25.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:25 smithi094 bash[59677]: cluster 2023-10-09T09:22:24.871779+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:25.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:25 smithi094 bash[59677]: cluster 2023-10-09T09:22:24.872711+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:25.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:25 smithi094 bash[59677]: cluster 2023-10-09T09:22:24.878196+0000 osd.6 (osd.6) 284 : cluster 0 11.14 starting backfill to osd.1 from (0'0,0'0] MAX to 339'11129 2023-10-09T09:22:25.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:25 smithi094 bash[59677]: cluster 2023-10-09T09:22:24.888992+0000 osd.6 (osd.6) 285 : cluster 0 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 339'11129 2023-10-09T09:22:25.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:25 smithi094 bash[59677]: audit 2023-10-09T09:22:25.433467+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:25.948 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:25 smithi094 bash[59677]: cluster 2023-10-09T09:22:25.877836+0000 mon.a (mon.0) 1847 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2023-10-09T09:22:25.962 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:25.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:25.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:26.065 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:26.092 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:26.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:26.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:26.200 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:26.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:26.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:26.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:26.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:24.868964+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:26.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:24.871779+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:26.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:24.872711+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:26.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:24.878196+0000 osd.6 (osd.6) 284 : cluster 0 11.14 starting backfill to osd.1 from (0'0,0'0] MAX to 339'11129 2023-10-09T09:22:26.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:24.888992+0000 osd.6 (osd.6) 285 : cluster 0 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 339'11129 2023-10-09T09:22:26.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: audit 2023-10-09T09:22:25.433467+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:26.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:25 smithi016 bash[75179]: cluster 2023-10-09T09:22:25.877836+0000 mon.a (mon.0) 1847 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2023-10-09T09:22:26.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:24.868964+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:26.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:24.871779+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:26.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:24.872711+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T09:22:26.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:24.878196+0000 osd.6 (osd.6) 284 : cluster 0 11.14 starting backfill to osd.1 from (0'0,0'0] MAX to 339'11129 2023-10-09T09:22:26.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:24.888992+0000 osd.6 (osd.6) 285 : cluster 0 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 339'11129 2023-10-09T09:22:26.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: audit 2023-10-09T09:22:25.433467+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:26.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:25.877836+0000 mon.a (mon.0) 1847 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2023-10-09T09:22:26.335 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:26.358 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:26.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:26.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:26.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:26.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:26.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:26.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:26.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:26.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:26.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:26.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:26.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:26.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:26.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:26.875 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:26.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:26.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:26.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:26.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:25.878891+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:25.880148+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:25.880201+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:25.880836+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:25.881359+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:25.883186+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:26.280041+0000 mgr.y (mgr.25421) 1358 : cluster 0 pgmap v1521: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 97 active+clean; 202 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[75179]: cluster 2023-10-09T09:22:26.882574+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:25.878891+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:25.880148+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:25.880201+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.989 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:25.880836+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:25.881359+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:25.883186+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:26.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:26.280041+0000 mgr.y (mgr.25421) 1358 : cluster 0 pgmap v1521: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 97 active+clean; 202 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:22:26.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:26 smithi016 bash[77892]: cluster 2023-10-09T09:22:26.882574+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2023-10-09T09:22:27.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:27.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:27.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:27.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:27.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:25.878891+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:25.880148+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:25.880201+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:25.880836+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:25.881359+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:25.883186+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:26.280041+0000 mgr.y (mgr.25421) 1358 : cluster 0 pgmap v1521: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 97 active+clean; 202 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:22:27.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:26 smithi094 bash[59677]: cluster 2023-10-09T09:22:26.882574+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2023-10-09T09:22:27.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:27.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:27.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:27.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:27.420 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:27.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:27.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:27.556 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:27.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:27.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:27.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:27.801 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:27.824 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:27.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:27.929 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:27.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:27.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:27.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:27.997 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:27 smithi016 bash[77892]: cluster 2023-10-09T09:22:26.885163+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T09:22:28.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:28.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:28.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:28.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:28.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:28.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:28.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:28.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:27 smithi094 bash[59677]: cluster 2023-10-09T09:22:26.885163+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T09:22:28.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:28.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:28.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:28.404 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:27 smithi016 bash[75179]: cluster 2023-10-09T09:22:26.885163+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T09:22:28.447 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:28.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:28.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:28.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:28.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:28.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:28.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:28.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:28.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:28.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:28.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:28.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:28.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:28.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:28.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:29.012 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:29.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:28 smithi094 bash[59677]: cluster 2023-10-09T09:22:28.280508+0000 mgr.y (mgr.25421) 1359 : cluster 0 pgmap v1523: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 97 active+clean; 158 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:22:29.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:29.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:29.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:29.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:28 smithi016 bash[75179]: cluster 2023-10-09T09:22:28.280508+0000 mgr.y (mgr.25421) 1359 : cluster 0 pgmap v1523: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 97 active+clean; 158 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:22:29.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:29 smithi016 bash[77892]: cluster 2023-10-09T09:22:28.280508+0000 mgr.y (mgr.25421) 1359 : cluster 0 pgmap v1523: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 97 active+clean; 158 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 1.4 MiB/s, 21 objects/s recovering 2023-10-09T09:22:29.147 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:29.176 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:29.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:29.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:29.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:29.313 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:29.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:29.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:29.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:29.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:29.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:29.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:29.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:29.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:29.732 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:29.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:29.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:29.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:29.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:29.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:29.943 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 93135 objects 2023-10-09T09:22:29.943 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :149.295 2023-10-09T09:22:29.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:29.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:29 smithi094 bash[59677]: audit 2023-10-09T09:22:29.256416+0000 mon.a (mon.0) 1849 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:22:30.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:30.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:29 smithi016 bash[75179]: audit 2023-10-09T09:22:29.256416+0000 mon.a (mon.0) 1849 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:22:30.098 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:29 smithi016 bash[77892]: audit 2023-10-09T09:22:29.256416+0000 mon.a (mon.0) 1849 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:22:30.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:30.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:30.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:30.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:30.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:30.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:30.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:30.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:30.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:30.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:30.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:30.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:30.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:30.771 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:30.803 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:30.907 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:30.938 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:30.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:30.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:31.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:31.075 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:31.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:31.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:31.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:30 smithi016 bash[75179]: cluster 2023-10-09T09:22:30.281166+0000 mgr.y (mgr.25421) 1360 : cluster 0 pgmap v1524: 99 pgs: 1 active+clean+remapped, 98 active+clean; 125 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:22:31.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:30 smithi016 bash[75179]: audit 2023-10-09T09:22:30.282455+0000 mon.a (mon.0) 1850 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T09:22:31.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:30 smithi016 bash[77892]: cluster 2023-10-09T09:22:30.281166+0000 mgr.y (mgr.25421) 1360 : cluster 0 pgmap v1524: 99 pgs: 1 active+clean+remapped, 98 active+clean; 125 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:22:31.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:30 smithi016 bash[77892]: audit 2023-10-09T09:22:30.282455+0000 mon.a (mon.0) 1850 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T09:22:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:30 smithi094 bash[59677]: cluster 2023-10-09T09:22:30.281166+0000 mgr.y (mgr.25421) 1360 : cluster 0 pgmap v1524: 99 pgs: 1 active+clean+remapped, 98 active+clean; 125 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:22:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:30 smithi094 bash[59677]: audit 2023-10-09T09:22:30.282455+0000 mon.a (mon.0) 1850 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T09:22:31.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:31.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:31.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:31.438 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:31.470 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:31.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:31.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:31.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:31.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:31.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:31.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:31.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:31.832 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:31.866 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:31.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:31.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:31.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:31.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:31.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:32.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:32.111 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:30.948508+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:32.111 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: audit 2023-10-09T09:22:30.950675+0000 mon.a (mon.0) 1851 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T09:22:32.111 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:30.950715+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2023-10-09T09:22:32.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:30.948508+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:32.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: audit 2023-10-09T09:22:30.950675+0000 mon.a (mon.0) 1851 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T09:22:32.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:30.950715+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2023-10-09T09:22:32.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:32.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:32.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:32.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:31 smithi094 bash[59677]: cluster 2023-10-09T09:22:30.948508+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:32.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:31 smithi094 bash[59677]: audit 2023-10-09T09:22:30.950675+0000 mon.a (mon.0) 1851 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T09:22:32.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:31 smithi094 bash[59677]: cluster 2023-10-09T09:22:30.950715+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2023-10-09T09:22:32.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:32.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:32.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:32.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:32.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:32.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:32.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:32.527 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:32.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:32.631 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:32.667 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:32.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:32.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:32.769 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:32.800 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:32.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:32.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:32.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:32.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:32.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:32.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:33.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:33.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:33.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:33.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:33.177 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:31.656501+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:31.663822+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:31.956584+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T09:22:33.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:31.956903+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:31.957315+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2023-10-09T09:22:33.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: cluster 2023-10-09T09:22:32.281904+0000 mgr.y (mgr.25421) 1361 : cluster 0 pgmap v1527: 99 pgs: 99 active+clean; 62 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:22:33.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[77892]: audit 2023-10-09T09:22:32.283027+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T09:22:33.178 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:31.656501+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:31.663822+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:31.956584+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T09:22:33.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:31.956903+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:31.957315+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2023-10-09T09:22:33.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: cluster 2023-10-09T09:22:32.281904+0000 mgr.y (mgr.25421) 1361 : cluster 0 pgmap v1527: 99 pgs: 99 active+clean; 62 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:22:33.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:32 smithi016 bash[75179]: audit 2023-10-09T09:22:32.283027+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T09:22:33.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:33.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:32 smithi094 bash[59677]: cluster 2023-10-09T09:22:31.656501+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:32 smithi094 bash[59677]: cluster 2023-10-09T09:22:31.663822+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:32 smithi094 bash[59677]: cluster 2023-10-09T09:22:31.956584+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T09:22:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:32 smithi094 bash[59677]: cluster 2023-10-09T09:22:31.956903+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T09:22:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:32 smithi094 bash[59677]: cluster 2023-10-09T09:22:31.957315+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2023-10-09T09:22:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:32 smithi094 bash[59677]: cluster 2023-10-09T09:22:32.281904+0000 mgr.y (mgr.25421) 1361 : cluster 0 pgmap v1527: 99 pgs: 99 active+clean; 62 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:22:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:32 smithi094 bash[59677]: audit 2023-10-09T09:22:32.283027+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T09:22:33.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:33.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:33.426 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:33.456 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:33.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:33.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:33.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:33.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:33.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:33.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:33.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:33.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:33.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:33.964 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:33.991 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:33.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:33.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:34.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:34.128 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:34.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:34.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:34.232 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:34.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:33 smithi094 bash[59677]: cluster 2023-10-09T09:22:32.964737+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:33 smithi094 bash[59677]: audit 2023-10-09T09:22:32.967082+0000 mon.a (mon.0) 1855 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T09:22:34.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:33 smithi094 bash[59677]: cluster 2023-10-09T09:22:32.967173+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2023-10-09T09:22:34.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:33 smithi094 bash[59677]: cluster 2023-10-09T09:22:32.970293+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:33 smithi094 bash[59677]: cluster 2023-10-09T09:22:33.004188+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[75179]: cluster 2023-10-09T09:22:32.964737+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[75179]: audit 2023-10-09T09:22:32.967082+0000 mon.a (mon.0) 1855 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[75179]: cluster 2023-10-09T09:22:32.967173+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[75179]: cluster 2023-10-09T09:22:32.970293+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[75179]: cluster 2023-10-09T09:22:33.004188+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[77892]: cluster 2023-10-09T09:22:32.964737+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[77892]: audit 2023-10-09T09:22:32.967082+0000 mon.a (mon.0) 1855 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[77892]: cluster 2023-10-09T09:22:32.967173+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[77892]: cluster 2023-10-09T09:22:32.970293+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:33 smithi016 bash[77892]: cluster 2023-10-09T09:22:33.004188+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T09:22:34.257 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:34.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:34.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:34.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:34.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:34.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:34.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:34.488 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:34.512 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:34.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:34.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:34.640 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:34.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:34.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:34.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:34.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:34.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:34.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:34.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:34.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:34.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:35.009 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:32.969552+0000 osd.3 (osd.3) 11 : cluster 0 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.009 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:32.979938+0000 osd.3 (osd.3) 12 : cluster 0 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.009 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:32.990036+0000 osd.3 (osd.3) 13 : cluster 0 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.009 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:33.967112+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:33.967829+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:33.968900+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:33.971105+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:33.976355+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:33.978473+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: audit 2023-10-09T09:22:34.264170+0000 mon.a (mon.0) 1858 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.286285+0000 mgr.y (mgr.25421) 1362 : cluster 0 pgmap v1530: 99 pgs: 99 active+clean; 9.9 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-10-09T09:22:35.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[75179]: audit 2023-10-09T09:22:34.287112+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:32.969552+0000 osd.3 (osd.3) 11 : cluster 0 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:32.979938+0000 osd.3 (osd.3) 12 : cluster 0 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:32.990036+0000 osd.3 (osd.3) 13 : cluster 0 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:33.967112+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:33.967829+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:33.968900+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:33.971105+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:33.976355+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:33.978473+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: audit 2023-10-09T09:22:34.264170+0000 mon.a (mon.0) 1858 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:35.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.286285+0000 mgr.y (mgr.25421) 1362 : cluster 0 pgmap v1530: 99 pgs: 99 active+clean; 9.9 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-10-09T09:22:35.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:34 smithi016 bash[77892]: audit 2023-10-09T09:22:34.287112+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T09:22:35.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:35.130 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:32.969552+0000 osd.3 (osd.3) 11 : cluster 0 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:32.979938+0000 osd.3 (osd.3) 12 : cluster 0 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:32.990036+0000 osd.3 (osd.3) 13 : cluster 0 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 345'10967 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:33.967112+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:33.967829+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:33.968900+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:33.971105+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:33.976355+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:33.978473+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T09:22:35.145 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: audit 2023-10-09T09:22:34.264170+0000 mon.a (mon.0) 1858 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:35.145 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.286285+0000 mgr.y (mgr.25421) 1362 : cluster 0 pgmap v1530: 99 pgs: 99 active+clean; 9.9 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-10-09T09:22:35.145 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:34 smithi094 bash[59677]: audit 2023-10-09T09:22:34.287112+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T09:22:35.162 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:35.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:35.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:35.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:35.390 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.416 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:35.519 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.551 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:35.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:35.683 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:35.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.810 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:35.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:35.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:35.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:35.997 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:22:35] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T09:22:35.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.979222+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:35.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: audit 2023-10-09T09:22:34.982429+0000 mon.a (mon.0) 1860 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T09:22:35.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.982728+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2023-10-09T09:22:35.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.985317+0000 osd.0 (osd.0) 32 : cluster 0 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:35.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.985605+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:35.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.986924+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:35.998 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.989074+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.043 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:36.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:36.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:36.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:36.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:36.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:36.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:36.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.979222+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: audit 2023-10-09T09:22:34.982429+0000 mon.a (mon.0) 1860 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T09:22:36.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.982728+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2023-10-09T09:22:36.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.985317+0000 osd.0 (osd.0) 32 : cluster 0 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.985605+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.986924+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.989074+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:34.998709+0000 osd.0 (osd.0) 33 : cluster 0 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: cluster 2023-10-09T09:22:35.010241+0000 osd.0 (osd.0) 34 : cluster 0 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: audit 2023-10-09T09:22:35.452240+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: audit 2023-10-09T09:22:35.457667+0000 mon.a (mon.0) 1863 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:35 smithi094 bash[59677]: audit 2023-10-09T09:22:35.464758+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:34.998709+0000 osd.0 (osd.0) 33 : cluster 0 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: cluster 2023-10-09T09:22:35.010241+0000 osd.0 (osd.0) 34 : cluster 0 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: audit 2023-10-09T09:22:35.452240+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: audit 2023-10-09T09:22:35.457667+0000 mon.a (mon.0) 1863 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[75179]: audit 2023-10-09T09:22:35.464758+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.979222+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: audit 2023-10-09T09:22:34.982429+0000 mon.a (mon.0) 1860 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T09:22:36.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.982728+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2023-10-09T09:22:36.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.985317+0000 osd.0 (osd.0) 32 : cluster 0 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.985605+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.986924+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.989074+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T09:22:36.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:34.998709+0000 osd.0 (osd.0) 33 : cluster 0 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: cluster 2023-10-09T09:22:35.010241+0000 osd.0 (osd.0) 34 : cluster 0 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11230 2023-10-09T09:22:36.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: audit 2023-10-09T09:22:35.452240+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: audit 2023-10-09T09:22:35.457667+0000 mon.a (mon.0) 1863 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:35 smithi016 bash[77892]: audit 2023-10-09T09:22:35.464758+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:36.308 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:36.342 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:36.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:36.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:36.362 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:22:36.364 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:22:36.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:36.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:36.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:36.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:36.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:36.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:36.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:36.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:36.702 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:36.734 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:36.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:36.738 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:22:36.738 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:22:36.838 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:36.871 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:36.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:36.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:36.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:37.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:37.103 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:37.134 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:37.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: cluster 2023-10-09T09:22:35.986377+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: cluster 2023-10-09T09:22:35.988679+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2023-10-09T09:22:37.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: cluster 2023-10-09T09:22:35.989046+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: cluster 2023-10-09T09:22:35.990275+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: cluster 2023-10-09T09:22:35.992780+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: cluster 2023-10-09T09:22:35.994574+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: cluster 2023-10-09T09:22:36.286804+0000 mgr.y (mgr.25421) 1363 : cluster 0 pgmap v1533: 99 pgs: 99 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 229 op/s 2023-10-09T09:22:37.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: audit 2023-10-09T09:22:36.287506+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T09:22:37.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[75179]: audit 2023-10-09T09:22:36.346767+0000 mgr.y (mgr.25421) 1364 : audit 0 from='client.45848 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:22:37.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: cluster 2023-10-09T09:22:35.986377+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: cluster 2023-10-09T09:22:35.988679+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: cluster 2023-10-09T09:22:35.989046+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: cluster 2023-10-09T09:22:35.990275+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: cluster 2023-10-09T09:22:35.992780+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: cluster 2023-10-09T09:22:35.994574+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: cluster 2023-10-09T09:22:36.286804+0000 mgr.y (mgr.25421) 1363 : cluster 0 pgmap v1533: 99 pgs: 99 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 229 op/s 2023-10-09T09:22:37.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: audit 2023-10-09T09:22:36.287506+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T09:22:37.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:36 smithi016 bash[77892]: audit 2023-10-09T09:22:36.346767+0000 mgr.y (mgr.25421) 1364 : audit 0 from='client.45848 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:22:37.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:37.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:37.400 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: cluster 2023-10-09T09:22:35.986377+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: cluster 2023-10-09T09:22:35.988679+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: cluster 2023-10-09T09:22:35.989046+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: cluster 2023-10-09T09:22:35.990275+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: cluster 2023-10-09T09:22:35.992780+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: cluster 2023-10-09T09:22:35.994574+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: cluster 2023-10-09T09:22:36.286804+0000 mgr.y (mgr.25421) 1363 : cluster 0 pgmap v1533: 99 pgs: 99 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 229 op/s 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: audit 2023-10-09T09:22:36.287506+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T09:22:37.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:36 smithi094 bash[59677]: audit 2023-10-09T09:22:36.346767+0000 mgr.y (mgr.25421) 1364 : audit 0 from='client.45848 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:22:37.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:37.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:37.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:37.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:37.697 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:37.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:37.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:37.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:37.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:37.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:37.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:38.061 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:38.095 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:38.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:38.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:38.232 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:38.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: cluster 2023-10-09T09:22:36.993916+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: audit 2023-10-09T09:22:36.996403+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: cluster 2023-10-09T09:22:36.996475+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: cluster 2023-10-09T09:22:36.997388+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: cluster 2023-10-09T09:22:36.997457+0000 osd.0 (osd.0) 35 : cluster 0 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11062 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: cluster 2023-10-09T09:22:36.999888+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: cluster 2023-10-09T09:22:37.007384+0000 osd.0 (osd.0) 36 : cluster 0 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11062 2023-10-09T09:22:38.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: audit 2023-10-09T09:22:37.710405+0000 mon.a (mon.0) 1869 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[75179]: audit 2023-10-09T09:22:37.718088+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: cluster 2023-10-09T09:22:36.993916+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: audit 2023-10-09T09:22:36.996403+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: cluster 2023-10-09T09:22:36.996475+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: cluster 2023-10-09T09:22:36.997388+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: cluster 2023-10-09T09:22:36.997457+0000 osd.0 (osd.0) 35 : cluster 0 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11062 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: cluster 2023-10-09T09:22:36.999888+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: cluster 2023-10-09T09:22:37.007384+0000 osd.0 (osd.0) 36 : cluster 0 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11062 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: audit 2023-10-09T09:22:37.710405+0000 mon.a (mon.0) 1869 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:38.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:37 smithi016 bash[77892]: audit 2023-10-09T09:22:37.718088+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:38.336 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:38.349 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: cluster 2023-10-09T09:22:36.993916+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: audit 2023-10-09T09:22:36.996403+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: cluster 2023-10-09T09:22:36.996475+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: cluster 2023-10-09T09:22:36.997388+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: cluster 2023-10-09T09:22:36.997457+0000 osd.0 (osd.0) 35 : cluster 0 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11062 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: cluster 2023-10-09T09:22:36.999888+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: cluster 2023-10-09T09:22:37.007384+0000 osd.0 (osd.0) 36 : cluster 0 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11062 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: audit 2023-10-09T09:22:37.710405+0000 mon.a (mon.0) 1869 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:38.350 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:37 smithi094 bash[59677]: audit 2023-10-09T09:22:37.718088+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:38.367 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:38.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:38.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:38.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:38.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:38.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:38.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:38.635 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:38.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:38.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:38.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:38.773 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:38.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:38.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:38.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:38.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:38.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:39.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:39.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:39.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:39.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:39.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:39.168 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: cluster 2023-10-09T09:22:38.001567+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: cluster 2023-10-09T09:22:38.002964+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: cluster 2023-10-09T09:22:38.004430+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: cluster 2023-10-09T09:22:38.004549+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: cluster 2023-10-09T09:22:38.004643+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: audit 2023-10-09T09:22:38.190832+0000 mon.a (mon.0) 1872 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: audit 2023-10-09T09:22:38.192584+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: audit 2023-10-09T09:22:38.200422+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: cluster 2023-10-09T09:22:38.287301+0000 mgr.y (mgr.25421) 1365 : cluster 0 pgmap v1536: 99 pgs: 1 active+clean+remapped, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-10-09T09:22:39.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[75179]: audit 2023-10-09T09:22:38.288105+0000 mon.a (mon.0) 1875 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: cluster 2023-10-09T09:22:38.001567+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: cluster 2023-10-09T09:22:38.002964+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: cluster 2023-10-09T09:22:38.004430+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: cluster 2023-10-09T09:22:38.004549+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: cluster 2023-10-09T09:22:38.004643+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: audit 2023-10-09T09:22:38.190832+0000 mon.a (mon.0) 1872 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: audit 2023-10-09T09:22:38.192584+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: audit 2023-10-09T09:22:38.200422+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: cluster 2023-10-09T09:22:38.287301+0000 mgr.y (mgr.25421) 1365 : cluster 0 pgmap v1536: 99 pgs: 1 active+clean+remapped, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-10-09T09:22:39.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:39 smithi016 bash[77892]: audit 2023-10-09T09:22:38.288105+0000 mon.a (mon.0) 1875 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T09:22:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: cluster 2023-10-09T09:22:38.001567+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: cluster 2023-10-09T09:22:38.002964+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2023-10-09T09:22:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: cluster 2023-10-09T09:22:38.004430+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: cluster 2023-10-09T09:22:38.004549+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: cluster 2023-10-09T09:22:38.004643+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T09:22:39.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: audit 2023-10-09T09:22:38.190832+0000 mon.a (mon.0) 1872 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:22:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: audit 2023-10-09T09:22:38.192584+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:22:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: audit 2023-10-09T09:22:38.200422+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: cluster 2023-10-09T09:22:38.287301+0000 mgr.y (mgr.25421) 1365 : cluster 0 pgmap v1536: 99 pgs: 1 active+clean+remapped, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-10-09T09:22:39.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:39 smithi094 bash[59677]: audit 2023-10-09T09:22:38.288105+0000 mon.a (mon.0) 1875 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T09:22:39.272 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:39.300 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1533,"stamp":"2023-10-09T09:22:36.286580+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7772064,"num_objects":132,"num_object_clones":0,"num_object_copies":396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":132,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":186532,"num_write_kb":5963086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7530,"num_bytes_recovered":227237026,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":221183,"ondisk_log_size":221183,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7729912,"kb_used_data":28984,"kb_used_omap":126,"kb_used_meta":7700737,"kb_avail":742231304,"statfs":{"total":767960285184,"available":760044855296,"internally_reserved":0,"allocated":29679616,"data_stored":22278132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":7885555323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-57311256,"num_objects":-876,"num_object_clones":0,"num_object_copies":-2628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":918,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.005070"},"pg_stats":[{"pgid":"11.12","version":"345'11164","reported_seq":13063,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979693+0000","last_change":"2023-10-09T09:22:18.497776+0000","last_active":"2023-10-09T09:22:33.979693+0000","last_peered":"2023-10-09T09:22:33.979693+0000","last_clean":"2023-10-09T09:22:33.979693+0000","last_became_active":"2023-10-09T09:22:18.445661+0000","last_became_peered":"2023-10-09T09:22:18.445661+0000","last_unstale":"2023-10-09T09:22:33.979693+0000","last_undegraded":"2023-10-09T09:22:33.979693+0000","last_fullsized":"2023-10-09T09:22:33.979693+0000","mapping_epoch":337,"log_start":"298'1900","ondisk_log_start":"298'1900","created":328,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":9264,"log_dups_size":0,"ondisk_log_size":9264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5806,"num_write_kb":186240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":570,"num_bytes_recovered":16809984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":256,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.970905+0000","last_change":"2023-10-09T09:20:19.894893+0000","last_active":"2023-10-09T09:22:32.970905+0000","last_peered":"2023-10-09T09:22:32.970905+0000","last_clean":"2023-10-09T09:22:32.970905+0000","last_became_active":"2023-10-09T09:20:19.893953+0000","last_became_peered":"2023-10-09T09:20:19.893953+0000","last_unstale":"2023-10-09T09:22:32.970905+0000","last_undegraded":"2023-10-09T09:22:32.970905+0000","last_fullsized":"2023-10-09T09:22:32.970905+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950156+0000","last_change":"2023-10-09T09:14:48.316760+0000","last_active":"2023-10-09T09:22:30.950156+0000","last_peered":"2023-10-09T09:22:30.950156+0000","last_clean":"2023-10-09T09:22:30.950156+0000","last_became_active":"2023-10-09T09:14:48.316432+0000","last_became_peered":"2023-10-09T09:14:48.316432+0000","last_unstale":"2023-10-09T09:22:30.950156+0000","last_undegraded":"2023-10-09T09:22:30.950156+0000","last_fullsized":"2023-10-09T09:22:30.950156+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:41.630374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.11","version":"345'11149","reported_seq":12134,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.967611+0000","last_change":"2023-10-09T09:22:14.433729+0000","last_active":"2023-10-09T09:22:31.967611+0000","last_peered":"2023-10-09T09:22:31.967611+0000","last_clean":"2023-10-09T09:22:31.967611+0000","last_became_active":"2023-10-09T09:22:14.402039+0000","last_became_peered":"2023-10-09T09:22:14.402039+0000","last_unstale":"2023-10-09T09:22:31.967611+0000","last_undegraded":"2023-10-09T09:22:31.967611+0000","last_fullsized":"2023-10-09T09:22:31.967611+0000","mapping_epoch":335,"log_start":"298'2300","ondisk_log_start":"298'2300","created":328,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8849,"log_dups_size":0,"ondisk_log_size":8849,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5804,"num_write_kb":185696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":19595264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949587+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:22:30.949587+0000","last_peered":"2023-10-09T09:22:30.949587+0000","last_clean":"2023-10-09T09:22:30.949587+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:22:30.949587+0000","last_undegraded":"2023-10-09T09:22:30.949587+0000","last_fullsized":"2023-10-09T09:22:30.949587+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":384,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.882057+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:22:26.882057+0000","last_peered":"2023-10-09T09:22:26.882057+0000","last_clean":"2023-10-09T09:22:26.882057+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:22:26.882057+0000","last_undegraded":"2023-10-09T09:22:26.882057+0000","last_fullsized":"2023-10-09T09:22:26.882057+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"345'11337","reported_seq":12581,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:29.853266+0000","last_change":"2023-10-09T09:22:13.430545+0000","last_active":"2023-10-09T09:22:29.853266+0000","last_peered":"2023-10-09T09:22:29.853266+0000","last_clean":"2023-10-09T09:22:29.853266+0000","last_became_active":"2023-10-09T09:22:13.397424+0000","last_became_peered":"2023-10-09T09:22:13.397424+0000","last_unstale":"2023-10-09T09:22:29.853266+0000","last_undegraded":"2023-10-09T09:22:29.853266+0000","last_fullsized":"2023-10-09T09:22:29.853266+0000","mapping_epoch":334,"log_start":"301'3300","ondisk_log_start":"301'3300","created":328,"last_epoch_clean":335,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8037,"log_dups_size":0,"ondisk_log_size":8037,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425984,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5893,"num_write_kb":188768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":327,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.879533+0000","last_change":"2023-10-09T09:14:48.312713+0000","last_active":"2023-10-09T09:22:26.879533+0000","last_peered":"2023-10-09T09:22:26.879533+0000","last_clean":"2023-10-09T09:22:26.879533+0000","last_became_active":"2023-10-09T09:14:48.312429+0000","last_became_peered":"2023-10-09T09:14:48.312429+0000","last_unstale":"2023-10-09T09:22:26.879533+0000","last_undegraded":"2023-10-09T09:22:26.879533+0000","last_fullsized":"2023-10-09T09:22:26.879533+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:56.755105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":262,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983405+0000","last_change":"2023-10-09T09:19:42.872407+0000","last_active":"2023-10-09T09:22:33.983405+0000","last_peered":"2023-10-09T09:22:33.983405+0000","last_clean":"2023-10-09T09:22:33.983405+0000","last_became_active":"2023-10-09T09:19:42.871637+0000","last_became_peered":"2023-10-09T09:19:42.871637+0000","last_unstale":"2023-10-09T09:22:33.983405+0000","last_undegraded":"2023-10-09T09:22:33.983405+0000","last_fullsized":"2023-10-09T09:22:33.983405+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":336,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.880174+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:22:26.880174+0000","last_peered":"2023-10-09T09:22:26.880174+0000","last_clean":"2023-10-09T09:22:26.880174+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:22:26.880174+0000","last_undegraded":"2023-10-09T09:22:26.880174+0000","last_fullsized":"2023-10-09T09:22:26.880174+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949547+0000","last_change":"2023-10-09T09:14:48.306040+0000","last_active":"2023-10-09T09:22:30.949547+0000","last_peered":"2023-10-09T09:22:30.949547+0000","last_clean":"2023-10-09T09:22:30.949547+0000","last_became_active":"2023-10-09T09:14:48.304137+0000","last_became_peered":"2023-10-09T09:14:48.304137+0000","last_unstale":"2023-10-09T09:22:30.949547+0000","last_undegraded":"2023-10-09T09:22:30.949547+0000","last_fullsized":"2023-10-09T09:22:30.949547+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:47.446761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":154,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.980625+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:22:33.980625+0000","last_peered":"2023-10-09T09:22:33.980625+0000","last_clean":"2023-10-09T09:22:33.980625+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:22:33.980625+0000","last_undegraded":"2023-10-09T09:22:33.980625+0000","last_fullsized":"2023-10-09T09:22:33.980625+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":313,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983049+0000","last_change":"2023-10-09T09:13:48.190068+0000","last_active":"2023-10-09T09:22:33.983049+0000","last_peered":"2023-10-09T09:22:33.983049+0000","last_clean":"2023-10-09T09:22:33.983049+0000","last_became_active":"2023-10-09T09:13:48.189740+0000","last_became_peered":"2023-10-09T09:13:48.189740+0000","last_unstale":"2023-10-09T09:22:33.983049+0000","last_undegraded":"2023-10-09T09:22:33.983049+0000","last_fullsized":"2023-10-09T09:22:33.983049+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":136,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.981096+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:22:33.981096+0000","last_peered":"2023-10-09T09:22:33.981096+0000","last_clean":"2023-10-09T09:22:33.981096+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:22:33.981096+0000","last_undegraded":"2023-10-09T09:22:33.981096+0000","last_fullsized":"2023-10-09T09:22:33.981096+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949921+0000","last_change":"2023-10-09T09:20:19.898526+0000","last_active":"2023-10-09T09:22:30.949921+0000","last_peered":"2023-10-09T09:22:30.949921+0000","last_clean":"2023-10-09T09:22:30.949921+0000","last_became_active":"2023-10-09T09:20:19.898197+0000","last_became_peered":"2023-10-09T09:20:19.898197+0000","last_unstale":"2023-10-09T09:22:30.949921+0000","last_undegraded":"2023-10-09T09:22:30.949921+0000","last_fullsized":"2023-10-09T09:22:30.949921+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:30.135339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":250,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971854+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:22:32.971854+0000","last_peered":"2023-10-09T09:22:32.971854+0000","last_clean":"2023-10-09T09:22:32.971854+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:22:32.971854+0000","last_undegraded":"2023-10-09T09:22:32.971854+0000","last_fullsized":"2023-10-09T09:22:32.971854+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":238,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971039+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:22:32.971039+0000","last_peered":"2023-10-09T09:22:32.971039+0000","last_clean":"2023-10-09T09:22:32.971039+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:22:32.971039+0000","last_undegraded":"2023-10-09T09:22:32.971039+0000","last_fullsized":"2023-10-09T09:22:32.971039+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":196,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.980660+0000","last_change":"2023-10-09T09:20:19.876172+0000","last_active":"2023-10-09T09:22:33.980660+0000","last_peered":"2023-10-09T09:22:33.980660+0000","last_clean":"2023-10-09T09:22:33.980660+0000","last_became_active":"2023-10-09T09:20:19.765721+0000","last_became_peered":"2023-10-09T09:20:19.765721+0000","last_unstale":"2023-10-09T09:22:33.980660+0000","last_undegraded":"2023-10-09T09:22:33.980660+0000","last_fullsized":"2023-10-09T09:22:33.980660+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950676+0000","last_change":"2023-10-09T09:14:48.319108+0000","last_active":"2023-10-09T09:22:30.950676+0000","last_peered":"2023-10-09T09:22:30.950676+0000","last_clean":"2023-10-09T09:22:30.950676+0000","last_became_active":"2023-10-09T09:14:48.318443+0000","last_became_peered":"2023-10-09T09:14:48.318443+0000","last_unstale":"2023-10-09T09:22:30.950676+0000","last_undegraded":"2023-10-09T09:22:30.950676+0000","last_fullsized":"2023-10-09T09:22:30.950676+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:31.939020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":363,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.880096+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:22:26.880096+0000","last_peered":"2023-10-09T09:22:26.880096+0000","last_clean":"2023-10-09T09:22:26.880096+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:22:26.880096+0000","last_undegraded":"2023-10-09T09:22:26.880096+0000","last_fullsized":"2023-10-09T09:22:26.880096+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":149,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.981247+0000","last_change":"2023-10-09T09:15:23.415336+0000","last_active":"2023-10-09T09:22:33.981247+0000","last_peered":"2023-10-09T09:22:33.981247+0000","last_clean":"2023-10-09T09:22:33.981247+0000","last_became_active":"2023-10-09T09:15:23.414900+0000","last_became_peered":"2023-10-09T09:15:23.414900+0000","last_unstale":"2023-10-09T09:22:33.981247+0000","last_undegraded":"2023-10-09T09:22:33.981247+0000","last_fullsized":"2023-10-09T09:22:33.981247+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.19","version":"345'11314","reported_seq":11383,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.982984+0000","last_change":"2023-10-09T09:22:04.038199+0000","last_active":"2023-10-09T09:22:33.982984+0000","last_peered":"2023-10-09T09:22:33.982984+0000","last_clean":"2023-10-09T09:22:33.982984+0000","last_became_active":"2023-10-09T09:22:04.009922+0000","last_became_peered":"2023-10-09T09:22:04.009922+0000","last_unstale":"2023-10-09T09:22:33.982984+0000","last_undegraded":"2023-10-09T09:22:33.982984+0000","last_fullsized":"2023-10-09T09:22:33.982984+0000","mapping_epoch":328,"log_start":"298'2500","ondisk_log_start":"298'2500","created":328,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8814,"log_dups_size":0,"ondisk_log_size":8814,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5885,"num_write_kb":188128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":192,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971108+0000","last_change":"2023-10-09T09:13:48.205269+0000","last_active":"2023-10-09T09:22:32.971108+0000","last_peered":"2023-10-09T09:22:32.971108+0000","last_clean":"2023-10-09T09:22:32.971108+0000","last_became_active":"2023-10-09T09:13:48.204894+0000","last_became_peered":"2023-10-09T09:13:48.204894+0000","last_unstale":"2023-10-09T09:22:32.971108+0000","last_undegraded":"2023-10-09T09:22:32.971108+0000","last_fullsized":"2023-10-09T09:22:32.971108+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":336,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.882204+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:22:26.882204+0000","last_peered":"2023-10-09T09:22:26.882204+0000","last_clean":"2023-10-09T09:22:26.882204+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:22:26.882204+0000","last_undegraded":"2023-10-09T09:22:26.882204+0000","last_fullsized":"2023-10-09T09:22:26.882204+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"345'11157","reported_seq":11233,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983281+0000","last_change":"2023-10-09T09:22:04.041348+0000","last_active":"2023-10-09T09:22:33.983281+0000","last_peered":"2023-10-09T09:22:33.983281+0000","last_clean":"2023-10-09T09:22:33.983281+0000","last_became_active":"2023-10-09T09:22:04.003921+0000","last_became_peered":"2023-10-09T09:22:04.003921+0000","last_unstale":"2023-10-09T09:22:33.983281+0000","last_undegraded":"2023-10-09T09:22:33.983281+0000","last_fullsized":"2023-10-09T09:22:33.983281+0000","mapping_epoch":328,"log_start":"298'2300","ondisk_log_start":"298'2300","created":328,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5839,"num_write_kb":185472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":266,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983651+0000","last_change":"2023-10-09T09:15:23.420333+0000","last_active":"2023-10-09T09:22:33.983651+0000","last_peered":"2023-10-09T09:22:33.983651+0000","last_clean":"2023-10-09T09:22:33.983651+0000","last_became_active":"2023-10-09T09:15:23.412581+0000","last_became_peered":"2023-10-09T09:15:23.412581+0000","last_unstale":"2023-10-09T09:22:33.983651+0000","last_undegraded":"2023-10-09T09:22:33.983651+0000","last_fullsized":"2023-10-09T09:22:33.983651+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.968188+0000","last_change":"2023-10-09T09:21:51.915241+0000","last_active":"2023-10-09T09:22:31.968188+0000","last_peered":"2023-10-09T09:22:31.968188+0000","last_clean":"2023-10-09T09:22:31.968188+0000","last_became_active":"2023-10-09T09:21:51.914856+0000","last_became_peered":"2023-10-09T09:21:51.914856+0000","last_unstale":"2023-10-09T09:22:31.968188+0000","last_undegraded":"2023-10-09T09:22:31.968188+0000","last_fullsized":"2023-10-09T09:22:31.968188+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:21.372893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.7","version":"345'11070","reported_seq":11144,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.960277+0000","last_change":"2023-10-09T09:22:03.974501+0000","last_active":"2023-10-09T09:22:31.960277+0000","last_peered":"2023-10-09T09:22:31.960277+0000","last_clean":"2023-10-09T09:22:31.960277+0000","last_became_active":"2023-10-09T09:22:03.946129+0000","last_became_peered":"2023-10-09T09:22:03.946129+0000","last_unstale":"2023-10-09T09:22:31.960277+0000","last_undegraded":"2023-10-09T09:22:31.960277+0000","last_fullsized":"2023-10-09T09:22:31.960277+0000","mapping_epoch":294,"log_start":"301'3100","ondisk_log_start":"301'3100","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":7970,"log_dups_size":0,"ondisk_log_size":7970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5761,"num_write_kb":184224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":147,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.981889+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:22:33.981889+0000","last_peered":"2023-10-09T09:22:33.981889+0000","last_clean":"2023-10-09T09:22:33.981889+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:22:33.981889+0000","last_undegraded":"2023-10-09T09:22:33.981889+0000","last_fullsized":"2023-10-09T09:22:33.981889+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":109,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979950+0000","last_change":"2023-10-09T09:15:23.414649+0000","last_active":"2023-10-09T09:22:33.979950+0000","last_peered":"2023-10-09T09:22:33.979950+0000","last_clean":"2023-10-09T09:22:33.979950+0000","last_became_active":"2023-10-09T09:15:23.414272+0000","last_became_peered":"2023-10-09T09:15:23.414272+0000","last_unstale":"2023-10-09T09:22:33.979950+0000","last_undegraded":"2023-10-09T09:22:33.979950+0000","last_fullsized":"2023-10-09T09:22:33.979950+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"345'11195","reported_seq":14157,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.965395+0000","last_change":"2023-10-09T09:22:04.048892+0000","last_active":"2023-10-09T09:22:31.965395+0000","last_peered":"2023-10-09T09:22:31.965395+0000","last_clean":"2023-10-09T09:22:31.965395+0000","last_became_active":"2023-10-09T09:22:03.973888+0000","last_became_peered":"2023-10-09T09:22:03.973888+0000","last_unstale":"2023-10-09T09:22:31.965395+0000","last_undegraded":"2023-10-09T09:22:31.965395+0000","last_fullsized":"2023-10-09T09:22:31.965395+0000","mapping_epoch":320,"log_start":"298'1900","ondisk_log_start":"298'1900","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":9295,"log_dups_size":0,"ondisk_log_size":9295,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1146880,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5816,"num_write_kb":186688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":716,"num_bytes_recovered":24477696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":147,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.981838+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:22:33.981838+0000","last_peered":"2023-10-09T09:22:33.981838+0000","last_clean":"2023-10-09T09:22:33.981838+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:22:33.981838+0000","last_undegraded":"2023-10-09T09:22:33.981838+0000","last_fullsized":"2023-10-09T09:22:33.981838+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":238,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.970830+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:22:32.970830+0000","last_peered":"2023-10-09T09:22:32.970830+0000","last_clean":"2023-10-09T09:22:32.970830+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:22:32.970830+0000","last_undegraded":"2023-10-09T09:22:32.970830+0000","last_fullsized":"2023-10-09T09:22:32.970830+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"345'10943","reported_seq":11017,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983586+0000","last_change":"2023-10-09T09:22:03.966864+0000","last_active":"2023-10-09T09:22:33.983586+0000","last_peered":"2023-10-09T09:22:33.983586+0000","last_clean":"2023-10-09T09:22:33.983586+0000","last_became_active":"2023-10-09T09:22:03.944820+0000","last_became_peered":"2023-10-09T09:22:03.944820+0000","last_unstale":"2023-10-09T09:22:33.983586+0000","last_undegraded":"2023-10-09T09:22:33.983586+0000","last_fullsized":"2023-10-09T09:22:33.983586+0000","mapping_epoch":294,"log_start":"298'2100","ondisk_log_start":"298'2100","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8843,"log_dups_size":0,"ondisk_log_size":8843,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":786432,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5692,"num_write_kb":182528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":223,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983691+0000","last_change":"2023-10-09T09:15:23.437420+0000","last_active":"2023-10-09T09:22:33.983691+0000","last_peered":"2023-10-09T09:22:33.983691+0000","last_clean":"2023-10-09T09:22:33.983691+0000","last_became_active":"2023-10-09T09:15:23.429358+0000","last_became_peered":"2023-10-09T09:15:23.429358+0000","last_unstale":"2023-10-09T09:22:33.983691+0000","last_undegraded":"2023-10-09T09:22:33.983691+0000","last_fullsized":"2023-10-09T09:22:33.983691+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":253,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983720+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:22:33.983720+0000","last_peered":"2023-10-09T09:22:33.983720+0000","last_clean":"2023-10-09T09:22:33.983720+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:22:33.983720+0000","last_undegraded":"2023-10-09T09:22:33.983720+0000","last_fullsized":"2023-10-09T09:22:33.983720+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"345'11241","reported_seq":12663,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.980519+0000","last_change":"2023-10-09T09:22:03.947099+0000","last_active":"2023-10-09T09:22:33.980519+0000","last_peered":"2023-10-09T09:22:33.980519+0000","last_clean":"2023-10-09T09:22:33.980519+0000","last_became_active":"2023-10-09T09:22:03.924856+0000","last_became_peered":"2023-10-09T09:22:03.924856+0000","last_unstale":"2023-10-09T09:22:33.980519+0000","last_undegraded":"2023-10-09T09:22:33.980519+0000","last_fullsized":"2023-10-09T09:22:33.980519+0000","mapping_epoch":315,"log_start":"298'2200","ondisk_log_start":"298'2200","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":9041,"log_dups_size":0,"ondisk_log_size":9041,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5851,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":331,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":137,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979856+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:22:33.979856+0000","last_peered":"2023-10-09T09:22:33.979856+0000","last_clean":"2023-10-09T09:22:33.979856+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:22:33.979856+0000","last_undegraded":"2023-10-09T09:22:33.979856+0000","last_fullsized":"2023-10-09T09:22:33.979856+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":364,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.880658+0000","last_change":"2023-10-09T09:18:51.869617+0000","last_active":"2023-10-09T09:22:26.880658+0000","last_peered":"2023-10-09T09:22:26.880658+0000","last_clean":"2023-10-09T09:22:26.880658+0000","last_became_active":"2023-10-09T09:18:51.869212+0000","last_became_peered":"2023-10-09T09:18:51.869212+0000","last_unstale":"2023-10-09T09:22:26.880658+0000","last_undegraded":"2023-10-09T09:22:26.880658+0000","last_fullsized":"2023-10-09T09:22:26.880658+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:36.753958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"345'11076","reported_seq":11150,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983040+0000","last_change":"2023-10-09T09:22:03.974224+0000","last_active":"2023-10-09T09:22:33.983040+0000","last_peered":"2023-10-09T09:22:33.983040+0000","last_clean":"2023-10-09T09:22:33.983040+0000","last_became_active":"2023-10-09T09:22:03.938348+0000","last_became_peered":"2023-10-09T09:22:03.938348+0000","last_unstale":"2023-10-09T09:22:33.983040+0000","last_undegraded":"2023-10-09T09:22:33.983040+0000","last_fullsized":"2023-10-09T09:22:33.983040+0000","mapping_epoch":294,"log_start":"298'2200","ondisk_log_start":"298'2200","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8876,"log_dups_size":0,"ondisk_log_size":8876,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5765,"num_write_kb":184416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":157,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.880810+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:22:26.880810+0000","last_peered":"2023-10-09T09:22:26.880810+0000","last_clean":"2023-10-09T09:22:26.880810+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:22:26.880810+0000","last_undegraded":"2023-10-09T09:22:26.880810+0000","last_fullsized":"2023-10-09T09:22:26.880810+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":211,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971707+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:22:32.971707+0000","last_peered":"2023-10-09T09:22:32.971707+0000","last_clean":"2023-10-09T09:22:32.971707+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:22:32.971707+0000","last_undegraded":"2023-10-09T09:22:32.971707+0000","last_fullsized":"2023-10-09T09:22:32.971707+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"345'11192","reported_seq":12571,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:29.932840+0000","last_change":"2023-10-09T09:22:04.067596+0000","last_active":"2023-10-09T09:22:29.932840+0000","last_peered":"2023-10-09T09:22:29.932840+0000","last_clean":"2023-10-09T09:22:29.932840+0000","last_became_active":"2023-10-09T09:22:03.976128+0000","last_became_peered":"2023-10-09T09:22:03.976128+0000","last_unstale":"2023-10-09T09:22:29.932840+0000","last_undegraded":"2023-10-09T09:22:29.932840+0000","last_fullsized":"2023-10-09T09:22:29.932840+0000","mapping_epoch":309,"log_start":"298'1900","ondisk_log_start":"298'1900","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":9292,"log_dups_size":0,"ondisk_log_size":9292,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5834,"num_write_kb":186240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":320,"num_bytes_recovered":491520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":217,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983605+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:22:33.983605+0000","last_peered":"2023-10-09T09:22:33.983605+0000","last_clean":"2023-10-09T09:22:33.983605+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:22:33.983605+0000","last_undegraded":"2023-10-09T09:22:33.983605+0000","last_fullsized":"2023-10-09T09:22:33.983605+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":256,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971925+0000","last_change":"2023-10-09T09:20:19.935938+0000","last_active":"2023-10-09T09:22:32.971925+0000","last_peered":"2023-10-09T09:22:32.971925+0000","last_clean":"2023-10-09T09:22:32.971925+0000","last_became_active":"2023-10-09T09:20:19.935423+0000","last_became_peered":"2023-10-09T09:20:19.935423+0000","last_unstale":"2023-10-09T09:22:32.971925+0000","last_undegraded":"2023-10-09T09:22:32.971925+0000","last_fullsized":"2023-10-09T09:22:32.971925+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"345'11148","reported_seq":11222,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.982781+0000","last_change":"2023-10-09T09:22:03.971960+0000","last_active":"2023-10-09T09:22:33.982781+0000","last_peered":"2023-10-09T09:22:33.982781+0000","last_clean":"2023-10-09T09:22:33.982781+0000","last_became_active":"2023-10-09T09:22:03.934586+0000","last_became_peered":"2023-10-09T09:22:03.934586+0000","last_unstale":"2023-10-09T09:22:33.982781+0000","last_undegraded":"2023-10-09T09:22:33.982781+0000","last_fullsized":"2023-10-09T09:22:33.982781+0000","mapping_epoch":294,"log_start":"298'2300","ondisk_log_start":"298'2300","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8848,"log_dups_size":0,"ondisk_log_size":8848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5802,"num_write_kb":185696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":283,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.970772+0000","last_change":"2023-10-09T09:20:19.755858+0000","last_active":"2023-10-09T09:22:32.970772+0000","last_peered":"2023-10-09T09:22:32.970772+0000","last_clean":"2023-10-09T09:22:32.970772+0000","last_became_active":"2023-10-09T09:20:19.755442+0000","last_became_peered":"2023-10-09T09:20:19.755442+0000","last_unstale":"2023-10-09T09:22:32.970772+0000","last_undegraded":"2023-10-09T09:22:32.970772+0000","last_fullsized":"2023-10-09T09:22:32.970772+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":229,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971259+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:22:32.971259+0000","last_peered":"2023-10-09T09:22:32.971259+0000","last_clean":"2023-10-09T09:22:32.971259+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:22:32.971259+0000","last_undegraded":"2023-10-09T09:22:32.971259+0000","last_fullsized":"2023-10-09T09:22:32.971259+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"345'11350","reported_seq":11426,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949824+0000","last_change":"2023-10-09T09:22:04.008412+0000","last_active":"2023-10-09T09:22:30.949824+0000","last_peered":"2023-10-09T09:22:30.949824+0000","last_clean":"2023-10-09T09:22:30.949824+0000","last_became_active":"2023-10-09T09:22:03.943532+0000","last_became_peered":"2023-10-09T09:22:03.943532+0000","last_unstale":"2023-10-09T09:22:30.949824+0000","last_undegraded":"2023-10-09T09:22:30.949824+0000","last_fullsized":"2023-10-09T09:22:30.949824+0000","mapping_epoch":294,"log_start":"301'3300","ondisk_log_start":"301'3300","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8050,"log_dups_size":0,"ondisk_log_size":8050,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5905,"num_write_kb":188768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949909+0000","last_change":"2023-10-09T09:14:48.306639+0000","last_active":"2023-10-09T09:22:30.949909+0000","last_peered":"2023-10-09T09:22:30.949909+0000","last_clean":"2023-10-09T09:22:30.949909+0000","last_became_active":"2023-10-09T09:14:48.305729+0000","last_became_peered":"2023-10-09T09:14:48.305729+0000","last_unstale":"2023-10-09T09:22:30.949909+0000","last_undegraded":"2023-10-09T09:22:30.949909+0000","last_fullsized":"2023-10-09T09:22:30.949909+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:08.892989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950154+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:22:30.950154+0000","last_peered":"2023-10-09T09:22:30.950154+0000","last_clean":"2023-10-09T09:22:30.950154+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:22:30.950154+0000","last_undegraded":"2023-10-09T09:22:30.950154+0000","last_fullsized":"2023-10-09T09:22:30.950154+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"345'11390","reported_seq":13765,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.968033+0000","last_change":"2023-10-09T09:20:40.409450+0000","last_active":"2023-10-09T09:22:31.968033+0000","last_peered":"2023-10-09T09:22:31.968033+0000","last_clean":"2023-10-09T09:22:31.968033+0000","last_became_active":"2023-10-09T09:20:40.389612+0000","last_became_peered":"2023-10-09T09:20:40.389612+0000","last_unstale":"2023-10-09T09:22:31.968033+0000","last_undegraded":"2023-10-09T09:22:31.968033+0000","last_fullsized":"2023-10-09T09:22:31.968033+0000","mapping_epoch":312,"log_start":"298'1300","ondisk_log_start":"298'1300","created":294,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":10090,"log_dups_size":0,"ondisk_log_size":10090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:24.706615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11390,"num_write_kb":364417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1140,"num_bytes_recovered":30867480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950837+0000","last_change":"2023-10-09T09:14:48.299630+0000","last_active":"2023-10-09T09:22:30.950837+0000","last_peered":"2023-10-09T09:22:30.950837+0000","last_clean":"2023-10-09T09:22:30.950837+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:22:30.950837+0000","last_undegraded":"2023-10-09T09:22:30.950837+0000","last_fullsized":"2023-10-09T09:22:30.950837+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":173,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979629+0000","last_change":"2023-10-09T09:20:19.757594+0000","last_active":"2023-10-09T09:22:33.979629+0000","last_peered":"2023-10-09T09:22:33.979629+0000","last_clean":"2023-10-09T09:22:33.979629+0000","last_became_active":"2023-10-09T09:20:19.757216+0000","last_became_peered":"2023-10-09T09:20:19.757216+0000","last_unstale":"2023-10-09T09:22:33.979629+0000","last_undegraded":"2023-10-09T09:22:33.979629+0000","last_fullsized":"2023-10-09T09:22:33.979629+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949870+0000","last_change":"2023-10-09T09:15:23.413972+0000","last_active":"2023-10-09T09:22:30.949870+0000","last_peered":"2023-10-09T09:22:30.949870+0000","last_clean":"2023-10-09T09:22:30.949870+0000","last_became_active":"2023-10-09T09:15:23.413531+0000","last_became_peered":"2023-10-09T09:15:23.413531+0000","last_unstale":"2023-10-09T09:22:30.949870+0000","last_undegraded":"2023-10-09T09:22:30.949870+0000","last_fullsized":"2023-10-09T09:22:30.949870+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:54.516595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"345'11394","reported_seq":11457,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.951463+0000","last_change":"2023-10-09T09:18:10.914938+0000","last_active":"2023-10-09T09:22:30.951463+0000","last_peered":"2023-10-09T09:22:30.951463+0000","last_clean":"2023-10-09T09:22:30.951463+0000","last_became_active":"2023-10-09T09:18:10.914146+0000","last_became_peered":"2023-10-09T09:18:10.914146+0000","last_unstale":"2023-10-09T09:22:30.951463+0000","last_undegraded":"2023-10-09T09:22:30.951463+0000","last_fullsized":"2023-10-09T09:22:30.951463+0000","mapping_epoch":294,"log_start":"302'4497","ondisk_log_start":"302'4497","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:15.765459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11394,"num_write_kb":364608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":146,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979290+0000","last_change":"2023-10-09T09:14:48.319163+0000","last_active":"2023-10-09T09:22:33.979290+0000","last_peered":"2023-10-09T09:22:33.979290+0000","last_clean":"2023-10-09T09:22:33.979290+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:22:33.979290+0000","last_undegraded":"2023-10-09T09:22:33.979290+0000","last_fullsized":"2023-10-09T09:22:33.979290+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":211,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971486+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:22:32.971486+0000","last_peered":"2023-10-09T09:22:32.971486+0000","last_clean":"2023-10-09T09:22:32.971486+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:22:32.971486+0000","last_undegraded":"2023-10-09T09:22:32.971486+0000","last_fullsized":"2023-10-09T09:22:32.971486+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":123,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950757+0000","last_change":"2023-10-09T09:20:19.932428+0000","last_active":"2023-10-09T09:22:30.950757+0000","last_peered":"2023-10-09T09:22:30.950757+0000","last_clean":"2023-10-09T09:22:30.950757+0000","last_became_active":"2023-10-09T09:20:19.931684+0000","last_became_peered":"2023-10-09T09:20:19.931684+0000","last_unstale":"2023-10-09T09:22:30.950757+0000","last_undegraded":"2023-10-09T09:22:30.950757+0000","last_fullsized":"2023-10-09T09:22:30.950757+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:40.774870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"345'11072","reported_seq":11145,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.982753+0000","last_change":"2023-10-09T09:22:03.989391+0000","last_active":"2023-10-09T09:22:33.982753+0000","last_peered":"2023-10-09T09:22:33.982753+0000","last_clean":"2023-10-09T09:22:33.982753+0000","last_became_active":"2023-10-09T09:22:03.941843+0000","last_became_peered":"2023-10-09T09:22:03.941843+0000","last_unstale":"2023-10-09T09:22:33.982753+0000","last_undegraded":"2023-10-09T09:22:33.982753+0000","last_fullsized":"2023-10-09T09:22:33.982753+0000","mapping_epoch":294,"log_start":"298'2300","ondisk_log_start":"298'2300","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8772,"log_dups_size":0,"ondisk_log_size":8772,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2785280,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5753,"num_write_kb":185472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":52,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.967735+0000","last_change":"2023-10-09T09:20:19.757802+0000","last_active":"2023-10-09T09:22:31.967735+0000","last_peered":"2023-10-09T09:22:31.967735+0000","last_clean":"2023-10-09T09:22:31.967735+0000","last_became_active":"2023-10-09T09:20:19.757450+0000","last_became_peered":"2023-10-09T09:20:19.757450+0000","last_unstale":"2023-10-09T09:22:31.967735+0000","last_undegraded":"2023-10-09T09:22:31.967735+0000","last_fullsized":"2023-10-09T09:22:31.967735+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:57.053230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949439+0000","last_change":"2023-10-09T09:20:19.902503+0000","last_active":"2023-10-09T09:22:30.949439+0000","last_peered":"2023-10-09T09:22:30.949439+0000","last_clean":"2023-10-09T09:22:30.949439+0000","last_became_active":"2023-10-09T09:20:19.902131+0000","last_became_peered":"2023-10-09T09:20:19.902131+0000","last_unstale":"2023-10-09T09:22:30.949439+0000","last_undegraded":"2023-10-09T09:22:30.949439+0000","last_fullsized":"2023-10-09T09:22:30.949439+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:43.897708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.967382+0000","last_change":"2023-10-09T09:21:51.904699+0000","last_active":"2023-10-09T09:22:31.967382+0000","last_peered":"2023-10-09T09:22:31.967382+0000","last_clean":"2023-10-09T09:22:31.967382+0000","last_became_active":"2023-10-09T09:21:51.904381+0000","last_became_peered":"2023-10-09T09:21:51.904381+0000","last_unstale":"2023-10-09T09:22:31.967382+0000","last_undegraded":"2023-10-09T09:22:31.967382+0000","last_fullsized":"2023-10-09T09:22:31.967382+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:46.915626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.a","version":"345'11762","reported_seq":11826,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.982936+0000","last_change":"2023-10-09T09:18:10.917223+0000","last_active":"2023-10-09T09:22:33.982936+0000","last_peered":"2023-10-09T09:22:33.982936+0000","last_clean":"2023-10-09T09:22:33.982936+0000","last_became_active":"2023-10-09T09:18:10.916265+0000","last_became_peered":"2023-10-09T09:18:10.916265+0000","last_unstale":"2023-10-09T09:22:33.982936+0000","last_undegraded":"2023-10-09T09:22:33.982936+0000","last_fullsized":"2023-10-09T09:22:33.982936+0000","mapping_epoch":294,"log_start":"302'4548","ondisk_log_start":"302'4548","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":7214,"log_dups_size":0,"ondisk_log_size":7214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11762,"num_write_kb":376384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":182,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979605+0000","last_change":"2023-10-09T09:20:19.759670+0000","last_active":"2023-10-09T09:22:33.979605+0000","last_peered":"2023-10-09T09:22:33.979605+0000","last_clean":"2023-10-09T09:22:33.979605+0000","last_became_active":"2023-10-09T09:20:19.759254+0000","last_became_peered":"2023-10-09T09:20:19.759254+0000","last_unstale":"2023-10-09T09:22:33.979605+0000","last_undegraded":"2023-10-09T09:22:33.979605+0000","last_fullsized":"2023-10-09T09:22:33.979605+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":588,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979851+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:22:33.979851+0000","last_peered":"2023-10-09T09:22:33.979851+0000","last_clean":"2023-10-09T09:22:33.979851+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:22:33.979851+0000","last_undegraded":"2023-10-09T09:22:33.979851+0000","last_fullsized":"2023-10-09T09:22:33.979851+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":176,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.980964+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:22:33.980964+0000","last_peered":"2023-10-09T09:22:33.980964+0000","last_clean":"2023-10-09T09:22:33.980964+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:22:33.980964+0000","last_undegraded":"2023-10-09T09:22:33.980964+0000","last_fullsized":"2023-10-09T09:22:33.980964+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":208,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983758+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:22:33.983758+0000","last_peered":"2023-10-09T09:22:33.983758+0000","last_clean":"2023-10-09T09:22:33.983758+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:22:33.983758+0000","last_undegraded":"2023-10-09T09:22:33.983758+0000","last_fullsized":"2023-10-09T09:22:33.983758+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"345'11302","reported_seq":11378,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.982781+0000","last_change":"2023-10-09T09:22:04.060640+0000","last_active":"2023-10-09T09:22:33.982781+0000","last_peered":"2023-10-09T09:22:33.982781+0000","last_clean":"2023-10-09T09:22:33.982781+0000","last_became_active":"2023-10-09T09:22:03.984721+0000","last_became_peered":"2023-10-09T09:22:03.984721+0000","last_unstale":"2023-10-09T09:22:33.982781+0000","last_undegraded":"2023-10-09T09:22:33.982781+0000","last_fullsized":"2023-10-09T09:22:33.982781+0000","mapping_epoch":294,"log_start":"298'2500","ondisk_log_start":"298'2500","created":294,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8802,"log_dups_size":0,"ondisk_log_size":8802,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5873,"num_write_kb":188128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949552+0000","last_change":"2023-10-09T09:14:48.301697+0000","last_active":"2023-10-09T09:22:30.949552+0000","last_peered":"2023-10-09T09:22:30.949552+0000","last_clean":"2023-10-09T09:22:30.949552+0000","last_became_active":"2023-10-09T09:14:48.301379+0000","last_became_peered":"2023-10-09T09:14:48.301379+0000","last_unstale":"2023-10-09T09:22:30.949552+0000","last_undegraded":"2023-10-09T09:22:30.949552+0000","last_fullsized":"2023-10-09T09:22:30.949552+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:11.963433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":383,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.880346+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:22:26.880346+0000","last_peered":"2023-10-09T09:22:26.880346+0000","last_clean":"2023-10-09T09:22:26.880346+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:22:26.880346+0000","last_undegraded":"2023-10-09T09:22:26.880346+0000","last_fullsized":"2023-10-09T09:22:26.880346+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":406,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.881754+0000","last_change":"2023-10-09T09:20:19.890699+0000","last_active":"2023-10-09T09:22:26.881754+0000","last_peered":"2023-10-09T09:22:26.881754+0000","last_clean":"2023-10-09T09:22:26.881754+0000","last_became_active":"2023-10-09T09:20:19.889949+0000","last_became_peered":"2023-10-09T09:20:19.889949+0000","last_unstale":"2023-10-09T09:22:26.881754+0000","last_undegraded":"2023-10-09T09:22:26.881754+0000","last_fullsized":"2023-10-09T09:22:26.881754+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:59.920953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"345'11638","reported_seq":13034,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950426+0000","last_change":"2023-10-09T09:21:45.577572+0000","last_active":"2023-10-09T09:22:30.950426+0000","last_peered":"2023-10-09T09:22:30.950426+0000","last_clean":"2023-10-09T09:22:30.950426+0000","last_became_active":"2023-10-09T09:21:45.553863+0000","last_became_peered":"2023-10-09T09:21:45.553863+0000","last_unstale":"2023-10-09T09:22:30.950426+0000","last_undegraded":"2023-10-09T09:22:30.950426+0000","last_fullsized":"2023-10-09T09:22:30.950426+0000","mapping_epoch":322,"log_start":"302'4740","ondisk_log_start":"302'4740","created":294,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:18.810085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11638,"num_write_kb":372416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":655,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":130,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.980884+0000","last_change":"2023-10-09T09:09:44.439090+0000","last_active":"2023-10-09T09:22:33.980884+0000","last_peered":"2023-10-09T09:22:33.980884+0000","last_clean":"2023-10-09T09:22:33.980884+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:22:33.980884+0000","last_undegraded":"2023-10-09T09:22:33.980884+0000","last_fullsized":"2023-10-09T09:22:33.980884+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950529+0000","last_change":"2023-10-09T09:20:19.912177+0000","last_active":"2023-10-09T09:22:30.950529+0000","last_peered":"2023-10-09T09:22:30.950529+0000","last_clean":"2023-10-09T09:22:30.950529+0000","last_became_active":"2023-10-09T09:20:19.908966+0000","last_became_peered":"2023-10-09T09:20:19.908966+0000","last_unstale":"2023-10-09T09:22:30.950529+0000","last_undegraded":"2023-10-09T09:22:30.950529+0000","last_fullsized":"2023-10-09T09:22:30.950529+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:54.778943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":359,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.881150+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:22:26.881150+0000","last_peered":"2023-10-09T09:22:26.881150+0000","last_clean":"2023-10-09T09:22:26.881150+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:22:26.881150+0000","last_undegraded":"2023-10-09T09:22:26.881150+0000","last_fullsized":"2023-10-09T09:22:26.881150+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"345'12062","reported_seq":15850,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.949975+0000","last_change":"2023-10-09T09:21:23.439146+0000","last_active":"2023-10-09T09:22:30.949975+0000","last_peered":"2023-10-09T09:22:30.949975+0000","last_clean":"2023-10-09T09:22:30.949975+0000","last_became_active":"2023-10-09T09:21:23.402629+0000","last_became_peered":"2023-10-09T09:21:23.402629+0000","last_unstale":"2023-10-09T09:22:30.949975+0000","last_undegraded":"2023-10-09T09:22:30.949975+0000","last_fullsized":"2023-10-09T09:22:30.949975+0000","mapping_epoch":318,"log_start":"302'5190","ondisk_log_start":"302'5190","created":294,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":6872,"log_dups_size":0,"ondisk_log_size":6872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:17.924211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12062,"num_write_kb":385984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1850,"num_bytes_recovered":73662464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":115,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.880559+0000","last_change":"2023-10-09T09:14:19.749169+0000","last_active":"2023-10-09T09:22:26.880559+0000","last_peered":"2023-10-09T09:22:26.880559+0000","last_clean":"2023-10-09T09:22:26.880559+0000","last_became_active":"2023-10-09T09:14:19.740888+0000","last_became_peered":"2023-10-09T09:14:19.740888+0000","last_unstale":"2023-10-09T09:22:26.880559+0000","last_undegraded":"2023-10-09T09:22:26.880559+0000","last_fullsized":"2023-10-09T09:22:26.880559+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:01.361061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.951666+0000","last_change":"2023-10-09T09:14:48.317228+0000","last_active":"2023-10-09T09:22:30.951666+0000","last_peered":"2023-10-09T09:22:30.951666+0000","last_clean":"2023-10-09T09:22:30.951666+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:22:30.951666+0000","last_undegraded":"2023-10-09T09:22:30.951666+0000","last_fullsized":"2023-10-09T09:22:30.951666+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":130,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979652+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:22:33.979652+0000","last_peered":"2023-10-09T09:22:33.979652+0000","last_clean":"2023-10-09T09:22:33.979652+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:22:33.979652+0000","last_undegraded":"2023-10-09T09:22:33.979652+0000","last_fullsized":"2023-10-09T09:22:33.979652+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"345'11804","reported_seq":11867,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971462+0000","last_change":"2023-10-09T09:18:10.373277+0000","last_active":"2023-10-09T09:22:32.971462+0000","last_peered":"2023-10-09T09:22:32.971462+0000","last_clean":"2023-10-09T09:22:32.971462+0000","last_became_active":"2023-10-09T09:18:10.372954+0000","last_became_peered":"2023-10-09T09:18:10.372954+0000","last_unstale":"2023-10-09T09:22:32.971462+0000","last_undegraded":"2023-10-09T09:22:32.971462+0000","last_fullsized":"2023-10-09T09:22:32.971462+0000","mapping_epoch":294,"log_start":"302'4477","ondisk_log_start":"302'4477","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11804,"num_write_kb":377728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":244,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983509+0000","last_change":"2023-10-09T09:20:19.758027+0000","last_active":"2023-10-09T09:22:33.983509+0000","last_peered":"2023-10-09T09:22:33.983509+0000","last_clean":"2023-10-09T09:22:33.983509+0000","last_became_active":"2023-10-09T09:20:19.757552+0000","last_became_peered":"2023-10-09T09:20:19.757552+0000","last_unstale":"2023-10-09T09:22:33.983509+0000","last_undegraded":"2023-10-09T09:22:33.983509+0000","last_fullsized":"2023-10-09T09:22:33.983509+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":401,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.881918+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:22:26.881918+0000","last_peered":"2023-10-09T09:22:26.881918+0000","last_clean":"2023-10-09T09:22:26.881918+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:22:26.881918+0000","last_undegraded":"2023-10-09T09:22:26.881918+0000","last_fullsized":"2023-10-09T09:22:26.881918+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.970985+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:22:32.970985+0000","last_peered":"2023-10-09T09:22:32.970985+0000","last_clean":"2023-10-09T09:22:32.970985+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:22:32.970985+0000","last_undegraded":"2023-10-09T09:22:32.970985+0000","last_fullsized":"2023-10-09T09:22:32.970985+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"345'11072","reported_seq":11486,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971692+0000","last_change":"2023-10-09T09:22:25.953183+0000","last_active":"2023-10-09T09:22:32.971692+0000","last_peered":"2023-10-09T09:22:32.971692+0000","last_clean":"2023-10-09T09:22:32.971692+0000","last_became_active":"2023-10-09T09:22:25.892008+0000","last_became_peered":"2023-10-09T09:22:25.892008+0000","last_unstale":"2023-10-09T09:22:32.971692+0000","last_undegraded":"2023-10-09T09:22:32.971692+0000","last_fullsized":"2023-10-09T09:22:32.971692+0000","mapping_epoch":343,"log_start":"298'2200","ondisk_log_start":"298'2200","created":328,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8872,"log_dups_size":0,"ondisk_log_size":8872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131072,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5761,"num_write_kb":184416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":7077888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":369,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.881036+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:22:26.881036+0000","last_peered":"2023-10-09T09:22:26.881036+0000","last_clean":"2023-10-09T09:22:26.881036+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:22:26.881036+0000","last_undegraded":"2023-10-09T09:22:26.881036+0000","last_fullsized":"2023-10-09T09:22:26.881036+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":51,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.967924+0000","last_change":"2023-10-09T09:20:19.916151+0000","last_active":"2023-10-09T09:22:31.967924+0000","last_peered":"2023-10-09T09:22:31.967924+0000","last_clean":"2023-10-09T09:22:31.967924+0000","last_became_active":"2023-10-09T09:20:19.915607+0000","last_became_peered":"2023-10-09T09:20:19.915607+0000","last_unstale":"2023-10-09T09:22:31.967924+0000","last_undegraded":"2023-10-09T09:22:31.967924+0000","last_fullsized":"2023-10-09T09:22:31.967924+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:57:51.697949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"345'11250","reported_seq":12978,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983061+0000","last_change":"2023-10-09T09:22:26.926149+0000","last_active":"2023-10-09T09:22:33.983061+0000","last_peered":"2023-10-09T09:22:33.983061+0000","last_clean":"2023-10-09T09:22:33.983061+0000","last_became_active":"2023-10-09T09:22:26.899573+0000","last_became_peered":"2023-10-09T09:22:26.899573+0000","last_unstale":"2023-10-09T09:22:33.983061+0000","last_undegraded":"2023-10-09T09:22:33.983061+0000","last_fullsized":"2023-10-09T09:22:33.983061+0000","mapping_epoch":344,"log_start":"298'2200","ondisk_log_start":"298'2200","created":328,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":9050,"log_dups_size":0,"ondisk_log_size":9050,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5861,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":6684672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":154,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979803+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:22:33.979803+0000","last_peered":"2023-10-09T09:22:33.979803+0000","last_clean":"2023-10-09T09:22:33.979803+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:22:33.979803+0000","last_undegraded":"2023-10-09T09:22:33.979803+0000","last_fullsized":"2023-10-09T09:22:33.979803+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":229,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.971240+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:22:32.971240+0000","last_peered":"2023-10-09T09:22:32.971240+0000","last_clean":"2023-10-09T09:22:32.971240+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:22:32.971240+0000","last_undegraded":"2023-10-09T09:22:32.971240+0000","last_fullsized":"2023-10-09T09:22:32.971240+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"345'10967","reported_seq":11035,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:29.771325+0000","last_change":"2023-10-09T09:22:04.005852+0000","last_active":"2023-10-09T09:22:29.771325+0000","last_peered":"2023-10-09T09:22:29.771325+0000","last_clean":"2023-10-09T09:22:29.771325+0000","last_became_active":"2023-10-09T09:22:03.974306+0000","last_became_peered":"2023-10-09T09:22:03.974306+0000","last_unstale":"2023-10-09T09:22:29.771325+0000","last_undegraded":"2023-10-09T09:22:29.771325+0000","last_fullsized":"2023-10-09T09:22:29.771325+0000","mapping_epoch":328,"log_start":"298'2100","ondisk_log_start":"298'2100","created":328,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":8867,"log_dups_size":0,"ondisk_log_size":8867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5716,"num_write_kb":182528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":141,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.979529+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:22:33.979529+0000","last_peered":"2023-10-09T09:22:33.979529+0000","last_clean":"2023-10-09T09:22:33.979529+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:22:33.979529+0000","last_undegraded":"2023-10-09T09:22:33.979529+0000","last_fullsized":"2023-10-09T09:22:33.979529+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":211,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.970831+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:22:32.970831+0000","last_peered":"2023-10-09T09:22:32.970831+0000","last_clean":"2023-10-09T09:22:32.970831+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:22:32.970831+0000","last_undegraded":"2023-10-09T09:22:32.970831+0000","last_fullsized":"2023-10-09T09:22:32.970831+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"345'11230","reported_seq":14186,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:29.829008+0000","last_change":"2023-10-09T09:22:04.035302+0000","last_active":"2023-10-09T09:22:29.829008+0000","last_peered":"2023-10-09T09:22:29.829008+0000","last_clean":"2023-10-09T09:22:29.829008+0000","last_became_active":"2023-10-09T09:22:04.008718+0000","last_became_peered":"2023-10-09T09:22:04.008718+0000","last_unstale":"2023-10-09T09:22:29.829008+0000","last_undegraded":"2023-10-09T09:22:29.829008+0000","last_fullsized":"2023-10-09T09:22:29.829008+0000","mapping_epoch":328,"log_start":"298'1900","ondisk_log_start":"298'1900","created":328,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":9330,"log_dups_size":0,"ondisk_log_size":9330,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5852,"num_write_kb":186688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":715,"num_bytes_recovered":24477696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":331,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T09:22:33.983422+0000","last_change":"2023-10-09T09:20:19.760410+0000","last_active":"2023-10-09T09:22:33.983422+0000","last_peered":"2023-10-09T09:22:33.983422+0000","last_clean":"2023-10-09T09:22:33.983422+0000","last_became_active":"2023-10-09T09:20:19.759916+0000","last_became_peered":"2023-10-09T09:20:19.759916+0000","last_unstale":"2023-10-09T09:22:33.983422+0000","last_undegraded":"2023-10-09T09:22:33.983422+0000","last_fullsized":"2023-10-09T09:22:33.983422+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":51,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.967629+0000","last_change":"2023-10-09T09:20:19.759403+0000","last_active":"2023-10-09T09:22:31.967629+0000","last_peered":"2023-10-09T09:22:31.967629+0000","last_clean":"2023-10-09T09:22:31.967629+0000","last_became_active":"2023-10-09T09:20:19.759030+0000","last_became_peered":"2023-10-09T09:20:19.759030+0000","last_unstale":"2023-10-09T09:22:31.967629+0000","last_undegraded":"2023-10-09T09:22:31.967629+0000","last_fullsized":"2023-10-09T09:22:31.967629+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:10.903233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.17","version":"345'11062","reported_seq":11135,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.950093+0000","last_change":"2023-10-09T09:22:04.028768+0000","last_active":"2023-10-09T09:22:30.950093+0000","last_peered":"2023-10-09T09:22:30.950093+0000","last_clean":"2023-10-09T09:22:30.950093+0000","last_became_active":"2023-10-09T09:22:03.996668+0000","last_became_peered":"2023-10-09T09:22:03.996668+0000","last_unstale":"2023-10-09T09:22:30.950093+0000","last_undegraded":"2023-10-09T09:22:30.950093+0000","last_fullsized":"2023-10-09T09:22:30.950093+0000","mapping_epoch":328,"log_start":"301'3100","ondisk_log_start":"301'3100","created":328,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:18:09.351169+0000","last_clean_scrub_stamp":"2023-10-09T09:18:09.351169+0000","objects_scrubbed":0,"log_size":7962,"log_dups_size":0,"ondisk_log_size":7962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5753,"num_write_kb":184224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":489,"reported_epoch":345,"state":"active+clean","last_fresh":"2023-10-09T09:22:26.880504+0000","last_change":"2023-10-09T09:20:19.788084+0000","last_active":"2023-10-09T09:22:26.880504+0000","last_peered":"2023-10-09T09:22:26.880504+0000","last_clean":"2023-10-09T09:22:26.880504+0000","last_became_active":"2023-10-09T09:20:19.762112+0000","last_became_peered":"2023-10-09T09:20:19.762112+0000","last_unstale":"2023-10-09T09:22:26.880504+0000","last_undegraded":"2023-10-09T09:22:26.880504+0000","last_fullsized":"2023-10-09T09:22:26.880504+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:47.406781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":238,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.970481+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:22:32.970481+0000","last_peered":"2023-10-09T09:22:32.970481+0000","last_clean":"2023-10-09T09:22:32.970481+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:22:32.970481+0000","last_undegraded":"2023-10-09T09:22:32.970481+0000","last_fullsized":"2023-10-09T09:22:32.970481+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":7176192,"num_objects":108,"num_object_clones":0,"num_object_copies":324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186272,"num_write_kb":5960641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7423,"num_bytes_recovered":224264216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":220989,"ondisk_log_size":220989,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":97,"num_bytes_recovered":20970,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430825,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":911048,"kb_used_data":3648,"kb_used_omap":0,"kb_used_meta":907392,"kb_avail":92834104,"statfs":{"total":95995035648,"available":95062122496,"internally_reserved":0,"allocated":3735552,"data_stored":3127264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929169408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.927,"15min":2.077},"min":{"1min":0.248,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":15.178000000000001,"5min":50.515999999999998,"15min":51.328000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.982,"5min":2.153,"15min":2.0510000000000002},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28799999999999998},"max":{"1min":15.199,"5min":50.621000000000002,"15min":51.429000000000002},"last":0.76200000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.5669999999999999,"15min":1.421},"min":{"1min":0.29999999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":14.287000000000001,"5min":14.287000000000001,"15min":21.800999999999998},"last":0.73699999999999999},{"interface":"front","average":{"1min":2.5030000000000001,"5min":1.6140000000000001,"15min":1.3660000000000001},"min":{"1min":0.49199999999999999,"5min":0.35499999999999998,"15min":0.316},"max":{"1min":14.32,"5min":14.32,"15min":21.734000000000002},"last":0.88300000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.4630000000000001,"15min":1.659},"min":{"1min":0.371,"5min":0.32800000000000001,"15min":0.31},"max":{"1min":4.577,"5min":11.186,"15min":45.345999999999997},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.1339999999999999,"15min":1.6240000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.30599999999999999},"max":{"1min":5.6669999999999998,"5min":8.0709999999999997,"15min":46.228000000000002},"last":0.81000000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:22:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":1.6060000000000001,"15min":2.181},"min":{"1min":0.374,"5min":0.374,"15min":0.316},"max":{"1min":7.2469999999999999,"5min":18.178999999999998,"15min":62.060000000000002},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.397,"5min":1.48,"15min":2.1459999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":6.6130000000000004,"5min":17.931999999999999,"15min":63.268999999999998},"last":0.98099999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.1100000000000001,"15min":1.5329999999999999},"min":{"1min":0.34100000000000003,"5min":0.32000000000000001,"15min":0.28699999999999998},"max":{"1min":7.1799999999999997,"5min":7.1799999999999997,"15min":13.895},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.92500000000000004,"15min":1.458},"min":{"1min":0.377,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.1989999999999998,"5min":6.6669999999999998,"15min":16.433},"last":0.94699999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":0.879,"15min":1.3580000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":7.2789999999999999,"5min":7.2789999999999999,"15min":15.616},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.24,"5min":1.1020000000000001,"15min":1.6759999999999999},"min":{"1min":0.436,"5min":0.38,"15min":0.34499999999999997},"max":{"1min":4.7409999999999997,"5min":6.8029999999999999,"15min":18.501000000000001},"last":1.099}]},{"osd":6,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":1.4019999999999999,"15min":1.6890000000000001},"min":{"1min":0.53600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.7009999999999996,"5min":12.898,"15min":21.145},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.161,"15min":1.6859999999999999},"min":{"1min":0.41199999999999998,"5min":0.38900000000000001,"15min":0.34999999999999998},"max":{"1min":2.569,"5min":8.0779999999999994,"15min":19.068999999999999},"last":0.872}]}]},{"osd":6,"up_from":42,"seq":180388627058,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1506740,"kb_used_data":3628,"kb_used_omap":0,"kb_used_meta":1503104,"kb_avail":92238412,"statfs":{"total":95995035648,"available":94452133888,"internally_reserved":0,"allocated":3715072,"data_stored":3119196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1539178496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":1.5489999999999999,"15min":2.915},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.298999999999999,"5min":12.298999999999999,"15min":120.812},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.036,"5min":1.1950000000000001,"15min":2.7360000000000002},"min":{"1min":0.34100000000000003,"5min":0.25600000000000001,"15min":0.24099999999999999},"max":{"1min":4.3170000000000002,"5min":12.01,"15min":120.857},"last":0.64800000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:22:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":3.6099999999999999,"15min":2.2719999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28100000000000003},"max":{"1min":11.792,"5min":142.31100000000001,"15min":142.31100000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":2.468,"5min":3.6869999999999998,"15min":2.1509999999999998},"min":{"1min":0.318,"5min":0.29799999999999999,"15min":0.25700000000000001},"max":{"1min":8.2249999999999996,"5min":132.48099999999999,"15min":132.48099999999999},"last":0.85799999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6320000000000001,"5min":2.4009999999999998,"15min":2.0449999999999999},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.28999999999999998},"max":{"1min":10.488,"5min":82.170000000000002,"15min":82.170000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.843,"5min":2.0699999999999998,"15min":1.9139999999999999},"min":{"1min":0.35199999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.681,"5min":82.352000000000004,"15min":82.352000000000004},"last":0.53900000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:22:22 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":2.0089999999999999,"15min":1.865},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28999999999999998},"max":{"1min":7.0949999999999998,"5min":14.525,"15min":16.977},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.3400000000000001,"15min":1.6060000000000001},"min":{"1min":0.42399999999999999,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":4.1449999999999996,"5min":6.1440000000000001,"15min":20.573},"last":0.623}]},{"osd":4,"last update":"Mon Oct 9 09:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.171,"15min":1.6739999999999999},"min":{"1min":0.33800000000000002,"5min":0.33000000000000002,"15min":0.27600000000000002},"max":{"1min":7.8579999999999997,"5min":11.433999999999999,"15min":29.585999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.997,"5min":1.0289999999999999,"15min":1.647},"min":{"1min":0.36099999999999999,"5min":0.316,"15min":0.316},"max":{"1min":5.4779999999999998,"5min":6.0220000000000002,"15min":20.318000000000001},"last":0.79300000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.1220000000000001,"15min":1.708},"min":{"1min":0.377,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":4.7999999999999998,"5min":11.567,"15min":18.103000000000002},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.0820000000000001,"15min":1.617},"min":{"1min":0.39100000000000001,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":1.6160000000000001,"5min":6.1509999999999998,"15min":13.609},"last":0.73399999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.252,"15min":1.77},"min":{"1min":0.44500000000000001,"5min":0.38500000000000001,"15min":0.33700000000000002},"max":{"1min":7.8140000000000001,"5min":11.597,"15min":29.530000000000001},"last":0.75},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.2390000000000001,"15min":1.5680000000000001},"min":{"1min":0.46700000000000003,"5min":0.39600000000000002,"15min":0.36199999999999999},"max":{"1min":3.2450000000000001,"5min":7.9489999999999998,"15min":13.523999999999999},"last":0.68899999999999995}]}]},{"osd":1,"up_from":83,"seq":356482286005,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":623388,"kb_used_data":3732,"kb_used_omap":9,"kb_used_meta":619638,"kb_avail":93121764,"statfs":{"total":95995035648,"available":95356686336,"internally_reserved":0,"allocated":3821568,"data_stored":2591563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":634510313},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":4.0659999999999998,"15min":2.516},"min":{"1min":0.47499999999999998,"5min":0.215,"15min":0.189},"max":{"1min":4.0090000000000003,"5min":133.76499999999999,"15min":133.76499999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.5960000000000001,"5min":2.7909999999999999,"15min":2.3359999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.23300000000000001},"max":{"1min":10.225,"5min":127.395,"15min":127.395},"last":0.60399999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":3.6760000000000002,"15min":1.8500000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":6.6779999999999999,"5min":236.29599999999999,"15min":236.29599999999999},"last":0.90300000000000002},{"interface":"front","average":{"1min":2.25,"5min":3.702,"15min":1.7010000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.252},"max":{"1min":14.638999999999999,"5min":236.21000000000001,"15min":236.21000000000001},"last":0.56200000000000006}]},{"osd":3,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":3.0750000000000002,"15min":2.165},"min":{"1min":0.308,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":8.2810000000000006,"5min":152.637,"15min":152.637},"last":0.76100000000000001},{"interface":"front","average":{"1min":2.3799999999999999,"5min":4.9509999999999996,"15min":2.7850000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.22600000000000001},"max":{"1min":7.6870000000000003,"5min":152.374,"15min":152.374},"last":0.59399999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":1.468,"15min":1.3839999999999999},"min":{"1min":0.40300000000000002,"5min":0.38,"15min":0.376},"max":{"1min":8.1470000000000002,"5min":8.3059999999999992,"15min":8.3059999999999992},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.829,"5min":3.0670000000000002,"15min":1.837},"min":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.2190000000000003,"5min":133.702,"15min":133.702},"last":0.79400000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":1.3500000000000001,"15min":1.282},"min":{"1min":0.441,"5min":0.432,"15min":0.33800000000000002},"max":{"1min":6.0060000000000002,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":1.1000000000000001},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.3440000000000001,"15min":1.2130000000000001},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.2100000000000009,"5min":8.2100000000000009,"15min":8.2100000000000009},"last":0.98799999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":3.1779999999999999,"15min":2.0209999999999999},"min":{"1min":0.42999999999999999,"5min":0.40100000000000002,"15min":0.33200000000000002},"max":{"1min":3.6859999999999999,"5min":133.739,"15min":133.739},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.319,"15min":1.4399999999999999},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.377},"max":{"1min":3.9689999999999999,"5min":7.3040000000000003,"15min":8.9079999999999995},"last":1.2190000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:21:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.633,"15min":1.3919999999999999},"min":{"1min":0.46899999999999997,"5min":0.41499999999999998,"15min":0.38500000000000001},"max":{"1min":8.2469999999999999,"5min":19.861999999999998,"15min":19.861999999999998},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.7889999999999999,"5min":3.1789999999999998,"15min":1.95},"min":{"1min":0.47599999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":3.7040000000000002,"5min":133.66499999999999,"15min":133.66499999999999},"last":0.879}]}]},{"osd":0,"up_from":78,"seq":335007449531,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":984684,"kb_used_data":4324,"kb_used_omap":28,"kb_used_meta":980323,"kb_avail":92760468,"statfs":{"total":95995035648,"available":94986719232,"internally_reserved":0,"allocated":4427776,"data_stored":3168341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1003851600},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:22:29 2023","interfaces":[{"interface":"back","average":{"1min":6.1390000000000002,"5min":2.3889999999999998,"15min":1.496},"min":{"1min":0.36499999999999999,"5min":0.19700000000000001,"15min":0.193},"max":{"1min":83.671999999999997,"5min":83.671999999999997,"15min":83.671999999999997},"last":0.69599999999999995},{"interface":"front","average":{"1min":5.867,"5min":2.298,"15min":1.407},"min":{"1min":0.43099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":83.953000000000003,"5min":83.953000000000003,"15min":83.953000000000003},"last":3.0800000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.5740000000000001,"15min":1.968},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":9.5020000000000007,"5min":9.5020000000000007,"15min":74.200999999999993},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.5580000000000001,"5min":1.403,"15min":2.008},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":73.983999999999995},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.677,"15min":1.788},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.976,"5min":9.2569999999999997,"15min":20.364000000000001},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.329,"15min":1.9770000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.105,"5min":9.9269999999999996,"15min":137.94999999999999},"last":0.55400000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":1.1240000000000001,"15min":1.6299999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":3.0950000000000002,"5min":8.8040000000000003,"15min":14.163},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.718,"5min":1.486,"15min":1.923},"min":{"1min":0.40200000000000002,"5min":0.314,"15min":0.314},"max":{"1min":5.7290000000000001,"5min":9.5359999999999996,"15min":11.346},"last":0.628}]},{"osd":5,"last update":"Mon Oct 9 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.129,"15min":1.585},"min":{"1min":0.38100000000000001,"5min":0.316,"15min":0.316},"max":{"1min":3.282,"5min":5.5919999999999996,"15min":9.9770000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.1719999999999999,"15min":1.5209999999999999},"min":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.35299999999999998},"max":{"1min":4.335,"5min":8.8279999999999994,"15min":16.364000000000001},"last":0.754}]},{"osd":6,"last update":"Mon Oct 9 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.619,"15min":2.121},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.709,"5min":10.709,"15min":98.864000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.177,"5min":1.6160000000000001,"15min":2.1970000000000001},"min":{"1min":0.53700000000000003,"5min":0.39400000000000002,"15min":0.38300000000000001},"max":{"1min":10.656000000000001,"5min":16.911000000000001,"15min":98.878},"last":0.79600000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.323,"15min":1.593},"min":{"1min":0.498,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":2.4569999999999999,"5min":8.8640000000000008,"15min":9.6780000000000008},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.871,"5min":1.6950000000000001,"15min":1.9419999999999999},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.1310000000000002,"5min":9.5850000000000009,"15min":10.734},"last":4.3869999999999996}]}]},{"osd":2,"up_from":88,"seq":377957122479,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":930256,"kb_used_data":3784,"kb_used_omap":81,"kb_used_meta":926382,"kb_avail":92814896,"statfs":{"total":95995035648,"available":95042453504,"internally_reserved":0,"allocated":3874816,"data_stored":2602312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":948615386},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":7.1509999999999998,"5min":2.4700000000000002,"15min":2.1299999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.188},"max":{"1min":83.421999999999997,"5min":83.421999999999997,"15min":85.091999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":6.0069999999999997,"5min":2.149,"15min":2.0270000000000001},"min":{"1min":0.29599999999999999,"5min":0.23499999999999999,"15min":0.22700000000000001},"max":{"1min":83.373999999999995,"5min":83.373999999999995,"15min":84.373999999999995},"last":0.436}]},{"osd":1,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":1.849,"15min":1.8160000000000001},"min":{"1min":0.38400000000000001,"5min":0.245,"15min":0.214},"max":{"1min":5.8280000000000003,"5min":16.241,"15min":122.367},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.744,"15min":1.73},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.9610000000000003,"5min":16.948,"15min":122.777},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:22:28 2023","interfaces":[{"interface":"back","average":{"1min":10.651,"5min":4.5919999999999996,"15min":2.7429999999999999},"min":{"1min":0.27200000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":152.29300000000001,"5min":152.29300000000001,"15min":152.29300000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":11.081,"5min":5.3250000000000002,"15min":2.9489999999999998},"min":{"1min":0.39100000000000001,"5min":0.311,"15min":0.20100000000000001},"max":{"1min":152.547,"5min":153.70400000000001,"15min":153.70400000000001},"last":0.53200000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.3149999999999999,"15min":1.4930000000000001},"min":{"1min":0.37,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.492,"5min":7.1719999999999997,"15min":14.343999999999999},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.3220000000000001,"15min":1.5700000000000001},"min":{"1min":0.375,"5min":0.35799999999999998,"15min":0.29699999999999999},"max":{"1min":5.6260000000000003,"5min":6.8600000000000003,"15min":12.351000000000001},"last":0.999}]},{"osd":5,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.2509999999999999,"15min":1.5620000000000001},"min":{"1min":0.45600000000000002,"5min":0.35999999999999999,"15min":0.35799999999999998},"max":{"1min":10.135,"5min":10.135,"15min":72.311000000000007},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.5,"15min":1.825},"min":{"1min":0.39800000000000002,"5min":0.39700000000000002,"15min":0.36899999999999999},"max":{"1min":4.5359999999999996,"5min":7.2080000000000002,"15min":73.042000000000002},"last":0.94899999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":1.73,"15min":1.718},"min":{"1min":0.53400000000000003,"5min":0.35899999999999999,"15min":0.34300000000000003},"max":{"1min":6.8570000000000002,"5min":8.5730000000000004,"15min":14.714},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.6950000000000001,"15min":1.7190000000000001},"min":{"1min":0.495,"5min":0.44900000000000001,"15min":0.40999999999999998},"max":{"1min":10.167,"5min":10.167,"15min":14.571999999999999},"last":0.81699999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.7709999999999999,"15min":1.603},"min":{"1min":0.58899999999999997,"5min":0.41599999999999998,"15min":0.40200000000000002},"max":{"1min":5.7560000000000002,"5min":29.513999999999999,"15min":29.513999999999999},"last":0.90800000000000003},{"interface":"front","average":{"1min":1.4430000000000001,"5min":1.718,"15min":1.55},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.40200000000000002},"max":{"1min":10.065,"5min":30.216999999999999,"15min":30.216999999999999},"last":0.79900000000000004}]}]},{"osd":3,"up_from":93,"seq":399431958954,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":959992,"kb_used_data":3760,"kb_used_omap":7,"kb_used_meta":956216,"kb_avail":92785160,"statfs":{"total":95995035648,"available":95012003840,"internally_reserved":0,"allocated":3850240,"data_stored":2607472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":979165800},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":1.5669999999999999,"15min":2.0609999999999999},"min":{"1min":0.29099999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.943,"5min":10.943,"15min":130.45500000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.774,"5min":1.7170000000000001,"15min":2.3380000000000001},"min":{"1min":0.22700000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.8479999999999999,"5min":8.6769999999999996,"15min":129.74799999999999},"last":0.63200000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":1.8400000000000001,"15min":1.389},"min":{"1min":0.40999999999999998,"5min":0.309,"15min":0.23599999999999999},"max":{"1min":8.0259999999999998,"5min":9.7029999999999994,"15min":32.520000000000003},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.5780000000000001,"15min":1.23},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.22900000000000001},"max":{"1min":5.4349999999999996,"5min":9.6310000000000002,"15min":32.585999999999999},"last":0.42299999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":1.8180000000000001,"15min":1.9359999999999999},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.9390000000000001,"5min":37.997,"15min":129.67699999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.9950000000000001,"15min":2.0289999999999999},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":10.896000000000001,"5min":37.499000000000002,"15min":130.40000000000001},"last":0.44900000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":1.329,"15min":1.4159999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":10.445,"5min":10.445,"15min":10.445},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.4339999999999999,"15min":1.7589999999999999},"min":{"1min":0.34499999999999997,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":6.657,"5min":8.4510000000000005,"15min":12.617000000000001},"last":0.71899999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.504,"15min":1.7569999999999999},"min":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.726,"5min":8.5540000000000003,"15min":16.132999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.335,"15min":1.601},"min":{"1min":0.42399999999999999,"5min":0.39800000000000002,"15min":0.36899999999999999},"max":{"1min":7.9880000000000004,"5min":7.9880000000000004,"15min":14.361000000000001},"last":0.83899999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:22:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3260000000000001,"5min":1.835,"15min":1.9970000000000001},"min":{"1min":0.38800000000000001,"5min":0.38500000000000001,"15min":0.37},"max":{"1min":14.124000000000001,"5min":14.124000000000001,"15min":14.124000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":2.2949999999999999,"5min":1.7589999999999999,"15min":1.7649999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":13.778,"5min":13.778,"15min":13.778},"last":0.97299999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.5680000000000001,"15min":1.615},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.41499999999999998},"max":{"1min":8.0950000000000006,"5min":9.6669999999999998,"15min":9.6669999999999998},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.585,"5min":1.468,"15min":1.516},"min":{"1min":0.435,"5min":0.377,"15min":0.377},"max":{"1min":4.1509999999999998,"5min":5.2789999999999999,"15min":8.8930000000000007},"last":0.73399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019521,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905340,"kb_used_data":3060,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":92839812,"statfs":{"total":95995035648,"available":95067967488,"internally_reserved":0,"allocated":3133440,"data_stored":2529219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.6779999999999999,"15min":2.2949999999999999},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":23.32,"5min":23.32,"15min":116.59999999999999},"last":1.0609999999999999},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.5780000000000001,"15min":2.202},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":23.581,"5min":23.581,"15min":116.843},"last":0.73399999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:22:19 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":1.9239999999999999,"15min":1.4730000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":37.481999999999999,"5min":37.481999999999999,"15min":37.481999999999999},"last":0.82599999999999996},{"interface":"front","average":{"1min":3.8340000000000001,"5min":1.986,"15min":1.569},"min":{"1min":0.32800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":37.332999999999998,"5min":37.332999999999998,"15min":37.332999999999998},"last":0.79200000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.3420000000000001,"15min":1.913},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":8.1400000000000006,"5min":8.1400000000000006,"15min":116.7},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.22,"5min":1.218,"15min":1.9410000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":5.1340000000000003,"5min":10.817,"15min":116.63800000000001},"last":0.98299999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":2.1930000000000001,"15min":4.0759999999999996},"min":{"1min":0.374,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":2.1739999999999999,"5min":55.299999999999997,"15min":165.995},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.224,"5min":1.992,"15min":4.234},"min":{"1min":0.45400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.1600000000000001,"5min":55.622,"15min":165.53200000000001},"last":1.0980000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:21:43 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.3500000000000001,"15min":1.5820000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.30099999999999999},"max":{"1min":2.649,"5min":9.3989999999999991,"15min":13.907},"last":1.163},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.0940000000000001,"15min":1.3740000000000001},"min":{"1min":0.34699999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.887,"5min":10.794,"15min":11.042999999999999},"last":1.2549999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.167,"15min":1.843},"min":{"1min":0.39400000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.7719999999999998,"5min":12.074,"15min":74.700999999999993},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.2110000000000001,"15min":1.9359999999999999},"min":{"1min":0.39200000000000002,"5min":0.34000000000000002,"15min":0.33000000000000002},"max":{"1min":8.0640000000000001,"5min":8.0640000000000001,"15min":73.572999999999993},"last":1.2290000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.3240000000000001,"15min":1.6379999999999999},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.9180000000000001,"5min":15.484999999999999,"15min":15.484999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.46,"5min":1.3160000000000001,"15min":1.5940000000000001},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":12.003,"5min":12.003,"15min":13.916},"last":1.389}]}]},{"osd":5,"up_from":36,"seq":154618823292,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908464,"kb_used_data":3048,"kb_used_omap":0,"kb_used_meta":905408,"kb_avail":92836688,"statfs":{"total":95995035648,"available":95064768512,"internally_reserved":0,"allocated":3121152,"data_stored":2532765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927137792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.6150000000000002,"15min":2.923},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.215,"5min":139.81700000000001,"15min":139.81700000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.468,"5min":3.0270000000000001,"15min":2.9969999999999999},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":24.576000000000001,"5min":141.66900000000001,"15min":141.66900000000001},"last":0.44800000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:22:35 2023","interfaces":[{"interface":"back","average":{"1min":4.7270000000000003,"5min":2.1459999999999999,"15min":1.7689999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":36.609999999999999,"5min":36.609999999999999,"15min":36.609999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":4.3739999999999997,"5min":2.2789999999999999,"15min":1.6990000000000001},"min":{"1min":0.377,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":36.856999999999999,"5min":36.856999999999999,"15min":36.856999999999999},"last":0.71399999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:22:00 2023","interfaces":[{"interface":"back","average":{"1min":3.1379999999999999,"5min":1.345,"15min":2.1160000000000001},"min":{"1min":0.33500000000000002,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":38.191000000000003,"5min":38.191000000000003,"15min":128.77199999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.887,"5min":1.597,"15min":2.032},"min":{"1min":0.33000000000000002,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":36.281999999999996,"5min":36.281999999999996,"15min":128.72999999999999},"last":0.84599999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:22:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.403,"15min":1.8100000000000001},"min":{"1min":0.39300000000000002,"5min":0.34499999999999997,"15min":0.32000000000000001},"max":{"1min":5.8799999999999999,"5min":8.8219999999999992,"15min":57.963000000000001},"last":0.79600000000000004},{"interface":"front","average":{"1min":1.258,"5min":1.3049999999999999,"15min":1.925},"min":{"1min":0.372,"5min":0.33400000000000002,"15min":0.31},"max":{"1min":5.4509999999999996,"5min":10.630000000000001,"15min":57.603000000000002},"last":0.81599999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.752,"15min":1.732},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26600000000000001},"max":{"1min":1.889,"5min":8.0540000000000003,"15min":13.904},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.0629999999999999,"5min":0.95699999999999996,"15min":1.623},"min":{"1min":0.39600000000000002,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":2.665,"5min":4.5599999999999996,"15min":22.228000000000002},"last":0.89000000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.556,"15min":1.8149999999999999},"min":{"1min":0.44800000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.1099999999999999,"5min":53.840000000000003,"15min":53.840000000000003},"last":0.62},{"interface":"front","average":{"1min":1.214,"5min":1.6890000000000001,"15min":1.885},"min":{"1min":0.40799999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.7389999999999999,"5min":50.404000000000003,"15min":50.404000000000003},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:22:06 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.0509999999999999,"15min":1.522},"min":{"1min":0.48999999999999999,"5min":0.39200000000000002,"15min":0.34799999999999998},"max":{"1min":3.6720000000000002,"5min":4.649,"15min":12.159000000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.745,"5min":0.98699999999999999,"15min":1.569},"min":{"1min":0.47999999999999998,"5min":0.38600000000000001,"15min":0.33400000000000002},"max":{"1min":1.395,"5min":10.598000000000001,"15min":11.090999999999999},"last":0.76900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:22:39.307 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:22:39.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 36 2023-10-09T09:22:39.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.308 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:39.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:39.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:39.435 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:39.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:39.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:39.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:39.665 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:39.702 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:39.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:39.806 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:39.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:39.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:39.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:39.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:39.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:39.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:40.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:40.106 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:40.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:40.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:40.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:40.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.003877+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: audit 2023-10-09T09:22:39.005371+0000 mon.a (mon.0) 1876 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T09:22:40.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.005479+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2023-10-09T09:22:40.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.005487+0000 osd.3 (osd.3) 14 : cluster 0 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11157 2023-10-09T09:22:40.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.007926+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.010178+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.010201+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.020417+0000 osd.3 (osd.3) 15 : cluster 0 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 345'11157 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.875755+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.003877+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: audit 2023-10-09T09:22:39.005371+0000 mon.a (mon.0) 1876 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.005479+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.005487+0000 osd.3 (osd.3) 14 : cluster 0 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11157 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.007926+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.010178+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.010201+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.020417+0000 osd.3 (osd.3) 15 : cluster 0 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 345'11157 2023-10-09T09:22:40.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:40 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.875755+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2023-10-09T09:22:40.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:40.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:40.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:40.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:40.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:40.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:40.474 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:22:40.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.003877+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: audit 2023-10-09T09:22:39.005371+0000 mon.a (mon.0) 1876 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.005479+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.005487+0000 osd.3 (osd.3) 14 : cluster 0 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11157 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.007926+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.010178+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.010201+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.020417+0000 osd.3 (osd.3) 15 : cluster 0 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 345'11157 2023-10-09T09:22:40.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:40 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.875755+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2023-10-09T09:22:40.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:40.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:40.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:40.501 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:22:40.530 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:22:40.554 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:22:40.579 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:22:40.602 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:40.606 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:22:40.633 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:40.634 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:22:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:40.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:40.663 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.665 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:22:40.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:40.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:40.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:40.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:40.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:40.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:40.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:40.994 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:41.026 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:41.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:41.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:41.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.874740+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.876818+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.876980+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.877874+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.881007+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:39.881713+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:40.287865+0000 mgr.y (mgr.25421) 1366 : cluster 0 pgmap v1539: 99 pgs: 1 active+clean+remapped, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:41.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: audit 2023-10-09T09:22:40.446833+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.16:0/3020513685' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:22:41.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:40.878050+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2023-10-09T09:22:41.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: cluster 2023-10-09T09:22:40.879655+0000 osd.0 (osd.0) 37 : cluster 0 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11314 2023-10-09T09:22:41.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[75179]: audit 2023-10-09T09:22:40.961149+0000 mon.a (mon.0) 1880 : audit 1 from='client.? 172.21.15.16:0/2508174058' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.874740+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.876818+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.876980+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.877874+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.881007+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:39.881713+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:40.287865+0000 mgr.y (mgr.25421) 1366 : cluster 0 pgmap v1539: 99 pgs: 1 active+clean+remapped, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: audit 2023-10-09T09:22:40.446833+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.16:0/3020513685' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:40.878050+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: cluster 2023-10-09T09:22:40.879655+0000 osd.0 (osd.0) 37 : cluster 0 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11314 2023-10-09T09:22:41.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:41 smithi016 bash[77892]: audit 2023-10-09T09:22:40.961149+0000 mon.a (mon.0) 1880 : audit 1 from='client.? 172.21.15.16:0/2508174058' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:22:41.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:41.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:41.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:41.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:41.396 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.874740+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.876818+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.876980+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.877874+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.881007+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:39.881713+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:40.287865+0000 mgr.y (mgr.25421) 1366 : cluster 0 pgmap v1539: 99 pgs: 1 active+clean+remapped, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:41.410 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: audit 2023-10-09T09:22:40.446833+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.16:0/3020513685' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:22:41.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:40.878050+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2023-10-09T09:22:41.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: cluster 2023-10-09T09:22:40.879655+0000 osd.0 (osd.0) 37 : cluster 0 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11314 2023-10-09T09:22:41.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:41 smithi094 bash[59677]: audit 2023-10-09T09:22:40.961149+0000 mon.a (mon.0) 1880 : audit 1 from='client.? 172.21.15.16:0/2508174058' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:22:41.429 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:41.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:41.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:41.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:41.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:41.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:41.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:41.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:41.893 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:22:41.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:41.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:41.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:41.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:42.068 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:42.100 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:42.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: cluster 2023-10-09T09:22:40.878082+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: cluster 2023-10-09T09:22:40.879537+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: cluster 2023-10-09T09:22:40.895340+0000 osd.0 (osd.0) 38 : cluster 0 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11314 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: cluster 2023-10-09T09:22:40.953538+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: cluster 2023-10-09T09:22:41.086911+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: cluster 2023-10-09T09:22:41.874702+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: audit 2023-10-09T09:22:41.884503+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.16:0/2508174058' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:22:42.120 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[75179]: cluster 2023-10-09T09:22:41.884650+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: cluster 2023-10-09T09:22:40.878082+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: cluster 2023-10-09T09:22:40.879537+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: cluster 2023-10-09T09:22:40.895340+0000 osd.0 (osd.0) 38 : cluster 0 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11314 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: cluster 2023-10-09T09:22:40.953538+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: cluster 2023-10-09T09:22:41.086911+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: cluster 2023-10-09T09:22:41.874702+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: audit 2023-10-09T09:22:41.884503+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.16:0/2508174058' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:22:42.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:42 smithi016 bash[77892]: cluster 2023-10-09T09:22:41.884650+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2023-10-09T09:22:42.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:42.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: cluster 2023-10-09T09:22:40.878082+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: cluster 2023-10-09T09:22:40.879537+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: cluster 2023-10-09T09:22:40.895340+0000 osd.0 (osd.0) 38 : cluster 0 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11314 2023-10-09T09:22:42.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: cluster 2023-10-09T09:22:40.953538+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: cluster 2023-10-09T09:22:41.086911+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T09:22:42.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: cluster 2023-10-09T09:22:41.874702+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:22:42.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: audit 2023-10-09T09:22:41.884503+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.16:0/2508174058' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:22:42.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:42 smithi094 bash[59677]: cluster 2023-10-09T09:22:41.884650+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2023-10-09T09:22:42.233 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:42.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:42.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:42.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:42.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:42.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:42.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:42.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:42.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:42.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:42.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:42.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:42.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:42.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:43.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:43.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:43.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:43.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:43.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:43.179 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:43.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:43.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:43.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:43.295 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[77892]: cluster 2023-10-09T09:22:41.879029+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.295 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[77892]: cluster 2023-10-09T09:22:41.879531+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.295 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[77892]: cluster 2023-10-09T09:22:41.889528+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.295 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[77892]: cluster 2023-10-09T09:22:42.288359+0000 mgr.y (mgr.25421) 1367 : cluster 0 pgmap v1542: 99 pgs: 99 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[77892]: cluster 2023-10-09T09:22:42.293935+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[77892]: cluster 2023-10-09T09:22:42.884269+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[75179]: cluster 2023-10-09T09:22:41.879029+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[75179]: cluster 2023-10-09T09:22:41.879531+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[75179]: cluster 2023-10-09T09:22:41.889528+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[75179]: cluster 2023-10-09T09:22:42.288359+0000 mgr.y (mgr.25421) 1367 : cluster 0 pgmap v1542: 99 pgs: 99 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[75179]: cluster 2023-10-09T09:22:42.293935+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:43 smithi016 bash[75179]: cluster 2023-10-09T09:22:42.884269+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-09T09:22:43.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:43.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:43.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:43.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:43.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:43.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:43 smithi094 bash[59677]: cluster 2023-10-09T09:22:41.879029+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:43 smithi094 bash[59677]: cluster 2023-10-09T09:22:41.879531+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:43 smithi094 bash[59677]: cluster 2023-10-09T09:22:41.889528+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:43 smithi094 bash[59677]: cluster 2023-10-09T09:22:42.288359+0000 mgr.y (mgr.25421) 1367 : cluster 0 pgmap v1542: 99 pgs: 99 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:43 smithi094 bash[59677]: cluster 2023-10-09T09:22:42.293935+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T09:22:43.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:43 smithi094 bash[59677]: cluster 2023-10-09T09:22:42.884269+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-09T09:22:43.555 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:43.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:43.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:43.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:43.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:43.722 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:43.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:43.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:43.826 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:43.856 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:43.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:43.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:43.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:43.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:44.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:44.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:44.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:22:44.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:44.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:44.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:44 smithi094 bash[59677]: cluster 2023-10-09T09:22:42.881444+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:44 smithi094 bash[59677]: cluster 2023-10-09T09:22:42.881976+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:44 smithi094 bash[59677]: cluster 2023-10-09T09:22:42.968308+0000 osd.5 (osd.5) 261 : cluster 0 11.16 scrub starts 2023-10-09T09:22:44.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:44 smithi094 bash[59677]: cluster 2023-10-09T09:22:43.392904+0000 osd.1 (osd.1) 13 : cluster 0 11.7 scrub starts 2023-10-09T09:22:44.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:44 smithi094 bash[59677]: cluster 2023-10-09T09:22:43.405522+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:44 smithi094 bash[59677]: cluster 2023-10-09T09:22:43.409746+0000 osd.1 (osd.1) 14 : cluster 0 11.7 scrub ok 2023-10-09T09:22:44.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[75179]: cluster 2023-10-09T09:22:42.881444+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[75179]: cluster 2023-10-09T09:22:42.881976+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[75179]: cluster 2023-10-09T09:22:42.968308+0000 osd.5 (osd.5) 261 : cluster 0 11.16 scrub starts 2023-10-09T09:22:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[75179]: cluster 2023-10-09T09:22:43.392904+0000 osd.1 (osd.1) 13 : cluster 0 11.7 scrub starts 2023-10-09T09:22:44.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[75179]: cluster 2023-10-09T09:22:43.405522+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[75179]: cluster 2023-10-09T09:22:43.409746+0000 osd.1 (osd.1) 14 : cluster 0 11.7 scrub ok 2023-10-09T09:22:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[77892]: cluster 2023-10-09T09:22:42.881444+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[77892]: cluster 2023-10-09T09:22:42.881976+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[77892]: cluster 2023-10-09T09:22:42.968308+0000 osd.5 (osd.5) 261 : cluster 0 11.16 scrub starts 2023-10-09T09:22:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[77892]: cluster 2023-10-09T09:22:43.392904+0000 osd.1 (osd.1) 13 : cluster 0 11.7 scrub starts 2023-10-09T09:22:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[77892]: cluster 2023-10-09T09:22:43.405522+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T09:22:44.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:44 smithi016 bash[77892]: cluster 2023-10-09T09:22:43.409746+0000 osd.1 (osd.1) 14 : cluster 0 11.7 scrub ok 2023-10-09T09:22:44.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:44.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:44.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:44.419 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:44.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:44.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:44.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:44.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:44.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:44.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:44.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:44.930 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:44.964 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:44.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:44.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:45.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:22:45.067 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:45.097 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:45.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:45.201 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:45.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:45 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.279581+0000 osd.4 (osd.4) 325 : cluster 0 11.13 scrub starts 2023-10-09T09:22:45.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:45 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.288871+0000 mgr.y (mgr.25421) 1368 : cluster 0 pgmap v1544: 99 pgs: 1 peering, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:45.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:45 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.291320+0000 osd.4 (osd.4) 326 : cluster 0 11.13 scrub ok 2023-10-09T09:22:45.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:45 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.861814+0000 osd.0 (osd.0) 39 : cluster 0 11.2 scrub starts 2023-10-09T09:22:45.231 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:45.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:45.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.279581+0000 osd.4 (osd.4) 325 : cluster 0 11.13 scrub starts 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.288871+0000 mgr.y (mgr.25421) 1368 : cluster 0 pgmap v1544: 99 pgs: 1 peering, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.291320+0000 osd.4 (osd.4) 326 : cluster 0 11.13 scrub ok 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.861814+0000 osd.0 (osd.0) 39 : cluster 0 11.2 scrub starts 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.279581+0000 osd.4 (osd.4) 325 : cluster 0 11.13 scrub starts 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.288871+0000 mgr.y (mgr.25421) 1368 : cluster 0 pgmap v1544: 99 pgs: 1 peering, 98 active+clean; 7.4 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.291320+0000 osd.4 (osd.4) 326 : cluster 0 11.13 scrub ok 2023-10-09T09:22:45.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.861814+0000 osd.0 (osd.0) 39 : cluster 0 11.2 scrub starts 2023-10-09T09:22:45.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:45.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:45.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:45.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:45.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:45.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:45.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:45.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:45.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:45.634 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:45.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:45.648 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:45.649 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:45.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:45.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:45.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:45.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:45.874 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:45.905 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:45.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:46.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:22:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:22:45] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T09:22:46.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:46.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:46.154 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.736375+0000 osd.6 (osd.6) 303 : cluster 0 11.3 scrub starts 2023-10-09T09:22:46.154 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.753023+0000 osd.6 (osd.6) 304 : cluster 0 11.3 scrub ok 2023-10-09T09:22:46.154 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.888765+0000 osd.0 (osd.0) 40 : cluster 0 11.2 scrub ok 2023-10-09T09:22:46.154 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.941778+0000 osd.5 (osd.5) 262 : cluster 0 11.16 scrub starts 2023-10-09T09:22:46.154 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:44.951163+0000 osd.5 (osd.5) 263 : cluster 0 11.16 scrub ok 2023-10-09T09:22:46.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:45.196418+0000 osd.2 (osd.2) 16 : cluster 0 11.11 scrub starts 2023-10-09T09:22:46.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:45.210196+0000 osd.2 (osd.2) 17 : cluster 0 11.11 scrub ok 2023-10-09T09:22:46.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:45.401339+0000 osd.1 (osd.1) 15 : cluster 0 11.0 scrub starts 2023-10-09T09:22:46.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:46 smithi094 bash[59677]: cluster 2023-10-09T09:22:45.421806+0000 osd.1 (osd.1) 16 : cluster 0 11.0 scrub ok 2023-10-09T09:22:46.172 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:46.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.736375+0000 osd.6 (osd.6) 303 : cluster 0 11.3 scrub starts 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.753023+0000 osd.6 (osd.6) 304 : cluster 0 11.3 scrub ok 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.888765+0000 osd.0 (osd.0) 40 : cluster 0 11.2 scrub ok 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.941778+0000 osd.5 (osd.5) 262 : cluster 0 11.16 scrub starts 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:44.951163+0000 osd.5 (osd.5) 263 : cluster 0 11.16 scrub ok 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:45.196418+0000 osd.2 (osd.2) 16 : cluster 0 11.11 scrub starts 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:45.210196+0000 osd.2 (osd.2) 17 : cluster 0 11.11 scrub ok 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:45.401339+0000 osd.1 (osd.1) 15 : cluster 0 11.0 scrub starts 2023-10-09T09:22:46.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[75179]: cluster 2023-10-09T09:22:45.421806+0000 osd.1 (osd.1) 16 : cluster 0 11.0 scrub ok 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.736375+0000 osd.6 (osd.6) 303 : cluster 0 11.3 scrub starts 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.753023+0000 osd.6 (osd.6) 304 : cluster 0 11.3 scrub ok 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.888765+0000 osd.0 (osd.0) 40 : cluster 0 11.2 scrub ok 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.941778+0000 osd.5 (osd.5) 262 : cluster 0 11.16 scrub starts 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:44.951163+0000 osd.5 (osd.5) 263 : cluster 0 11.16 scrub ok 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:45.196418+0000 osd.2 (osd.2) 16 : cluster 0 11.11 scrub starts 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:45.210196+0000 osd.2 (osd.2) 17 : cluster 0 11.11 scrub ok 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:45.401339+0000 osd.1 (osd.1) 15 : cluster 0 11.0 scrub starts 2023-10-09T09:22:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:46 smithi016 bash[77892]: cluster 2023-10-09T09:22:45.421806+0000 osd.1 (osd.1) 16 : cluster 0 11.0 scrub ok 2023-10-09T09:22:46.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:46.406 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:46.440 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:46.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:46.681 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:46.714 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:46.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:46.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:46.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:46.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:46.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:47.076 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:47.105 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:47.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:47.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:47.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:47.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[77892]: cluster 2023-10-09T09:22:45.521578+0000 osd.7 (osd.7) 254 : cluster 0 11.4 scrub starts 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[77892]: cluster 2023-10-09T09:22:45.541592+0000 osd.7 (osd.7) 255 : cluster 0 11.4 scrub ok 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[77892]: cluster 2023-10-09T09:22:46.150069+0000 osd.2 (osd.2) 18 : cluster 0 11.18 scrub starts 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[77892]: cluster 2023-10-09T09:22:46.172306+0000 osd.2 (osd.2) 19 : cluster 0 11.18 scrub ok 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[77892]: cluster 2023-10-09T09:22:46.289501+0000 mgr.y (mgr.25421) 1369 : cluster 0 pgmap v1545: 99 pgs: 1 peering, 98 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[75179]: cluster 2023-10-09T09:22:45.521578+0000 osd.7 (osd.7) 254 : cluster 0 11.4 scrub starts 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[75179]: cluster 2023-10-09T09:22:45.541592+0000 osd.7 (osd.7) 255 : cluster 0 11.4 scrub ok 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[75179]: cluster 2023-10-09T09:22:46.150069+0000 osd.2 (osd.2) 18 : cluster 0 11.18 scrub starts 2023-10-09T09:22:47.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[75179]: cluster 2023-10-09T09:22:46.172306+0000 osd.2 (osd.2) 19 : cluster 0 11.18 scrub ok 2023-10-09T09:22:47.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:47 smithi016 bash[75179]: cluster 2023-10-09T09:22:46.289501+0000 mgr.y (mgr.25421) 1369 : cluster 0 pgmap v1545: 99 pgs: 1 peering, 98 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:47.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:47.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:47.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:47.472 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:47.483 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:47 smithi094 bash[59677]: cluster 2023-10-09T09:22:45.521578+0000 osd.7 (osd.7) 254 : cluster 0 11.4 scrub starts 2023-10-09T09:22:47.484 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:47 smithi094 bash[59677]: cluster 2023-10-09T09:22:45.541592+0000 osd.7 (osd.7) 255 : cluster 0 11.4 scrub ok 2023-10-09T09:22:47.484 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:47 smithi094 bash[59677]: cluster 2023-10-09T09:22:46.150069+0000 osd.2 (osd.2) 18 : cluster 0 11.18 scrub starts 2023-10-09T09:22:47.484 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:47 smithi094 bash[59677]: cluster 2023-10-09T09:22:46.172306+0000 osd.2 (osd.2) 19 : cluster 0 11.18 scrub ok 2023-10-09T09:22:47.484 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:47 smithi094 bash[59677]: cluster 2023-10-09T09:22:46.289501+0000 mgr.y (mgr.25421) 1369 : cluster 0 pgmap v1545: 99 pgs: 1 peering, 98 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:47.498 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:47.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:47.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:47.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:47.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:47.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:47.733 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:47.766 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:47.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:47.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:47.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:47.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:47.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:48.003 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:48.037 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:48.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:48.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:48 smithi016 bash[75179]: cluster 2023-10-09T09:22:46.699962+0000 osd.6 (osd.6) 305 : cluster 0 11.8 scrub starts 2023-10-09T09:22:48.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:48 smithi016 bash[75179]: cluster 2023-10-09T09:22:46.717759+0000 osd.6 (osd.6) 306 : cluster 0 11.8 scrub ok 2023-10-09T09:22:48.154 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:48 smithi016 bash[77892]: cluster 2023-10-09T09:22:46.699962+0000 osd.6 (osd.6) 305 : cluster 0 11.8 scrub starts 2023-10-09T09:22:48.154 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:48 smithi016 bash[77892]: cluster 2023-10-09T09:22:46.717759+0000 osd.6 (osd.6) 306 : cluster 0 11.8 scrub ok 2023-10-09T09:22:48.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:48.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:48.278 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:48.291 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:48 smithi094 bash[59677]: cluster 2023-10-09T09:22:46.699962+0000 osd.6 (osd.6) 305 : cluster 0 11.8 scrub starts 2023-10-09T09:22:48.291 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:48 smithi094 bash[59677]: cluster 2023-10-09T09:22:46.717759+0000 osd.6 (osd.6) 306 : cluster 0 11.8 scrub ok 2023-10-09T09:22:48.310 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:48.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:48.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:48.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:48.550 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:48.585 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:48.689 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:48.690 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:48.720 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:48.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:48.824 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:48.854 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:48.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:48.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:48.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:49.092 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:49.105 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:49 smithi094 bash[59677]: cluster 2023-10-09T09:22:48.290453+0000 mgr.y (mgr.25421) 1370 : cluster 0 pgmap v1546: 99 pgs: 99 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:49.106 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:49 smithi094 bash[59677]: cluster 2023-10-09T09:22:48.880789+0000 osd.0 (osd.0) 41 : cluster 0 11.6 scrub starts 2023-10-09T09:22:49.122 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:49.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:49.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:49.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:49 smithi016 bash[75179]: cluster 2023-10-09T09:22:48.290453+0000 mgr.y (mgr.25421) 1370 : cluster 0 pgmap v1546: 99 pgs: 99 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:49 smithi016 bash[75179]: cluster 2023-10-09T09:22:48.880789+0000 osd.0 (osd.0) 41 : cluster 0 11.6 scrub starts 2023-10-09T09:22:49.234 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:49 smithi016 bash[77892]: cluster 2023-10-09T09:22:48.290453+0000 mgr.y (mgr.25421) 1370 : cluster 0 pgmap v1546: 99 pgs: 99 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:49.234 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:49 smithi016 bash[77892]: cluster 2023-10-09T09:22:48.880789+0000 osd.0 (osd.0) 41 : cluster 0 11.6 scrub starts 2023-10-09T09:22:49.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:49.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:49.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:49.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:49.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:49.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:49.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:49.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:49.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:49.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:49.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:49.703 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:49.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:49.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:49.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:49.885 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:49.914 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:49.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:50.018 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:50.047 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:50.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:50.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:50.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:50.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:50 smithi094 bash[59677]: cluster 2023-10-09T09:22:48.904024+0000 osd.0 (osd.0) 42 : cluster 0 11.6 scrub ok 2023-10-09T09:22:50.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:50 smithi094 bash[59677]: audit 2023-10-09T09:22:49.263879+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:50.275 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:50.305 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:50.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:50.409 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:50.439 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:50.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:50.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:50 smithi016 bash[75179]: cluster 2023-10-09T09:22:48.904024+0000 osd.0 (osd.0) 42 : cluster 0 11.6 scrub ok 2023-10-09T09:22:50.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:50 smithi016 bash[75179]: audit 2023-10-09T09:22:49.263879+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:50.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:50 smithi016 bash[77892]: cluster 2023-10-09T09:22:48.904024+0000 osd.0 (osd.0) 42 : cluster 0 11.6 scrub ok 2023-10-09T09:22:50.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:50 smithi016 bash[77892]: audit 2023-10-09T09:22:49.263879+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:50.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:50.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:50.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:50.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:50.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:50.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:50.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:50.947 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:50.978 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:50.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:51.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:51.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:51.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:51.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:51.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:51.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:51.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:51.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:51.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:51 smithi094 bash[59677]: cluster 2023-10-09T09:22:49.602466+0000 osd.6 (osd.6) 307 : cluster 0 11.1 scrub starts 2023-10-09T09:22:51.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:51 smithi094 bash[59677]: cluster 2023-10-09T09:22:49.623814+0000 osd.6 (osd.6) 308 : cluster 0 11.1 scrub ok 2023-10-09T09:22:51.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:51 smithi094 bash[59677]: cluster 2023-10-09T09:22:50.291097+0000 mgr.y (mgr.25421) 1371 : cluster 0 pgmap v1547: 99 pgs: 99 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:51.337 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:51.369 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:51.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:51.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:51.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:51 smithi016 bash[75179]: cluster 2023-10-09T09:22:49.602466+0000 osd.6 (osd.6) 307 : cluster 0 11.1 scrub starts 2023-10-09T09:22:51.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:51 smithi016 bash[75179]: cluster 2023-10-09T09:22:49.623814+0000 osd.6 (osd.6) 308 : cluster 0 11.1 scrub ok 2023-10-09T09:22:51.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:51 smithi016 bash[75179]: cluster 2023-10-09T09:22:50.291097+0000 mgr.y (mgr.25421) 1371 : cluster 0 pgmap v1547: 99 pgs: 99 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:51.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:51 smithi016 bash[77892]: cluster 2023-10-09T09:22:49.602466+0000 osd.6 (osd.6) 307 : cluster 0 11.1 scrub starts 2023-10-09T09:22:51.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:51 smithi016 bash[77892]: cluster 2023-10-09T09:22:49.623814+0000 osd.6 (osd.6) 308 : cluster 0 11.1 scrub ok 2023-10-09T09:22:51.455 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:51 smithi016 bash[77892]: cluster 2023-10-09T09:22:50.291097+0000 mgr.y (mgr.25421) 1371 : cluster 0 pgmap v1547: 99 pgs: 99 active+clean; 7.0 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:51.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:51.505 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:51.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:51.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:51.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:51.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:51.739 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:51.769 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:51.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:51.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:51.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:51.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:52.006 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:52.038 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:52.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:52.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:52.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:52.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:52.285 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:52 smithi094 bash[59677]: cluster 2023-10-09T09:22:51.647098+0000 osd.7 (osd.7) 256 : cluster 0 11.12 scrub starts 2023-10-09T09:22:52.285 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:52 smithi094 bash[59677]: cluster 2023-10-09T09:22:51.663960+0000 osd.7 (osd.7) 257 : cluster 0 11.12 scrub ok 2023-10-09T09:22:52.304 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:52.407 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:52.437 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:52 smithi016 bash[75179]: cluster 2023-10-09T09:22:51.647098+0000 osd.7 (osd.7) 256 : cluster 0 11.12 scrub starts 2023-10-09T09:22:52.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:52 smithi016 bash[75179]: cluster 2023-10-09T09:22:51.663960+0000 osd.7 (osd.7) 257 : cluster 0 11.12 scrub ok 2023-10-09T09:22:52.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:52 smithi016 bash[77892]: cluster 2023-10-09T09:22:51.647098+0000 osd.7 (osd.7) 256 : cluster 0 11.12 scrub starts 2023-10-09T09:22:52.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:52 smithi016 bash[77892]: cluster 2023-10-09T09:22:51.663960+0000 osd.7 (osd.7) 257 : cluster 0 11.12 scrub ok 2023-10-09T09:22:52.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:52.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:52.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:52.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:52.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:52.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:52.839 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:52.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:52.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:52.978 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:52.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:52.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:53.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:53.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:53.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:53.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:53.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:53.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:53.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:53.338 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:53.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:53.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:53.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:53.471 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:53.485 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:53 smithi094 bash[59677]: cluster 2023-10-09T09:22:51.612341+0000 osd.6 (osd.6) 309 : cluster 0 11.5 scrub starts 2023-10-09T09:22:53.485 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:53 smithi094 bash[59677]: cluster 2023-10-09T09:22:51.630453+0000 osd.6 (osd.6) 310 : cluster 0 11.5 scrub ok 2023-10-09T09:22:53.485 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:53 smithi094 bash[59677]: cluster 2023-10-09T09:22:52.291825+0000 mgr.y (mgr.25421) 1372 : cluster 0 pgmap v1548: 99 pgs: 99 active+clean; 4.3 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:53.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:53 smithi016 bash[75179]: cluster 2023-10-09T09:22:51.612341+0000 osd.6 (osd.6) 309 : cluster 0 11.5 scrub starts 2023-10-09T09:22:53.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:53 smithi016 bash[75179]: cluster 2023-10-09T09:22:51.630453+0000 osd.6 (osd.6) 310 : cluster 0 11.5 scrub ok 2023-10-09T09:22:53.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:53 smithi016 bash[75179]: cluster 2023-10-09T09:22:52.291825+0000 mgr.y (mgr.25421) 1372 : cluster 0 pgmap v1548: 99 pgs: 99 active+clean; 4.3 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:53.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:53 smithi016 bash[77892]: cluster 2023-10-09T09:22:51.612341+0000 osd.6 (osd.6) 309 : cluster 0 11.5 scrub starts 2023-10-09T09:22:53.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:53 smithi016 bash[77892]: cluster 2023-10-09T09:22:51.630453+0000 osd.6 (osd.6) 310 : cluster 0 11.5 scrub ok 2023-10-09T09:22:53.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:53 smithi016 bash[77892]: cluster 2023-10-09T09:22:52.291825+0000 mgr.y (mgr.25421) 1372 : cluster 0 pgmap v1548: 99 pgs: 99 active+clean; 4.3 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:53.506 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:53.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:53.610 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:53.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:53.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:53.747 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:53.779 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:53.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:53.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:53.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:53.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:54.015 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:54.046 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:54.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:54.150 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:54.181 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:54.286 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:54.316 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:54.420 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:54.450 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:54.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:54.553 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:54.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:54.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:54.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:54.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:54.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:54.940 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:54.970 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:54.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:55.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:55.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:55.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:55.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:55.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:55.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:55.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:55.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:55.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:55.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:55.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:55.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:55.469 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:55.481 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:55 smithi094 bash[59677]: cluster 2023-10-09T09:22:54.292401+0000 mgr.y (mgr.25421) 1373 : cluster 0 pgmap v1549: 99 pgs: 99 active+clean; 3.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:55.501 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:55.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:55.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:55.600 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:55 smithi016 bash[75179]: cluster 2023-10-09T09:22:54.292401+0000 mgr.y (mgr.25421) 1373 : cluster 0 pgmap v1549: 99 pgs: 99 active+clean; 3.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:55.600 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:55 smithi016 bash[77892]: cluster 2023-10-09T09:22:54.292401+0000 mgr.y (mgr.25421) 1373 : cluster 0 pgmap v1549: 99 pgs: 99 active+clean; 3.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:55.606 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:55.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:55.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:55.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:55.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:55.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:55.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:55.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:55.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:55.879 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:22:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:22:55] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.33.4" 2023-10-09T09:22:55.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:55.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:55.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:22:55.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:56.031 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:56.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:56.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:56.167 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:56.271 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:56.308 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:56.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:56.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:56.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:56.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:56.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:56.669 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:56.681 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:56 smithi094 bash[59677]: audit 2023-10-09T09:22:55.473759+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:56.681 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:56 smithi094 bash[59677]: cluster 2023-10-09T09:22:55.856681+0000 osd.0 (osd.0) 43 : cluster 0 11.10 scrub starts 2023-10-09T09:22:56.681 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:56 smithi094 bash[59677]: cluster 2023-10-09T09:22:55.876101+0000 osd.0 (osd.0) 44 : cluster 0 11.10 scrub ok 2023-10-09T09:22:56.703 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:56.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:56.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:56.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:56.839 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:56 smithi016 bash[77892]: audit 2023-10-09T09:22:55.473759+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:56.839 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:56 smithi016 bash[77892]: cluster 2023-10-09T09:22:55.856681+0000 osd.0 (osd.0) 43 : cluster 0 11.10 scrub starts 2023-10-09T09:22:56.839 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:56 smithi016 bash[77892]: cluster 2023-10-09T09:22:55.876101+0000 osd.0 (osd.0) 44 : cluster 0 11.10 scrub ok 2023-10-09T09:22:56.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:56 smithi016 bash[75179]: audit 2023-10-09T09:22:55.473759+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:22:56.840 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:56 smithi016 bash[75179]: cluster 2023-10-09T09:22:55.856681+0000 osd.0 (osd.0) 43 : cluster 0 11.10 scrub starts 2023-10-09T09:22:56.840 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:56 smithi016 bash[75179]: cluster 2023-10-09T09:22:55.876101+0000 osd.0 (osd.0) 44 : cluster 0 11.10 scrub ok 2023-10-09T09:22:56.934 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:56.966 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:56.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:57.070 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:57.098 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:57.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:57.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:57.202 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:57.235 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:57.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:57.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:57.339 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:57.368 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:57.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:57.472 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:57.504 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:57.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:57.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:57.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:57.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:57.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:57.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:57.740 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:57 smithi094 bash[59677]: cluster 2023-10-09T09:22:56.293283+0000 mgr.y (mgr.25421) 1374 : cluster 0 pgmap v1550: 99 pgs: 99 active+clean; 1.6 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:57 smithi094 bash[59677]: cluster 2023-10-09T09:22:56.300650+0000 osd.1 (osd.1) 17 : cluster 0 11.15 scrub starts 2023-10-09T09:22:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:57 smithi094 bash[59677]: cluster 2023-10-09T09:22:56.314206+0000 osd.1 (osd.1) 18 : cluster 0 11.15 scrub ok 2023-10-09T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:57 smithi016 bash[75179]: cluster 2023-10-09T09:22:56.293283+0000 mgr.y (mgr.25421) 1374 : cluster 0 pgmap v1550: 99 pgs: 99 active+clean; 1.6 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:57 smithi016 bash[75179]: cluster 2023-10-09T09:22:56.300650+0000 osd.1 (osd.1) 17 : cluster 0 11.15 scrub starts 2023-10-09T09:22:57.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:57 smithi016 bash[75179]: cluster 2023-10-09T09:22:56.314206+0000 osd.1 (osd.1) 18 : cluster 0 11.15 scrub ok 2023-10-09T09:22:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:57 smithi016 bash[77892]: cluster 2023-10-09T09:22:56.293283+0000 mgr.y (mgr.25421) 1374 : cluster 0 pgmap v1550: 99 pgs: 99 active+clean; 1.6 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:57 smithi016 bash[77892]: cluster 2023-10-09T09:22:56.300650+0000 osd.1 (osd.1) 17 : cluster 0 11.15 scrub starts 2023-10-09T09:22:57.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:57 smithi016 bash[77892]: cluster 2023-10-09T09:22:56.314206+0000 osd.1 (osd.1) 18 : cluster 0 11.15 scrub ok 2023-10-09T09:22:57.772 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:57.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:57.875 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:57.907 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:57.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:58.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:58.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:58.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:58.149 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:58.182 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:58.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:58.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:58.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:58.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:58.417 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:58.451 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:58.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:58.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:58.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:58.679 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:58.709 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:58.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:58.813 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:58.846 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:58.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:58.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:58.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:22:58.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:58.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:22:59.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:59.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:22:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:59.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:22:59.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:59.245 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:22:59.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:59.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:22:59.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:59.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:22:59.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:59.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:22:59.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:59.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:22:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:59.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:22:59.603 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:59.637 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:22:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:59.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:22:59.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:22:59 smithi016 bash[75179]: cluster 2023-10-09T09:22:58.294193+0000 mgr.y (mgr.25421) 1375 : cluster 0 pgmap v1551: 99 pgs: 99 active+clean; 1.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:59.665 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:22:59 smithi016 bash[77892]: cluster 2023-10-09T09:22:58.294193+0000 mgr.y (mgr.25421) 1375 : cluster 0 pgmap v1551: 99 pgs: 99 active+clean; 1.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:59.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:22:59 smithi094 bash[59677]: cluster 2023-10-09T09:22:58.294193+0000 mgr.y (mgr.25421) 1375 : cluster 0 pgmap v1551: 99 pgs: 99 active+clean; 1.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:22:59.741 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:59.773 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:22:59.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:59.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:22:59.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:59.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:22:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:22:59.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:00.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:00.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:00.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:00.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:00.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:00.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:00.255 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:00.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:00.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:00.391 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:00.426 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:00.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:00.471 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:23:00.471 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T09:23:00.471 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T09:23:00.471 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35619.0:101159 10.8 e256/248 10:11555de3:::benchmark_data_smithi016_193276_object101158:head v 256'6452, mlcod=256'6450)", 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:13:43.167337+0000", 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 557.30047449999995, 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27235795200000001, 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.472 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167337+0000", 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167337+0000", 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167336+0000", 2023-10-09T09:23:00.473 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167359+0000", 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2203000000000001e-05 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.474 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167370+0000", 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1116e-05 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167376+0000", 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1979999999999998e-06 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.475 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167419+0000", 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3061e-05 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.167442+0000", 2023-10-09T09:23:00.476 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2929e-05 2023-10-09T09:23:00.477 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.477 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.477 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:23:00.477 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.439479+0000", 2023-10-09T09:23:00.477 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.272036736 2023-10-09T09:23:00.477 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.477 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:13:43.439695+0000", 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021662499999999999 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.478 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.479 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:81029 11.d 11:bb2d0a19:::benchmark_data_smithi016_218422_object81028:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T09:23:00.479 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.115260+0000", 2023-10-09T09:23:00.479 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 212.35255147399999, 2023-10-09T09:23:00.479 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33635030700000002, 2023-10-09T09:23:00.479 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.479 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:23:00.479 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81029 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.480 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115260+0000", 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115260+0000", 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.481 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115258+0000", 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115293+0000", 2023-10-09T09:23:00.482 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4649e-05 2023-10-09T09:23:00.483 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.483 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.483 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.483 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115307+0000", 2023-10-09T09:23:00.483 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3596e-05 2023-10-09T09:23:00.483 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.483 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115316+0000", 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5340000000000005e-06 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115362+0000", 2023-10-09T09:23:00.484 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5614000000000001e-05 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115496+0000", 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134215 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.485 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:23:00.486 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115661+0000", 2023-10-09T09:23:00.486 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016480800000000001 2023-10-09T09:23:00.486 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.486 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.486 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:23:00.486 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.450605+0000", 2023-10-09T09:23:00.486 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33494459599999998 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.450647+0000", 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1304999999999998e-05 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.487 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.451436+0000", 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00078907499999999998 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.451558+0000", 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012280799999999999 2023-10-09T09:23:00.488 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.451611+0000", 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2120000000000002e-05 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.489 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:81024 11.2 e302/294 11:467a9161:::benchmark_data_smithi016_218422_object81023:head v 302'5048, mlcod=302'5046)", 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.115787+0000", 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 212.352024269, 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33051021200000003, 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115787+0000", 2023-10-09T09:23:00.490 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115787+0000", 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115787+0000", 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.491 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115807+0000", 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0565e-05 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115818+0000", 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0722e-05 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.492 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115823+0000", 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2950000000000001e-06 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115887+0000", 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3585999999999997e-05 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:23:00.493 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.115908+0000", 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1165000000000001e-05 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.119526+0000", 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0036180209999999999 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.446298+0000", 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32677169499999997 2023-10-09T09:23:00.494 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:81032 11.d 11:b95a9d89:::benchmark_data_smithi016_218422_object81031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.117153+0000", 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 212.35065873299999, 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33622837700000002, 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.495 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81032 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.117153+0000", 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.117153+0000", 2023-10-09T09:23:00.496 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.117152+0000", 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.117176+0000", 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4290999999999998e-05 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.497 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.117199+0000", 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2592e-05 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.117208+0000", 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8940000000000005e-06 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.498 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.446411+0000", 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.329202247 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.446766+0000", 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00035540499999999999 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.447817+0000", 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010513160000000001 2023-10-09T09:23:00.499 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.451396+0000", 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035786310000000001 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.452894+0000", 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00149759 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:23:00.500 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.453218+0000", 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00032476 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.453343+0000", 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012496100000000001 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.453381+0000", 2023-10-09T09:23:00.501 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8099999999999998e-05 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.35859.0:81029 11.d e302/294 ondisk, result = 0)", 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.119990+0000", 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 212.347821401, 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.330659117, 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:23:00.502 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.119990+0000", 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.119990+0000", 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.503 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.119990+0000", 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.119994+0000", 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6460000000000001e-06 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.504 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.446209+0000", 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32621488300000001 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.446226+0000", 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6918e-05 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.450633+0000", 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00440662 2023-10-09T09:23:00.505 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.450640+0000", 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6240000000000002e-06 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.450649+0000", 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2439999999999999e-06 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.506 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.35859.0:81029 11.d e302/294 ondisk, result = 0)", 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:19:28.128915+0000", 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 212.33889680999999, 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32270461900000003, 2023-10-09T09:23:00.507 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.128915+0000", 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.508 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.128915+0000", 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.128914+0000", 2023-10-09T09:23:00.509 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:23:00.510 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.510 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.510 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.510 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.128923+0000", 2023-10-09T09:23:00.510 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8210000000000008e-06 2023-10-09T09:23:00.510 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.510 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.446258+0000", 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31733561100000002 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.446282+0000", 2023-10-09T09:23:00.511 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3791e-05 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.451422+0000", 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0051395299999999998 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.512 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:23:00.513 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.451430+0000", 2023-10-09T09:23:00.513 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8879999999999997e-06 2023-10-09T09:23:00.513 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.513 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.513 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.513 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:19:28.451620+0000", 2023-10-09T09:23:00.513 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019006000000000001 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176261 11.11 11.54cd3951 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.498029+0000", 2023-10-09T09:23:00.514 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.969782617, 2023-10-09T09:23:00.515 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.87490573599999999, 2023-10-09T09:23:00.515 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.515 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.515 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.515 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.515 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.515 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176261 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498029+0000", 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.516 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498029+0000", 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498027+0000", 2023-10-09T09:23:00.517 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.518 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.518 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.518 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.518 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498036+0000", 2023-10-09T09:23:00.518 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2530000000000004e-06 2023-10-09T09:23:00.518 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.518 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498053+0000", 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7184999999999999e-05 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498062+0000", 2023-10-09T09:23:00.519 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6889999999999993e-06 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498124+0000", 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2656000000000005e-05 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.520 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.521 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.498133+0000", 2023-10-09T09:23:00.521 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0289999999999994e-06 2023-10-09T09:23:00.521 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.521 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.521 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.521 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.372923+0000", 2023-10-09T09:23:00.521 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.87478998200000002 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.372935+0000", 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1657e-05 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.522 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.523 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.523 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.523 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176313 11.11 11.d26ee4b1 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.523 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.556405+0000", 2023-10-09T09:23:00.523 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.911406401000001, 2023-10-09T09:23:00.523 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.81655973900000001, 2023-10-09T09:23:00.523 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176313 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556405+0000", 2023-10-09T09:23:00.524 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556405+0000", 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556404+0000", 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.525 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556412+0000", 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8259999999999999e-06 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556429+0000", 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7073000000000001e-05 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556440+0000", 2023-10-09T09:23:00.526 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0258000000000001e-05 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556494+0000", 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4361999999999999e-05 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.556504+0000", 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9559999999999992e-06 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.527 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.372957+0000", 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.81645343199999998 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.372965+0000", 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6240000000000002e-06 2023-10-09T09:23:00.528 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.529 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.529 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.529 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.529 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.529 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176335 11.11 11.4593cbd1 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.529 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.579195+0000", 2023-10-09T09:23:00.530 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.888616317999997, 2023-10-09T09:23:00.530 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.79379684800000005, 2023-10-09T09:23:00.530 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.530 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.530 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.530 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.531 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.531 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176335 2023-10-09T09:23:00.531 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.531 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.531 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.531 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.531 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579195+0000", 2023-10-09T09:23:00.532 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.532 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.532 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.532 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.532 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579195+0000", 2023-10-09T09:23:00.532 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.533 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.533 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.533 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.533 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579193+0000", 2023-10-09T09:23:00.533 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.533 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.533 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.534 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.534 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579231+0000", 2023-10-09T09:23:00.534 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7839999999999997e-05 2023-10-09T09:23:00.534 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.534 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.534 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.534 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579254+0000", 2023-10-09T09:23:00.535 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2918999999999999e-05 2023-10-09T09:23:00.535 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.535 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.535 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.535 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579265+0000", 2023-10-09T09:23:00.535 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0822000000000001e-05 2023-10-09T09:23:00.535 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.536 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.536 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.536 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579326+0000", 2023-10-09T09:23:00.536 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1061000000000004e-05 2023-10-09T09:23:00.536 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.536 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.536 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.537 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.579337+0000", 2023-10-09T09:23:00.537 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0626e-05 2023-10-09T09:23:00.537 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.537 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.537 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.537 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.372985+0000", 2023-10-09T09:23:00.537 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.793648099 2023-10-09T09:23:00.538 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.538 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.538 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.538 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.372992+0000", 2023-10-09T09:23:00.538 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.515e-06 2023-10-09T09:23:00.538 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.538 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.539 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.539 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.539 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.539 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176414 11.11 11.8b780f71 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.539 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.667690+0000", 2023-10-09T09:23:00.539 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.800121603999997, 2023-10-09T09:23:00.539 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.70533068099999996, 2023-10-09T09:23:00.540 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.540 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.540 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.540 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.540 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.540 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176414 2023-10-09T09:23:00.540 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.541 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.541 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.541 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.541 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667690+0000", 2023-10-09T09:23:00.541 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.541 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.541 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.542 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.542 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667690+0000", 2023-10-09T09:23:00.542 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.542 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.542 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.542 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667688+0000", 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667696+0000", 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2309999999999999e-06 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.543 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667709+0000", 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3237999999999999e-05 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667717+0000", 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3480000000000004e-06 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667783+0000", 2023-10-09T09:23:00.544 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6255999999999997e-05 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.667794+0000", 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0594000000000001e-05 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373012+0000", 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.70521810100000004 2023-10-09T09:23:00.545 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.546 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.546 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.546 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373021+0000", 2023-10-09T09:23:00.546 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7949999999999996e-06 2023-10-09T09:23:00.546 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.546 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.546 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176515 11.11 11.a14fb6f1 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.803972+0000", 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.663840180999998, 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.56907529000000001, 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.547 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176515 2023-10-09T09:23:00.548 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.548 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.548 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.548 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.548 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.803972+0000", 2023-10-09T09:23:00.548 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.548 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.549 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.549 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.549 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.803972+0000", 2023-10-09T09:23:00.549 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.549 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.549 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.549 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.550 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.803970+0000", 2023-10-09T09:23:00.550 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:23:00.550 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.550 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.550 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.550 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.803977+0000", 2023-10-09T09:23:00.550 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2359999999999998e-06 2023-10-09T09:23:00.551 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.551 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.551 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.551 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.803992+0000", 2023-10-09T09:23:00.551 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4732000000000001e-05 2023-10-09T09:23:00.551 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.552 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.552 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.552 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.804000+0000", 2023-10-09T09:23:00.552 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7230000000000004e-06 2023-10-09T09:23:00.552 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.552 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.804054+0000", 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4324999999999999e-05 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.804064+0000", 2023-10-09T09:23:00.553 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6609999999999995e-06 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373040+0000", 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.56897610799999998 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373047+0000", 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1640000000000004e-06 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.554 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176568 11.11 11.3ca4f3b1 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.876064+0000", 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.591747304000002, 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.49700663099999998, 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.555 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176568 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876064+0000", 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.556 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876064+0000", 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876063+0000", 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876070+0000", 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2169999999999997e-06 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.557 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876089+0000", 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9020000000000001e-05 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876098+0000", 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7979999999999995e-06 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.558 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876225+0000", 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012699900000000001 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.876232+0000", 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6939999999999998e-06 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373064+0000", 2023-10-09T09:23:00.559 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.49683244799999998 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373071+0000", 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0029999999999997e-06 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.560 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.561 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.561 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.561 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176577 11.11 11.7acf6df1 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.561 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.885092+0000", 2023-10-09T09:23:00.561 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.582719939999997, 2023-10-09T09:23:00.561 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.48800276199999998, 2023-10-09T09:23:00.561 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176577 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885092+0000", 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.562 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885092+0000", 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885090+0000", 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885097+0000", 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1130000000000003e-06 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885109+0000", 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1841000000000001e-05 2023-10-09T09:23:00.564 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885116+0000", 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5980000000000002e-06 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885525+0000", 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00040964000000000001 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.565 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.885537+0000", 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1225e-05 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373088+0000", 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.48755090800000001 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373095+0000", 2023-10-09T09:23:00.566 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9720000000000003e-06 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176620 11.11 11.c308031 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.939511+0000", 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.528300283, 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.43360718399999998, 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.567 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176620 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939511+0000", 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.568 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939511+0000", 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939510+0000", 2023-10-09T09:23:00.569 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:23:00.570 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.570 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.570 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.570 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939518+0000", 2023-10-09T09:23:00.570 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6129999999999998e-06 2023-10-09T09:23:00.570 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.570 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939529+0000", 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1761e-05 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939536+0000", 2023-10-09T09:23:00.571 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8759999999999999e-06 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939597+0000", 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0420000000000001e-05 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.572 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.939614+0000", 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6926e-05 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373112+0000", 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.433498314 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373119+0000", 2023-10-09T09:23:00.573 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8369999999999999e-06 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176652 11.11 11.12c13e31 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:10.982743+0000", 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.485068454, 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.39039876499999998, 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.574 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176652 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982743+0000", 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.575 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982743+0000", 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982741+0000", 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982749+0000", 2023-10-09T09:23:00.576 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8939999999999993e-06 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982763+0000", 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4018e-05 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982772+0000", 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9490000000000002e-06 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982835+0000", 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2531000000000002e-05 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:10.982847+0000", 2023-10-09T09:23:00.578 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2031000000000001e-05 2023-10-09T09:23:00.579 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.579 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.579 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373135+0000", 2023-10-09T09:23:00.579 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.39028858799999999 2023-10-09T09:23:00.579 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373142+0000", 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7360000000000004e-06 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.580 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176674 11.11 11.76f547d1 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:11.016940+0000", 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.450871841999998, 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35622604600000002, 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.581 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176674 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.016940+0000", 2023-10-09T09:23:00.582 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.583 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.583 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.583 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.583 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.016940+0000", 2023-10-09T09:23:00.583 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.583 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.583 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.584 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.584 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.016938+0000", 2023-10-09T09:23:00.584 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:23:00.584 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.584 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.584 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.584 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.016946+0000", 2023-10-09T09:23:00.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3339999999999996e-06 2023-10-09T09:23:00.585 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.585 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.585 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.585 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.016960+0000", 2023-10-09T09:23:00.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4428e-05 2023-10-09T09:23:00.585 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.016969+0000", 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5550000000000003e-06 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.017033+0000", 2023-10-09T09:23:00.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4136999999999999e-05 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.017044+0000", 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1241e-05 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373159+0000", 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.35611489299999999 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373166+0000", 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.973e-06 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.588 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176681 11.11 11.e60cdf11 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:11.029675+0000", 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.438136696999997, 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34351403800000002, 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176681 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.589 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029675+0000", 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029675+0000", 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029673+0000", 2023-10-09T09:23:00.590 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029680+0000", 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.951e-06 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029694+0000", 2023-10-09T09:23:00.591 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3817e-05 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029701+0000", 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2150000000000004e-06 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029756+0000", 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4994999999999998e-05 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.029767+0000", 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0511e-05 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373182+0000", 2023-10-09T09:23:00.593 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.34341571300000001 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373189+0000", 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8129999999999998e-06 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.594 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176688 11.11 11.a4fbe31 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:11.036841+0000", 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.430970590999998, 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.336371692, 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.595 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176688 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.596 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036841+0000", 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036841+0000", 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.597 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.598 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.598 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.598 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036840+0000", 2023-10-09T09:23:00.598 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:23:00.598 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.598 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.598 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036847+0000", 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4560000000000003e-06 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036863+0000", 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5472999999999999e-05 2023-10-09T09:23:00.599 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036873+0000", 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0329999999999999e-05 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.600 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036945+0000", 2023-10-09T09:23:00.601 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1753000000000002e-05 2023-10-09T09:23:00.601 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.601 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.601 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.601 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.036958+0000", 2023-10-09T09:23:00.601 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3169e-05 2023-10-09T09:23:00.601 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373206+0000", 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.33624801300000001 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373213+0000", 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7580000000000003e-06 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.602 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176693 11.11 11.ee950d71 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:11.046540+0000", 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.421271308000001, 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32669561600000002, 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.603 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176693 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046540+0000", 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046540+0000", 2023-10-09T09:23:00.604 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046539+0000", 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046547+0000", 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9449999999999993e-06 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046564+0000", 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6918e-05 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046573+0000", 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0040000000000005e-06 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.606 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046663+0000", 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0299999999999999e-05 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.046675+0000", 2023-10-09T09:23:00.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1623999999999999e-05 2023-10-09T09:23:00.608 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.608 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.608 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.608 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373229+0000", 2023-10-09T09:23:00.608 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.32655487500000002 2023-10-09T09:23:00.608 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.608 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.609 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.609 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373236+0000", 2023-10-09T09:23:00.609 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6290000000000004e-06 2023-10-09T09:23:00.609 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.609 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.609 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.609 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.35859.0:176695 11.11 11.9bc18f71 (undecoded) ondisk+write+known_if_redirected e332)", 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:11.053514+0000", 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.414297691000002, 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31974532900000002, 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:23:00.610 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.35859", 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1551376562", 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 176695 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:23:00.611 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053514+0000", 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053514+0000", 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.612 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053512+0000", 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053521+0000", 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7539999999999995e-06 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:23:00.613 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053537+0000", 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5764e-05 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053545+0000", 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1839999999999999e-06 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053611+0000", 2023-10-09T09:23:00.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6421999999999995e-05 2023-10-09T09:23:00.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.615 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-10-09T09:23:00.615 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.053623+0000", 2023-10-09T09:23:00.615 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1989e-05 2023-10-09T09:23:00.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373253+0000", 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31962940400000001 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:11.373259+0000", 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8569999999999997e-06 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.616 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:23:00.617 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:23:00.617 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:23:00.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:00.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:00.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:00.744 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:00.774 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:00.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:00.878 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:00.899 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 11 pg_num to 36 2023-10-09T09:23:00.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:00.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:00.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:01.013 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:01.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:00 smithi094 bash[59677]: cluster 2023-10-09T09:22:59.813193+0000 osd.0 (osd.0) 45 : cluster 0 11.19 scrub starts 2023-10-09T09:23:01.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:00 smithi094 bash[59677]: cluster 2023-10-09T09:22:59.826157+0000 osd.0 (osd.0) 46 : cluster 0 11.19 scrub ok 2023-10-09T09:23:01.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:00 smithi094 bash[59677]: audit 2023-10-09T09:23:00.038755+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.16:0/856092610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T09:23:01.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:00 smithi094 bash[59677]: audit 2023-10-09T09:23:00.039186+0000 mon.a (mon.0) 1887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T09:23:01.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:00 smithi094 bash[59677]: cluster 2023-10-09T09:23:00.295219+0000 mgr.y (mgr.25421) 1376 : cluster 0 pgmap v1552: 99 pgs: 99 active+clean; 1.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:01.043 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:01.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:01.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[77892]: cluster 2023-10-09T09:22:59.813193+0000 osd.0 (osd.0) 45 : cluster 0 11.19 scrub starts 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[77892]: cluster 2023-10-09T09:22:59.826157+0000 osd.0 (osd.0) 46 : cluster 0 11.19 scrub ok 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[77892]: audit 2023-10-09T09:23:00.038755+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.16:0/856092610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[77892]: audit 2023-10-09T09:23:00.039186+0000 mon.a (mon.0) 1887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[77892]: cluster 2023-10-09T09:23:00.295219+0000 mgr.y (mgr.25421) 1376 : cluster 0 pgmap v1552: 99 pgs: 99 active+clean; 1.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[75179]: cluster 2023-10-09T09:22:59.813193+0000 osd.0 (osd.0) 45 : cluster 0 11.19 scrub starts 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[75179]: cluster 2023-10-09T09:22:59.826157+0000 osd.0 (osd.0) 46 : cluster 0 11.19 scrub ok 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[75179]: audit 2023-10-09T09:23:00.038755+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.16:0/856092610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T09:23:01.160 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[75179]: audit 2023-10-09T09:23:00.039186+0000 mon.a (mon.0) 1887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T09:23:01.161 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:00 smithi016 bash[75179]: cluster 2023-10-09T09:23:00.295219+0000 mgr.y (mgr.25421) 1376 : cluster 0 pgmap v1552: 99 pgs: 99 active+clean; 1.2 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:01.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:01.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:01.279 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:01.310 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:01.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:01.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:01.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:01.581 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:01.685 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:01.715 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:01.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:01.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:01.946 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:01.977 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:01.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:02.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:02.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:02.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:02.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:02.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[75179]: cluster 2023-10-09T09:23:00.554661+0000 osd.6 (osd.6) 311 : cluster 0 11.9 scrub starts 2023-10-09T09:23:02.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[75179]: cluster 2023-10-09T09:23:00.573124+0000 osd.6 (osd.6) 312 : cluster 0 11.9 scrub ok 2023-10-09T09:23:02.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[75179]: cluster 2023-10-09T09:23:00.889994+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:02.126 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[75179]: audit 2023-10-09T09:23:00.891069+0000 mon.a (mon.0) 1888 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T09:23:02.126 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[75179]: cluster 2023-10-09T09:23:00.891204+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-09T09:23:02.126 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[77892]: cluster 2023-10-09T09:23:00.554661+0000 osd.6 (osd.6) 311 : cluster 0 11.9 scrub starts 2023-10-09T09:23:02.126 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[77892]: cluster 2023-10-09T09:23:00.573124+0000 osd.6 (osd.6) 312 : cluster 0 11.9 scrub ok 2023-10-09T09:23:02.126 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[77892]: cluster 2023-10-09T09:23:00.889994+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:02.126 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[77892]: audit 2023-10-09T09:23:00.891069+0000 mon.a (mon.0) 1888 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T09:23:02.126 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:01 smithi016 bash[77892]: cluster 2023-10-09T09:23:00.891204+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-09T09:23:02.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:02.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:01 smithi094 bash[59677]: cluster 2023-10-09T09:23:00.554661+0000 osd.6 (osd.6) 311 : cluster 0 11.9 scrub starts 2023-10-09T09:23:02.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:01 smithi094 bash[59677]: cluster 2023-10-09T09:23:00.573124+0000 osd.6 (osd.6) 312 : cluster 0 11.9 scrub ok 2023-10-09T09:23:02.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:01 smithi094 bash[59677]: cluster 2023-10-09T09:23:00.889994+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:02.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:01 smithi094 bash[59677]: audit 2023-10-09T09:23:00.891069+0000 mon.a (mon.0) 1888 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T09:23:02.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:01 smithi094 bash[59677]: cluster 2023-10-09T09:23:00.891204+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-09T09:23:02.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:02.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:02.345 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:02.376 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:02.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:02.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:02.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:02.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:02.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:02.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:02.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:02.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:02.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:02.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:02.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:02.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:02.877 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:02.909 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:02.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:02.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:23:03.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:03.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:03.122 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[77892]: cluster 2023-10-09T09:23:01.524330+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:03.122 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[77892]: cluster 2023-10-09T09:23:01.897749+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:03.122 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[77892]: cluster 2023-10-09T09:23:02.295940+0000 mgr.y (mgr.25421) 1377 : cluster 0 pgmap v1554: 99 pgs: 99 active+clean; 838 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:03.122 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[77892]: audit 2023-10-09T09:23:02.297683+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T09:23:03.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[75179]: cluster 2023-10-09T09:23:01.524330+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:03.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[75179]: cluster 2023-10-09T09:23:01.897749+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:03.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[75179]: cluster 2023-10-09T09:23:02.295940+0000 mgr.y (mgr.25421) 1377 : cluster 0 pgmap v1554: 99 pgs: 99 active+clean; 838 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:03.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:02 smithi016 bash[75179]: audit 2023-10-09T09:23:02.297683+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T09:23:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:03.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:03.225 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:03.233 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:02 smithi094 bash[59677]: cluster 2023-10-09T09:23:01.524330+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:03.233 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:02 smithi094 bash[59677]: cluster 2023-10-09T09:23:01.897749+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T09:23:03.233 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:02 smithi094 bash[59677]: cluster 2023-10-09T09:23:02.295940+0000 mgr.y (mgr.25421) 1377 : cluster 0 pgmap v1554: 99 pgs: 99 active+clean; 838 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:03.234 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:02 smithi094 bash[59677]: audit 2023-10-09T09:23:02.297683+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T09:23:03.249 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:03.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:03.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:03.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:03.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:03.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:03.486 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:03.519 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:03.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:03.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:03.622 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:03.655 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:03.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:03.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:03.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:03.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:03.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:03.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:03.921 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:03.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:03.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:04.007 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T09:23:04.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T09:23:04.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[75179]: cluster 2023-10-09T09:23:02.901484+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[75179]: audit 2023-10-09T09:23:02.904935+0000 mon.a (mon.0) 1891 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T09:23:04.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[75179]: cluster 2023-10-09T09:23:02.905019+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-09T09:23:04.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[75179]: cluster 2023-10-09T09:23:02.965361+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[75179]: cluster 2023-10-09T09:23:02.966687+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[75179]: cluster 2023-10-09T09:23:02.966928+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.019 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[77892]: cluster 2023-10-09T09:23:02.901484+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.020 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[77892]: audit 2023-10-09T09:23:02.904935+0000 mon.a (mon.0) 1891 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T09:23:04.020 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[77892]: cluster 2023-10-09T09:23:02.905019+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-09T09:23:04.020 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[77892]: cluster 2023-10-09T09:23:02.965361+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.020 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[77892]: cluster 2023-10-09T09:23:02.966687+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.020 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:03 smithi016 bash[77892]: cluster 2023-10-09T09:23:02.966928+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:04.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:04.177 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:23:04.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:04.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:04.201 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:23:04.228 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:23:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:03 smithi094 bash[59677]: cluster 2023-10-09T09:23:02.901484+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:03 smithi094 bash[59677]: audit 2023-10-09T09:23:02.904935+0000 mon.a (mon.0) 1891 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T09:23:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:03 smithi094 bash[59677]: cluster 2023-10-09T09:23:02.905019+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-09T09:23:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:03 smithi094 bash[59677]: cluster 2023-10-09T09:23:02.965361+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:03 smithi094 bash[59677]: cluster 2023-10-09T09:23:02.966687+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:03 smithi094 bash[59677]: cluster 2023-10-09T09:23:02.966928+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T09:23:04.250 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:23:04.279 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:23:04.298 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:04.310 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:23:04.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:04.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:04.337 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:23:04.361 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:23:04.363 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.363 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.363 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.364 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.364 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.364 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.364 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.364 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:23:04.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:04.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:04.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:04.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:04.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:04.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:04.928 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:23:04.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.957 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:03.559453+0000 osd.6 (osd.6) 315 : cluster 0 11.14 scrub starts 2023-10-09T09:23:04.957 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:03.574168+0000 osd.6 (osd.6) 316 : cluster 0 11.14 scrub ok 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:03.903413+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:03.904105+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:03.904274+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:03.905561+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:03.908771+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: audit 2023-10-09T09:23:04.149108+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.16:0/3940489115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: audit 2023-10-09T09:23:04.265080+0000 mon.a (mon.0) 1894 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.296527+0000 mgr.y (mgr.25421) 1378 : cluster 0 pgmap v1557: 109 pgs: 1 activating, 9 unknown, 99 active+clean; 838 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: audit 2023-10-09T09:23:04.540870+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.16:0/1092862179' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:04.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[75179]: audit 2023-10-09T09:23:04.541245+0000 mon.a (mon.0) 1895 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:03.559453+0000 osd.6 (osd.6) 315 : cluster 0 11.14 scrub starts 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:03.574168+0000 osd.6 (osd.6) 316 : cluster 0 11.14 scrub ok 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:03.903413+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:03.904105+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:03.904274+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:03.905561+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:03.908771+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: audit 2023-10-09T09:23:04.149108+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.16:0/3940489115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: audit 2023-10-09T09:23:04.265080+0000 mon.a (mon.0) 1894 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:04.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.296527+0000 mgr.y (mgr.25421) 1378 : cluster 0 pgmap v1557: 109 pgs: 1 activating, 9 unknown, 99 active+clean; 838 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:04.960 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: audit 2023-10-09T09:23:04.540870+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.16:0/1092862179' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:04.960 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:04 smithi016 bash[77892]: audit 2023-10-09T09:23:04.541245+0000 mon.a (mon.0) 1895 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:04.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:04.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:05.075 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:05.104 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:05.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:05.208 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:05.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:03.559453+0000 osd.6 (osd.6) 315 : cluster 0 11.14 scrub starts 2023-10-09T09:23:05.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:03.574168+0000 osd.6 (osd.6) 316 : cluster 0 11.14 scrub ok 2023-10-09T09:23:05.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:03.903413+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:05.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:03.904105+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:05.219 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:03.904274+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-09T09:23:05.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:03.905561+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:05.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:03.908771+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T09:23:05.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: audit 2023-10-09T09:23:04.149108+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.16:0/3940489115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:05.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: audit 2023-10-09T09:23:04.265080+0000 mon.a (mon.0) 1894 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:05.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.296527+0000 mgr.y (mgr.25421) 1378 : cluster 0 pgmap v1557: 109 pgs: 1 activating, 9 unknown, 99 active+clean; 838 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:05.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: audit 2023-10-09T09:23:04.540870+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.16:0/1092862179' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:05.220 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:04 smithi094 bash[59677]: audit 2023-10-09T09:23:04.541245+0000 mon.a (mon.0) 1895 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:05.237 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:05.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:05.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:05.374 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:05.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:05.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:05.477 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:05.508 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:05.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:05.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:05.612 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:05.642 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:05.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:05.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:05.746 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:05.776 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:05.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:05.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:05.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:05.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:23:05] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.33.4" 2023-10-09T09:23:06.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.551792+0000 osd.6 (osd.6) 318 : cluster 0 11.17 scrub starts 2023-10-09T09:23:06.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.560349+0000 osd.6 (osd.6) 319 : cluster 0 11.17 scrub ok 2023-10-09T09:23:06.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.910272+0000 mon.a (mon.0) 1896 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:23:06.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.910366+0000 mon.a (mon.0) 1897 : cluster 1 Cluster is now healthy 2023-10-09T09:23:06.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.919124+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.920133+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: audit 2023-10-09T09:23:04.923393+0000 mon.a (mon.0) 1898 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:04.923477+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:05.337387+0000 osd.1 (osd.1) 19 : cluster 0 11.0 scrub starts 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:05.353154+0000 osd.1 (osd.1) 20 : cluster 0 11.0 scrub ok 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:05.361198+0000 osd.4 (osd.4) 330 : cluster 0 11.e scrub starts 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:05.373611+0000 osd.4 (osd.4) 331 : cluster 0 11.e scrub ok 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:05.757837+0000 osd.0 (osd.0) 47 : cluster 0 11.2 scrub starts 2023-10-09T09:23:06.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[75179]: cluster 2023-10-09T09:23:05.775445+0000 osd.0 (osd.0) 48 : cluster 0 11.2 scrub ok 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.551792+0000 osd.6 (osd.6) 318 : cluster 0 11.17 scrub starts 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.560349+0000 osd.6 (osd.6) 319 : cluster 0 11.17 scrub ok 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.910272+0000 mon.a (mon.0) 1896 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.910366+0000 mon.a (mon.0) 1897 : cluster 1 Cluster is now healthy 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.919124+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.920133+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: audit 2023-10-09T09:23:04.923393+0000 mon.a (mon.0) 1898 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:04.923477+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-09T09:23:06.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:05.337387+0000 osd.1 (osd.1) 19 : cluster 0 11.0 scrub starts 2023-10-09T09:23:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:05.353154+0000 osd.1 (osd.1) 20 : cluster 0 11.0 scrub ok 2023-10-09T09:23:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:05.361198+0000 osd.4 (osd.4) 330 : cluster 0 11.e scrub starts 2023-10-09T09:23:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:05.373611+0000 osd.4 (osd.4) 331 : cluster 0 11.e scrub ok 2023-10-09T09:23:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:05.757837+0000 osd.0 (osd.0) 47 : cluster 0 11.2 scrub starts 2023-10-09T09:23:06.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:05 smithi016 bash[77892]: cluster 2023-10-09T09:23:05.775445+0000 osd.0 (osd.0) 48 : cluster 0 11.2 scrub ok 2023-10-09T09:23:06.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:06.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:06.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:06.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:06.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:06.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.551792+0000 osd.6 (osd.6) 318 : cluster 0 11.17 scrub starts 2023-10-09T09:23:06.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.560349+0000 osd.6 (osd.6) 319 : cluster 0 11.17 scrub ok 2023-10-09T09:23:06.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.910272+0000 mon.a (mon.0) 1896 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:23:06.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.910366+0000 mon.a (mon.0) 1897 : cluster 1 Cluster is now healthy 2023-10-09T09:23:06.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.919124+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.920133+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: audit 2023-10-09T09:23:04.923393+0000 mon.a (mon.0) 1898 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:04.923477+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:05.337387+0000 osd.1 (osd.1) 19 : cluster 0 11.0 scrub starts 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:05.353154+0000 osd.1 (osd.1) 20 : cluster 0 11.0 scrub ok 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:05.361198+0000 osd.4 (osd.4) 330 : cluster 0 11.e scrub starts 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:05.373611+0000 osd.4 (osd.4) 331 : cluster 0 11.e scrub ok 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:05.757837+0000 osd.0 (osd.0) 47 : cluster 0 11.2 scrub starts 2023-10-09T09:23:06.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:05 smithi094 bash[59677]: cluster 2023-10-09T09:23:05.775445+0000 osd.0 (osd.0) 48 : cluster 0 11.2 scrub ok 2023-10-09T09:23:06.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:06.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:06.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:06.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:06.536 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:06.569 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:06.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:06.673 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:06.702 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:06.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:06.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:06.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:06.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:06.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:06.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:06.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:06.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:07.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:07.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:06 smithi016 bash[77892]: cluster 2023-10-09T09:23:06.297459+0000 mgr.y (mgr.25421) 1379 : cluster 0 pgmap v1559: 109 pgs: 1 activating, 5 unknown, 103 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:07.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:06 smithi016 bash[75179]: cluster 2023-10-09T09:23:06.297459+0000 mgr.y (mgr.25421) 1379 : cluster 0 pgmap v1559: 109 pgs: 1 activating, 5 unknown, 103 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:07.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:07.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:07.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:07.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:07.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:06 smithi094 bash[59677]: cluster 2023-10-09T09:23:06.297459+0000 mgr.y (mgr.25421) 1379 : cluster 0 pgmap v1559: 109 pgs: 1 activating, 5 unknown, 103 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:07.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:07.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:07.346 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:07.379 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:07.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:07.483 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:07.518 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:07.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:07.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:07.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:07.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:07.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:07.790 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:07.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:07.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:07.924 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:07.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:08.027 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:08.041 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:07 smithi094 bash[59677]: cluster 2023-10-09T09:23:07.784872+0000 osd.0 (osd.0) 49 : cluster 0 11.22 scrub starts 2023-10-09T09:23:08.041 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:07 smithi094 bash[59677]: cluster 2023-10-09T09:23:07.803153+0000 osd.0 (osd.0) 50 : cluster 0 11.22 scrub ok 2023-10-09T09:23:08.061 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:08.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:08.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:23:08.165 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:08.192 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:08.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:08.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:08.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:07 smithi016 bash[75179]: cluster 2023-10-09T09:23:07.784872+0000 osd.0 (osd.0) 49 : cluster 0 11.22 scrub starts 2023-10-09T09:23:08.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:07 smithi016 bash[75179]: cluster 2023-10-09T09:23:07.803153+0000 osd.0 (osd.0) 50 : cluster 0 11.22 scrub ok 2023-10-09T09:23:08.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:07 smithi016 bash[77892]: cluster 2023-10-09T09:23:07.784872+0000 osd.0 (osd.0) 49 : cluster 0 11.22 scrub starts 2023-10-09T09:23:08.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:07 smithi016 bash[77892]: cluster 2023-10-09T09:23:07.803153+0000 osd.0 (osd.0) 50 : cluster 0 11.22 scrub ok 2023-10-09T09:23:08.297 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:08.330 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:08.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:08.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:08.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:08.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:08.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:08.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:08.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:08.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:08.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:08.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:08.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:08.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:08.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:08.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:08.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:08.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:08.990 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:08 smithi016 bash[75179]: cluster 2023-10-09T09:23:08.298361+0000 mgr.y (mgr.25421) 1380 : cluster 0 pgmap v1560: 109 pgs: 1 activating, 108 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:08.991 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:08 smithi016 bash[77892]: cluster 2023-10-09T09:23:08.298361+0000 mgr.y (mgr.25421) 1380 : cluster 0 pgmap v1560: 109 pgs: 1 activating, 108 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:09.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:09.008 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:23:09.008 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:23:09.008 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T09:23:09.008 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:23:09.008 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:23:09.009 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T09:23:09.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:09.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:23:09.113 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:09.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:09.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:08 smithi094 bash[59677]: cluster 2023-10-09T09:23:08.298361+0000 mgr.y (mgr.25421) 1380 : cluster 0 pgmap v1560: 109 pgs: 1 activating, 108 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:09.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:09.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:09.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:09.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:09.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:09.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:09.532 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:09.533 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:09.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:09.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:09.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:09.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:09.783 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:09.816 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:09.920 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:09.947 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:09.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:09.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:10.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:10.088 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:10.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:10.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:10.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:10.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:10.327 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:10.360 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:10.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:10.464 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:10.495 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:10.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:10.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:10.599 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:10.631 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:10.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:10.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:10.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:10.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:10.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:10.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:10.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:10.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:10.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:11.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:10 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.299256+0000 mgr.y (mgr.25421) 1381 : cluster 0 pgmap v1561: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:11.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:10 smithi016 bash[75179]: audit 2023-10-09T09:23:10.301260+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "36"}]: dispatch 2023-10-09T09:23:11.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:10 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.299256+0000 mgr.y (mgr.25421) 1381 : cluster 0 pgmap v1561: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:11.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:10 smithi016 bash[77892]: audit 2023-10-09T09:23:10.301260+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "36"}]: dispatch 2023-10-09T09:23:11.002 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:11.032 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:11.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:11.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:11.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:11.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:10 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.299256+0000 mgr.y (mgr.25421) 1381 : cluster 0 pgmap v1561: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:11.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:10 smithi094 bash[59677]: audit 2023-10-09T09:23:10.301260+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "36"}]: dispatch 2023-10-09T09:23:11.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:11.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:11.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:11.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:11.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:11.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:11.657 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:11.687 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:11.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:11.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:11.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:11.920 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:11.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.489840+0000 osd.4 (osd.4) 332 : cluster 0 11.1e scrub starts 2023-10-09T09:23:11.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.499591+0000 osd.4 (osd.4) 333 : cluster 0 11.1e scrub ok 2023-10-09T09:23:11.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.666687+0000 osd.6 (osd.6) 320 : cluster 0 11.1 scrub starts 2023-10-09T09:23:11.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.683643+0000 osd.6 (osd.6) 321 : cluster 0 11.1 scrub ok 2023-10-09T09:23:11.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.892000+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T09:23:11.929 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.892410+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T09:23:11.929 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: audit 2023-10-09T09:23:10.893478+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "36"}]': finished 2023-10-09T09:23:11.929 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:11 smithi094 bash[59677]: cluster 2023-10-09T09:23:10.893570+0000 mon.a (mon.0) 1902 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-09T09:23:11.946 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:11.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:11.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:12.051 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:12.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:12.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:12.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:12.220 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:12.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:12.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.489840+0000 osd.4 (osd.4) 332 : cluster 0 11.1e scrub starts 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.499591+0000 osd.4 (osd.4) 333 : cluster 0 11.1e scrub ok 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.666687+0000 osd.6 (osd.6) 320 : cluster 0 11.1 scrub starts 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.683643+0000 osd.6 (osd.6) 321 : cluster 0 11.1 scrub ok 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.892000+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.892410+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: audit 2023-10-09T09:23:10.893478+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "36"}]': finished 2023-10-09T09:23:12.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[77892]: cluster 2023-10-09T09:23:10.893570+0000 mon.a (mon.0) 1902 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.489840+0000 osd.4 (osd.4) 332 : cluster 0 11.1e scrub starts 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.499591+0000 osd.4 (osd.4) 333 : cluster 0 11.1e scrub ok 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.666687+0000 osd.6 (osd.6) 320 : cluster 0 11.1 scrub starts 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.683643+0000 osd.6 (osd.6) 321 : cluster 0 11.1 scrub ok 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.892000+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.892410+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: audit 2023-10-09T09:23:10.893478+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "36"}]': finished 2023-10-09T09:23:12.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:11 smithi016 bash[75179]: cluster 2023-10-09T09:23:10.893570+0000 mon.a (mon.0) 1902 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-09T09:23:12.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:12.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:12.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:12.458 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:12.491 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:12.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:12.595 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:12.629 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:12.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:12.732 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:12.762 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:12.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:12.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:12.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:12.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:12.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:13.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:13.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:13.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:13.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.901246+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.901442+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.902114+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.903890+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.904046+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.904524+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.906679+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.906698+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.906807+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:11.908620+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.300094+0000 mgr.y (mgr.25421) 1382 : cluster 0 pgmap v1564: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.359628+0000 osd.2 (osd.2) 20 : cluster 0 11.d scrub starts 2023-10-09T09:23:13.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.373006+0000 osd.2 (osd.2) 21 : cluster 0 11.d scrub ok 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.901246+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.901442+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.902114+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.903890+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.904046+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.904524+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.906679+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.906698+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.906807+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.141 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:11.908620+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.142 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.300094+0000 mgr.y (mgr.25421) 1382 : cluster 0 pgmap v1564: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:13.142 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.359628+0000 osd.2 (osd.2) 20 : cluster 0 11.d scrub starts 2023-10-09T09:23:13.142 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:12 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.373006+0000 osd.2 (osd.2) 21 : cluster 0 11.d scrub ok 2023-10-09T09:23:13.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.901246+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.901442+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.902114+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-09T09:23:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.903890+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.904046+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.904524+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.906679+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.906698+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.906807+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:11.908620+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T09:23:13.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.300094+0000 mgr.y (mgr.25421) 1382 : cluster 0 pgmap v1564: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:13.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.359628+0000 osd.2 (osd.2) 20 : cluster 0 11.d scrub starts 2023-10-09T09:23:13.242 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:12 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.373006+0000 osd.2 (osd.2) 21 : cluster 0 11.d scrub ok 2023-10-09T09:23:13.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:13.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:13.393 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.420 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:13.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.554 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:13.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:13.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:13.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:13.790 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:23:13.795 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.825 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:13.843 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:13.845 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:13.928 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:13.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:14.054 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.671111+0000 osd.6 (osd.6) 324 : cluster 0 11.a scrub starts 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.683723+0000 osd.6 (osd.6) 325 : cluster 0 11.a scrub ok 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.909533+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.912708+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.914780+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.914865+0000 osd.2 (osd.2) 22 : cluster 0 11.1d starting backfill to osd.0 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.915187+0000 osd.0 (osd.0) 51 : cluster 0 11.21 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.916177+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.916374+0000 osd.2 (osd.2) 23 : cluster 0 11.1f starting backfill to osd.3 from (0'0,0'0] MAX to 345'12062 2023-10-09T09:23:14.064 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.916741+0000 osd.1 (osd.1) 21 : cluster 0 11.20 starting backfill to osd.0 from (0'0,0'0] MAX to 345'11350 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.918505+0000 osd.0 (osd.0) 52 : cluster 0 11.1b starting backfill to osd.2 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.919167+0000 osd.1 (osd.1) 22 : cluster 0 11.1c starting backfill to osd.6 from (0'0,0'0] MAX to 345'11638 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.919306+0000 osd.0 (osd.0) 53 : cluster 0 11.22 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11192 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.919754+0000 osd.6 (osd.6) 328 : cluster 0 11.1a starting backfill to osd.2 from (0'0,0'0] MAX to 345'11762 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.923573+0000 osd.1 (osd.1) 23 : cluster 0 11.1e starting backfill to osd.2 from (0'0,0'0] MAX to 345'11804 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.929778+0000 osd.1 (osd.1) 24 : cluster 0 11.23 starting backfill to osd.3 from (0'0,0'0] MAX to 345'11076 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.933263+0000 osd.1 (osd.1) 25 : cluster 0 11.20 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11350 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.933856+0000 osd.1 (osd.1) 26 : cluster 0 11.1c starting backfill to osd.7 from (0'0,0'0] MAX to 345'11638 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.937213+0000 osd.2 (osd.2) 24 : cluster 0 11.1d starting backfill to osd.1 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.950894+0000 osd.2 (osd.2) 25 : cluster 0 11.1d starting backfill to osd.7 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.953074+0000 osd.0 (osd.0) 54 : cluster 0 11.21 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.065 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.954366+0000 osd.1 (osd.1) 27 : cluster 0 11.23 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11076 2023-10-09T09:23:14.066 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.956108+0000 osd.0 (osd.0) 55 : cluster 0 11.1b starting backfill to osd.4 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.066 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.964602+0000 osd.1 (osd.1) 28 : cluster 0 11.1e starting backfill to osd.6 from (0'0,0'0] MAX to 345'11804 2023-10-09T09:23:14.066 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.968029+0000 osd.0 (osd.0) 56 : cluster 0 11.22 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11192 2023-10-09T09:23:14.066 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.969809+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.066 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.975318+0000 osd.0 (osd.0) 57 : cluster 0 11.21 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.066 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:13 smithi094 bash[59677]: cluster 2023-10-09T09:23:12.994872+0000 osd.0 (osd.0) 58 : cluster 0 11.1b starting backfill to osd.7 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.076 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:14.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:14.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:14.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.671111+0000 osd.6 (osd.6) 324 : cluster 0 11.a scrub starts 2023-10-09T09:23:14.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.683723+0000 osd.6 (osd.6) 325 : cluster 0 11.a scrub ok 2023-10-09T09:23:14.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.909533+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.912708+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.914780+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.914865+0000 osd.2 (osd.2) 22 : cluster 0 11.1d starting backfill to osd.0 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.915187+0000 osd.0 (osd.0) 51 : cluster 0 11.21 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.916177+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.916374+0000 osd.2 (osd.2) 23 : cluster 0 11.1f starting backfill to osd.3 from (0'0,0'0] MAX to 345'12062 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.916741+0000 osd.1 (osd.1) 21 : cluster 0 11.20 starting backfill to osd.0 from (0'0,0'0] MAX to 345'11350 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.918505+0000 osd.0 (osd.0) 52 : cluster 0 11.1b starting backfill to osd.2 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.919167+0000 osd.1 (osd.1) 22 : cluster 0 11.1c starting backfill to osd.6 from (0'0,0'0] MAX to 345'11638 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.919306+0000 osd.0 (osd.0) 53 : cluster 0 11.22 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11192 2023-10-09T09:23:14.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.919754+0000 osd.6 (osd.6) 328 : cluster 0 11.1a starting backfill to osd.2 from (0'0,0'0] MAX to 345'11762 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.923573+0000 osd.1 (osd.1) 23 : cluster 0 11.1e starting backfill to osd.2 from (0'0,0'0] MAX to 345'11804 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.929778+0000 osd.1 (osd.1) 24 : cluster 0 11.23 starting backfill to osd.3 from (0'0,0'0] MAX to 345'11076 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.933263+0000 osd.1 (osd.1) 25 : cluster 0 11.20 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11350 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.933856+0000 osd.1 (osd.1) 26 : cluster 0 11.1c starting backfill to osd.7 from (0'0,0'0] MAX to 345'11638 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.937213+0000 osd.2 (osd.2) 24 : cluster 0 11.1d starting backfill to osd.1 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.950894+0000 osd.2 (osd.2) 25 : cluster 0 11.1d starting backfill to osd.7 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.953074+0000 osd.0 (osd.0) 54 : cluster 0 11.21 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.954366+0000 osd.1 (osd.1) 27 : cluster 0 11.23 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11076 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.956108+0000 osd.0 (osd.0) 55 : cluster 0 11.1b starting backfill to osd.4 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.964602+0000 osd.1 (osd.1) 28 : cluster 0 11.1e starting backfill to osd.6 from (0'0,0'0] MAX to 345'11804 2023-10-09T09:23:14.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.968029+0000 osd.0 (osd.0) 56 : cluster 0 11.22 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11192 2023-10-09T09:23:14.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.969809+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.975318+0000 osd.0 (osd.0) 57 : cluster 0 11.21 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[75179]: cluster 2023-10-09T09:23:12.994872+0000 osd.0 (osd.0) 58 : cluster 0 11.1b starting backfill to osd.7 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.671111+0000 osd.6 (osd.6) 324 : cluster 0 11.a scrub starts 2023-10-09T09:23:14.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.683723+0000 osd.6 (osd.6) 325 : cluster 0 11.a scrub ok 2023-10-09T09:23:14.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.909533+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.912708+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.914780+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.914865+0000 osd.2 (osd.2) 22 : cluster 0 11.1d starting backfill to osd.0 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.915187+0000 osd.0 (osd.0) 51 : cluster 0 11.21 starting backfill to osd.2 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.916177+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.916374+0000 osd.2 (osd.2) 23 : cluster 0 11.1f starting backfill to osd.3 from (0'0,0'0] MAX to 345'12062 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.916741+0000 osd.1 (osd.1) 21 : cluster 0 11.20 starting backfill to osd.0 from (0'0,0'0] MAX to 345'11350 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.918505+0000 osd.0 (osd.0) 52 : cluster 0 11.1b starting backfill to osd.2 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.919167+0000 osd.1 (osd.1) 22 : cluster 0 11.1c starting backfill to osd.6 from (0'0,0'0] MAX to 345'11638 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.919306+0000 osd.0 (osd.0) 53 : cluster 0 11.22 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11192 2023-10-09T09:23:14.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.919754+0000 osd.6 (osd.6) 328 : cluster 0 11.1a starting backfill to osd.2 from (0'0,0'0] MAX to 345'11762 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.923573+0000 osd.1 (osd.1) 23 : cluster 0 11.1e starting backfill to osd.2 from (0'0,0'0] MAX to 345'11804 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.929778+0000 osd.1 (osd.1) 24 : cluster 0 11.23 starting backfill to osd.3 from (0'0,0'0] MAX to 345'11076 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.933263+0000 osd.1 (osd.1) 25 : cluster 0 11.20 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11350 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.933856+0000 osd.1 (osd.1) 26 : cluster 0 11.1c starting backfill to osd.7 from (0'0,0'0] MAX to 345'11638 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.937213+0000 osd.2 (osd.2) 24 : cluster 0 11.1d starting backfill to osd.1 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.950894+0000 osd.2 (osd.2) 25 : cluster 0 11.1d starting backfill to osd.7 from (0'0,0'0] MAX to 345'11390 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.953074+0000 osd.0 (osd.0) 54 : cluster 0 11.21 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.954366+0000 osd.1 (osd.1) 27 : cluster 0 11.23 starting backfill to osd.5 from (0'0,0'0] MAX to 345'11076 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.956108+0000 osd.0 (osd.0) 55 : cluster 0 11.1b starting backfill to osd.4 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.964602+0000 osd.1 (osd.1) 28 : cluster 0 11.1e starting backfill to osd.6 from (0'0,0'0] MAX to 345'11804 2023-10-09T09:23:14.198 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.968029+0000 osd.0 (osd.0) 56 : cluster 0 11.22 starting backfill to osd.6 from (0'0,0'0] MAX to 345'11192 2023-10-09T09:23:14.198 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.969809+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T09:23:14.198 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.975318+0000 osd.0 (osd.0) 57 : cluster 0 11.21 starting backfill to osd.7 from (0'0,0'0] MAX to 345'11148 2023-10-09T09:23:14.198 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:13 smithi016 bash[77892]: cluster 2023-10-09T09:23:12.994872+0000 osd.0 (osd.0) 58 : cluster 0 11.1b starting backfill to osd.7 from (0'0,0'0] MAX to 345'11394 2023-10-09T09:23:14.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:14.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:14.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:14.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:14.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:14.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:14.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:14.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:14.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:14.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:14.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:14.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:14.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:14.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:14.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:14.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:14.950 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:14.991 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:14.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:15.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:15.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.902204+0000 osd.0 (osd.0) 59 : cluster 0 11.21 deep-scrub starts 2023-10-09T09:23:15.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.911972+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.912519+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.915195+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.917403+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.919968+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.920302+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.920782+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.922033+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.924075+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.924095+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.927262+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:13.936821+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.109 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.300824+0000 mgr.y (mgr.25421) 1383 : cluster 0 pgmap v1567: 109 pgs: 1 peering, 7 remapped+peering, 101 active+clean; 598 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 712 KiB/s wr, 22 op/s 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.902204+0000 osd.0 (osd.0) 59 : cluster 0 11.21 deep-scrub starts 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.911972+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.912519+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.915195+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.917403+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.919968+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.920302+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.110 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.920782+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.922033+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.924075+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.924095+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.927262+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:13.936821+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:14 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.300824+0000 mgr.y (mgr.25421) 1383 : cluster 0 pgmap v1567: 109 pgs: 1 peering, 7 remapped+peering, 101 active+clean; 598 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 712 KiB/s wr, 22 op/s 2023-10-09T09:23:15.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:15.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:15.227 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:15.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.902204+0000 osd.0 (osd.0) 59 : cluster 0 11.21 deep-scrub starts 2023-10-09T09:23:15.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.911972+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.912519+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.915195+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.917403+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.919968+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.920302+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.920782+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.922033+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.924075+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.924095+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.927262+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:13.936821+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T09:23:15.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:14 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.300824+0000 mgr.y (mgr.25421) 1383 : cluster 0 pgmap v1567: 109 pgs: 1 peering, 7 remapped+peering, 101 active+clean; 598 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 712 KiB/s wr, 22 op/s 2023-10-09T09:23:15.259 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:15.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:15.363 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:15.392 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:15.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:15.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:15.528 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:15.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:15.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:15.659 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:15.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:15.763 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:15.789 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:15.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:15.892 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:15.920 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:15.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:15.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.925441+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.928415+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.928715+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.929357+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.930736+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.931075+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.931672+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.931788+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.932055+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.933190+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.936181+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: cluster 2023-10-09T09:23:14.937701+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[77892]: audit 2023-10-09T09:23:15.495811+0000 mon.a (mon.0) 1907 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:23:15] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.33.4" 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.925441+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.928415+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.928715+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.929357+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.930736+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.931075+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.931672+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.931788+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.932055+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.933190+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.936181+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: cluster 2023-10-09T09:23:14.937701+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:15 smithi016 bash[75179]: audit 2023-10-09T09:23:15.495811+0000 mon.a (mon.0) 1907 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:16.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:16.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:16.158 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.925441+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.928415+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.928715+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.929357+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.930736+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.931075+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.931672+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.931788+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.172 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.932055+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.173 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.933190+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.173 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.936181+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.173 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: cluster 2023-10-09T09:23:14.937701+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T09:23:16.173 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:15 smithi094 bash[59677]: audit 2023-10-09T09:23:15.495811+0000 mon.a (mon.0) 1907 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:16.186 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:16.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:16.289 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:16.317 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:16.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:16.421 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:16.445 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:16.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:16.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:16.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:16.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:16.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:16.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:16.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:16.948 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:16.974 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:16.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:16.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:17.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:17.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:17.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:17.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:17.107 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[75179]: cluster 2023-10-09T09:23:15.783457+0000 osd.6 (osd.6) 339 : cluster 0 11.3 deep-scrub starts 2023-10-09T09:23:17.107 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[75179]: cluster 2023-10-09T09:23:15.796574+0000 osd.6 (osd.6) 340 : cluster 0 11.3 deep-scrub ok 2023-10-09T09:23:17.107 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[75179]: cluster 2023-10-09T09:23:16.301865+0000 mgr.y (mgr.25421) 1384 : cluster 0 pgmap v1569: 109 pgs: 2 peering, 107 active+clean; 598 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:17.107 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[75179]: cluster 2023-10-09T09:23:16.457426+0000 osd.2 (osd.2) 26 : cluster 0 11.1f scrub starts 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[75179]: cluster 2023-10-09T09:23:16.463668+0000 osd.2 (osd.2) 27 : cluster 0 11.1f scrub ok 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[75179]: cluster 2023-10-09T09:23:16.492214+0000 osd.1 (osd.1) 29 : cluster 0 11.f scrub starts 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[75179]: cluster 2023-10-09T09:23:16.501323+0000 osd.1 (osd.1) 30 : cluster 0 11.f scrub ok 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[77892]: cluster 2023-10-09T09:23:15.783457+0000 osd.6 (osd.6) 339 : cluster 0 11.3 deep-scrub starts 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[77892]: cluster 2023-10-09T09:23:15.796574+0000 osd.6 (osd.6) 340 : cluster 0 11.3 deep-scrub ok 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[77892]: cluster 2023-10-09T09:23:16.301865+0000 mgr.y (mgr.25421) 1384 : cluster 0 pgmap v1569: 109 pgs: 2 peering, 107 active+clean; 598 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[77892]: cluster 2023-10-09T09:23:16.457426+0000 osd.2 (osd.2) 26 : cluster 0 11.1f scrub starts 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[77892]: cluster 2023-10-09T09:23:16.463668+0000 osd.2 (osd.2) 27 : cluster 0 11.1f scrub ok 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[77892]: cluster 2023-10-09T09:23:16.492214+0000 osd.1 (osd.1) 29 : cluster 0 11.f scrub starts 2023-10-09T09:23:17.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:16 smithi016 bash[77892]: cluster 2023-10-09T09:23:16.501323+0000 osd.1 (osd.1) 30 : cluster 0 11.f scrub ok 2023-10-09T09:23:17.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:17.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:17.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:16 smithi094 bash[59677]: cluster 2023-10-09T09:23:15.783457+0000 osd.6 (osd.6) 339 : cluster 0 11.3 deep-scrub starts 2023-10-09T09:23:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:16 smithi094 bash[59677]: cluster 2023-10-09T09:23:15.796574+0000 osd.6 (osd.6) 340 : cluster 0 11.3 deep-scrub ok 2023-10-09T09:23:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:16 smithi094 bash[59677]: cluster 2023-10-09T09:23:16.301865+0000 mgr.y (mgr.25421) 1384 : cluster 0 pgmap v1569: 109 pgs: 2 peering, 107 active+clean; 598 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:16 smithi094 bash[59677]: cluster 2023-10-09T09:23:16.457426+0000 osd.2 (osd.2) 26 : cluster 0 11.1f scrub starts 2023-10-09T09:23:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:16 smithi094 bash[59677]: cluster 2023-10-09T09:23:16.463668+0000 osd.2 (osd.2) 27 : cluster 0 11.1f scrub ok 2023-10-09T09:23:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:16 smithi094 bash[59677]: cluster 2023-10-09T09:23:16.492214+0000 osd.1 (osd.1) 29 : cluster 0 11.f scrub starts 2023-10-09T09:23:17.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:16 smithi094 bash[59677]: cluster 2023-10-09T09:23:16.501323+0000 osd.1 (osd.1) 30 : cluster 0 11.f scrub ok 2023-10-09T09:23:17.332 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:17.363 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-10-09T09:23:17.363 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:23:17.364 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:17.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:17.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:17.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:17.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:17.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:17.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:17.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:17.717 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:17.751 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:17.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:17.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:17.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:17.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:17.958 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_5' removed 2023-10-09T09:23:17.991 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:17 smithi016 bash[75179]: audit 2023-10-09T09:23:17.745817+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.16:0/489607116' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:23:17.991 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:17 smithi016 bash[75179]: audit 2023-10-09T09:23:17.746397+0000 mon.a (mon.0) 1908 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:23:17.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:17 smithi094 bash[59677]: audit 2023-10-09T09:23:17.745817+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.16:0/489607116' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:23:17.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:17 smithi094 bash[59677]: audit 2023-10-09T09:23:17.746397+0000 mon.a (mon.0) 1908 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:23:17.992 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:17 smithi016 bash[77892]: audit 2023-10-09T09:23:17.745817+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.16:0/489607116' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:23:17.992 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:17 smithi016 bash[77892]: audit 2023-10-09T09:23:17.746397+0000 mon.a (mon.0) 1908 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:23:17.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:18.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:18.203 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:23:18.203 INFO:teuthology.orchestra.run.smithi094.stdout: "success": "" 2023-10-09T09:23:18.203 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:23:18.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:18.286 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:18.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:18.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:18.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:18.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:18.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:18.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:18.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:18.932 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:18.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:19.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: cluster 2023-10-09T09:23:17.444048+0000 osd.4 (osd.4) 343 : cluster 0 11.1b scrub starts 2023-10-09T09:23:19.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: cluster 2023-10-09T09:23:17.454338+0000 osd.4 (osd.4) 344 : cluster 0 11.1b scrub ok 2023-10-09T09:23:19.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: cluster 2023-10-09T09:23:17.738560+0000 osd.6 (osd.6) 341 : cluster 0 11.1e scrub starts 2023-10-09T09:23:19.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: cluster 2023-10-09T09:23:17.753788+0000 osd.6 (osd.6) 342 : cluster 0 11.1e scrub ok 2023-10-09T09:23:19.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: cluster 2023-10-09T09:23:17.943386+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T09:23:19.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: audit 2023-10-09T09:23:17.944806+0000 mon.a (mon.0) 1909 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:23:19.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: cluster 2023-10-09T09:23:17.945032+0000 mon.a (mon.0) 1910 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-09T09:23:19.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[75179]: cluster 2023-10-09T09:23:18.302481+0000 mgr.y (mgr.25421) 1385 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:19.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: cluster 2023-10-09T09:23:17.444048+0000 osd.4 (osd.4) 343 : cluster 0 11.1b scrub starts 2023-10-09T09:23:19.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: cluster 2023-10-09T09:23:17.454338+0000 osd.4 (osd.4) 344 : cluster 0 11.1b scrub ok 2023-10-09T09:23:19.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: cluster 2023-10-09T09:23:17.738560+0000 osd.6 (osd.6) 341 : cluster 0 11.1e scrub starts 2023-10-09T09:23:19.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: cluster 2023-10-09T09:23:17.753788+0000 osd.6 (osd.6) 342 : cluster 0 11.1e scrub ok 2023-10-09T09:23:19.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: cluster 2023-10-09T09:23:17.943386+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T09:23:19.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: audit 2023-10-09T09:23:17.944806+0000 mon.a (mon.0) 1909 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:23:19.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: cluster 2023-10-09T09:23:17.945032+0000 mon.a (mon.0) 1910 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-09T09:23:19.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:18 smithi016 bash[77892]: cluster 2023-10-09T09:23:18.302481+0000 mgr.y (mgr.25421) 1385 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:19.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:19.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:19.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:19.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:19.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: cluster 2023-10-09T09:23:17.444048+0000 osd.4 (osd.4) 343 : cluster 0 11.1b scrub starts 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: cluster 2023-10-09T09:23:17.454338+0000 osd.4 (osd.4) 344 : cluster 0 11.1b scrub ok 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: cluster 2023-10-09T09:23:17.738560+0000 osd.6 (osd.6) 341 : cluster 0 11.1e scrub starts 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: cluster 2023-10-09T09:23:17.753788+0000 osd.6 (osd.6) 342 : cluster 0 11.1e scrub ok 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: cluster 2023-10-09T09:23:17.943386+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: audit 2023-10-09T09:23:17.944806+0000 mon.a (mon.0) 1909 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: cluster 2023-10-09T09:23:17.945032+0000 mon.a (mon.0) 1910 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-09T09:23:19.216 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:18 smithi094 bash[59677]: cluster 2023-10-09T09:23:18.302481+0000 mgr.y (mgr.25421) 1385 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:19.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:19.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:19.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:19.429 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:19.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:19.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:19.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:19.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:19.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:19.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:19.815 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:19.848 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:19.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:19.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:19.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:20.082 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:20.114 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:20.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:20.218 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:20.251 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:20.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:20.354 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:20.366 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:20 smithi094 bash[59677]: audit 2023-10-09T09:23:19.268878+0000 mon.a (mon.0) 1911 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:20.366 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:20 smithi094 bash[59677]: audit 2023-10-09T09:23:19.270056+0000 mon.a (mon.0) 1912 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:20.383 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:20.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:20.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:20.487 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:20.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:20 smithi016 bash[75179]: audit 2023-10-09T09:23:19.268878+0000 mon.a (mon.0) 1911 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:20.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:20 smithi016 bash[75179]: audit 2023-10-09T09:23:19.270056+0000 mon.a (mon.0) 1912 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:20 smithi016 bash[77892]: audit 2023-10-09T09:23:19.268878+0000 mon.a (mon.0) 1911 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:20.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:20 smithi016 bash[77892]: audit 2023-10-09T09:23:19.270056+0000 mon.a (mon.0) 1912 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:20.522 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:20.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:20.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:20.626 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:20.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:20.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:20.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:20.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:20.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:20.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:20.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:20.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:20.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:20.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:21.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:21.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:21.151 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:21.184 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:21.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:21.288 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:21.320 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:21.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:21.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:21.437 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:21 smithi094 bash[59677]: cluster 2023-10-09T09:23:20.303253+0000 mgr.y (mgr.25421) 1386 : cluster 0 pgmap v1572: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:21.437 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:21 smithi094 bash[59677]: cluster 2023-10-09T09:23:20.768079+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T09:23:21.455 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:21.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:21 smithi016 bash[75179]: cluster 2023-10-09T09:23:20.303253+0000 mgr.y (mgr.25421) 1386 : cluster 0 pgmap v1572: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:21.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:21 smithi016 bash[75179]: cluster 2023-10-09T09:23:20.768079+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T09:23:21.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:21 smithi016 bash[77892]: cluster 2023-10-09T09:23:20.303253+0000 mgr.y (mgr.25421) 1386 : cluster 0 pgmap v1572: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:21.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:21 smithi016 bash[77892]: cluster 2023-10-09T09:23:20.768079+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T09:23:21.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:21.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:21.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:21.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:21.827 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:21.861 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:21.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:21.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:21.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:21.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:22.099 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:22.130 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:22.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:22.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:22.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:22.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:22.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:22.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:22.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:22.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:22.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:23:22.554 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:23:22.554 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T09:23:22.554 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T09:23:22.555 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T09:23:22.555 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T09:23:22.555 DEBUG:teuthology.orchestra.run.smithi016:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:23:22.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:22.665 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:22.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:22.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.710 INFO:teuthology.orchestra.run.smithi016.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T09:23:22.710 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:23:22.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:22.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:22.801 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T09:23:22.801 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T09:23:22.850 INFO:tasks.workunit:timeout=3h 2023-10-09T09:23:22.850 INFO:tasks.workunit:cleanup=True 2023-10-09T09:23:22.851 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:23:22.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:22.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:22.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:23.074 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:23.097 INFO:tasks.workunit.client.0.smithi016.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T09:23:23.100 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:23.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:23.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:23.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:23.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:23.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:23.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:23.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:23.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:23.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:23.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:23 smithi016 bash[75179]: cluster 2023-10-09T09:23:22.303839+0000 mgr.y (mgr.25421) 1387 : cluster 0 pgmap v1573: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:23.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:23 smithi016 bash[77892]: cluster 2023-10-09T09:23:22.303839+0000 mgr.y (mgr.25421) 1387 : cluster 0 pgmap v1573: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:23.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:23.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:23.592 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:23.621 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:23.623 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:23 smithi094 bash[59677]: cluster 2023-10-09T09:23:22.303839+0000 mgr.y (mgr.25421) 1387 : cluster 0 pgmap v1573: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:23.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:23.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:23.724 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:23.759 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:23.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:23.865 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:23.894 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:23.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:23.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:23.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:24.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:24.124 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:24.156 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:24.218 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:23:24.218 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:23:24.218 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T09:23:24.218 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:23:24.218 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:23:24.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T09:23:24.260 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:24.290 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:24.394 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:24.425 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:24.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:24.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:24.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:24.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:24.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:24.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:24.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:24.778 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:24.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:24.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:24.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:24.945 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:24.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:25.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:25.078 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:25.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:25.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:25.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:25.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:25.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:25.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:25.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:25.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:25.373 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-10-09T09:23:25.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:25.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:25 smithi016 bash[75179]: cluster 2023-10-09T09:23:24.304643+0000 mgr.y (mgr.25421) 1388 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:25.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:25 smithi016 bash[75179]: audit 2023-10-09T09:23:24.810881+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.16:0/2104253529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:23:25.449 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:25 smithi016 bash[75179]: audit 2023-10-09T09:23:24.811226+0000 mon.a (mon.0) 1913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:23:25.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:25 smithi016 bash[77892]: cluster 2023-10-09T09:23:24.304643+0000 mgr.y (mgr.25421) 1388 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:25.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:25 smithi016 bash[77892]: audit 2023-10-09T09:23:24.810881+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.16:0/2104253529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:23:25.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:25 smithi016 bash[77892]: audit 2023-10-09T09:23:24.811226+0000 mon.a (mon.0) 1913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:23:25.478 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:25.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:25.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:25.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:25.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:25.663 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:23:25.688 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:23:25.715 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:25.718 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:23:25.727 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:25 smithi094 bash[59677]: cluster 2023-10-09T09:23:24.304643+0000 mgr.y (mgr.25421) 1388 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:25.727 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:25 smithi094 bash[59677]: audit 2023-10-09T09:23:24.810881+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.16:0/2104253529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:23:25.727 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:25 smithi094 bash[59677]: audit 2023-10-09T09:23:24.811226+0000 mon.a (mon.0) 1913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:23:25.741 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:23:25.747 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:25.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:25.765 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:23:25.807 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:23:25.834 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:23:25.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:25.862 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:23:25.864 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.864 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.864 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.864 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.864 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.865 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.865 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.865 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:23:25.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:25.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:25.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:25.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:25.992 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:23:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:23:25] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.33.4" 2023-10-09T09:23:26.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:26.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:26.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:26.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:26.239 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:26.274 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:26.377 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:26.403 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:26.507 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:26.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:26 smithi094 bash[59677]: cluster 2023-10-09T09:23:25.360939+0000 mon.a (mon.0) 1914 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:23:26.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:26 smithi094 bash[59677]: cluster 2023-10-09T09:23:25.364783+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T09:23:26.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:26 smithi094 bash[59677]: cluster 2023-10-09T09:23:25.365068+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T09:23:26.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:26 smithi094 bash[59677]: audit 2023-10-09T09:23:25.366044+0000 mon.a (mon.0) 1915 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:23:26.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:26 smithi094 bash[59677]: cluster 2023-10-09T09:23:25.366142+0000 mon.a (mon.0) 1916 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-09T09:23:26.521 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:26 smithi094 bash[59677]: audit 2023-10-09T09:23:25.512875+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:26.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:26 smithi094 bash[59677]: audit 2023-10-09T09:23:25.638915+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.16:0/343686619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:26.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:26.647 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:26.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:26.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[77892]: cluster 2023-10-09T09:23:25.360939+0000 mon.a (mon.0) 1914 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[77892]: cluster 2023-10-09T09:23:25.364783+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[77892]: cluster 2023-10-09T09:23:25.365068+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[77892]: audit 2023-10-09T09:23:25.366044+0000 mon.a (mon.0) 1915 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[77892]: cluster 2023-10-09T09:23:25.366142+0000 mon.a (mon.0) 1916 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[77892]: audit 2023-10-09T09:23:25.512875+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[77892]: audit 2023-10-09T09:23:25.638915+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.16:0/343686619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[75179]: cluster 2023-10-09T09:23:25.360939+0000 mon.a (mon.0) 1914 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:23:26.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[75179]: cluster 2023-10-09T09:23:25.364783+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T09:23:26.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[75179]: cluster 2023-10-09T09:23:25.365068+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T09:23:26.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[75179]: audit 2023-10-09T09:23:25.366044+0000 mon.a (mon.0) 1915 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:23:26.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[75179]: cluster 2023-10-09T09:23:25.366142+0000 mon.a (mon.0) 1916 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-09T09:23:26.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[75179]: audit 2023-10-09T09:23:25.512875+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:26.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:26 smithi016 bash[75179]: audit 2023-10-09T09:23:25.638915+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.16:0/343686619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:26.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:26.817 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:26.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:26.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:26.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:26.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:27.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:27.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:27.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:27.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:27.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:27.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:27.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:27.306 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:27.340 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:27.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:27.446 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:27.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:27.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:27.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:27.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:27.593 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:27 smithi094 bash[59677]: cluster 2023-10-09T09:23:26.305509+0000 mgr.y (mgr.25421) 1389 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:27.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:27.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:27.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:27.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:27 smithi016 bash[75179]: cluster 2023-10-09T09:23:26.305509+0000 mgr.y (mgr.25421) 1389 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:27.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:27 smithi016 bash[77892]: cluster 2023-10-09T09:23:26.305509+0000 mgr.y (mgr.25421) 1389 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:27.751 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:27.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:27.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:27.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:27.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:27.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:27.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:28.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:28.128 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:28.157 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:28.261 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:28.295 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:28.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:28.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:28.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:28.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:28.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:28.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:28.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:28.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:28.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:28.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:28.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:28.949 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:28.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:23:28.981 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:28.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:29.085 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:29.112 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:29.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:29.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:29.216 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:29.243 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:29.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:29.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:29.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:29.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:29.479 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:29.514 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:29.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:29.618 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:29.648 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:29.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:29.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:29 smithi016 bash[75179]: cluster 2023-10-09T09:23:28.306394+0000 mgr.y (mgr.25421) 1390 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:29 smithi016 bash[77892]: cluster 2023-10-09T09:23:28.306394+0000 mgr.y (mgr.25421) 1390 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:29.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:29.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:29.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:29.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:23:29.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:29.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:29.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:29.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:29.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:29 smithi094 bash[59677]: cluster 2023-10-09T09:23:28.306394+0000 mgr.y (mgr.25421) 1390 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:30.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:30.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:30.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:30.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:30.168 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:30.204 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:30.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:30.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:30.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:30.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:30.377 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:30.380 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:30.442 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:30.477 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:30.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:30.581 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:30.615 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:30.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:30.719 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:30.752 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:30.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:30.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:30.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:30.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:30.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:30.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:30 smithi094 bash[59677]: cluster 2023-10-09T09:23:30.307123+0000 mgr.y (mgr.25421) 1391 : cluster 0 pgmap v1578: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:30.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:31.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:31.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:31.129 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:30 smithi016 bash[77892]: cluster 2023-10-09T09:23:30.307123+0000 mgr.y (mgr.25421) 1391 : cluster 0 pgmap v1578: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:31.129 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:30 smithi016 bash[75179]: cluster 2023-10-09T09:23:30.307123+0000 mgr.y (mgr.25421) 1391 : cluster 0 pgmap v1578: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:31.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:31.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:31.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:31.283 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:31.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:31.387 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:31.423 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:31.527 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:31.557 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:31.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:31.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:31.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:31.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:31.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:31.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:31.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:31.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:31.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:32.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:32.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:32.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:32.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:32.179 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:32.214 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:32.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:32.318 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:32.351 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:32.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:32.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:32.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:32.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:32.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:32.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:32.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:32.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:32.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:32.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:32.758 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T09:23:32.807 INFO:tasks.workunit.client.0.smithi016.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T09:23:32.810 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:23:32.811 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T09:23:32.838 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:32.870 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:32.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:32.975 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:33.008 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:33.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:33.031 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T09:23:33.113 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:33.141 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:33.245 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:33.277 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:33.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:33.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:33.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:33.420 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:33.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:33 smithi094 bash[59677]: cluster 2023-10-09T09:23:32.307621+0000 mgr.y (mgr.25421) 1392 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:33.512 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:33.542 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:33.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:33.569 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T09:23:33.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:33.647 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:23:33.649 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:23:33.649 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T09:23:33.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:33 smithi016 bash[75179]: cluster 2023-10-09T09:23:32.307621+0000 mgr.y (mgr.25421) 1392 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:33.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:33 smithi016 bash[77892]: cluster 2023-10-09T09:23:32.307621+0000 mgr.y (mgr.25421) 1392 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:33.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:33.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:33.724 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:23:33.727 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T09:23:33.727 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T09:23:33.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:33.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:33.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:33.906 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:33.941 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:33.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:34.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:34.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:34.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:34.174 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:34.208 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:34.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:34.311 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T09:23:34.312 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T09:23:34.313 DEBUG:teuthology.orchestra.run.smithi016:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T09:23:34.313 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:34.343 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:34.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:34.434 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:34.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:34.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:34.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:34.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:34.578 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:34.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:34.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:34.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:34.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:34.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:34.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:34.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:34.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:34.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:35.014 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:35.121 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:35.154 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:35.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:35.258 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:35.291 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:35.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:35.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:35.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:35.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:35 smithi016 bash[77892]: audit 2023-10-09T09:23:34.269315+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:35.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:35 smithi016 bash[77892]: audit 2023-10-09T09:23:34.270694+0000 mon.a (mon.0) 1919 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:35.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:35 smithi016 bash[77892]: cluster 2023-10-09T09:23:34.308358+0000 mgr.y (mgr.25421) 1393 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:35.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:35 smithi016 bash[75179]: audit 2023-10-09T09:23:34.269315+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:35.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:35 smithi016 bash[75179]: audit 2023-10-09T09:23:34.270694+0000 mon.a (mon.0) 1919 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:35.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:35 smithi016 bash[75179]: cluster 2023-10-09T09:23:34.308358+0000 mgr.y (mgr.25421) 1393 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:35.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:35.627 INFO:tasks.workunit.client.0.smithi016.stdout:Running main() from gmock_main.cc 2023-10-09T09:23:35.627 INFO:tasks.workunit.client.0.smithi016.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T09:23:35.627 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] Global test environment set-up. 2023-10-09T09:23:35.628 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T09:23:35.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:35.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:35 smithi094 bash[59677]: audit 2023-10-09T09:23:34.269315+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:35.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:35 smithi094 bash[59677]: audit 2023-10-09T09:23:34.270694+0000 mon.a (mon.0) 1919 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:35.664 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:35 smithi094 bash[59677]: cluster 2023-10-09T09:23:34.308358+0000 mgr.y (mgr.25421) 1393 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:35.682 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:35.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:35.793 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:23:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:23:35] "GET /metrics HTTP/1.1" 200 34043 "" "Prometheus/2.33.4" 2023-10-09T09:23:35.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:35.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:35.915 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:35.946 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:35.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:35.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:36.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:36.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:36.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:36.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:36.178 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:36.209 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:36.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:36.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:36.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:36.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:36.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:36.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:36.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:36.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:36.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:36.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:36.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:36.704 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:36.737 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:36.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:36.841 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:36.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:36.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:36.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:36.977 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:37.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:37.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:37.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:37.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.285 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:37.290 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T09:23:37.389 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.419 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:37.487 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_all_features (197 ms) 2023-10-09T09:23:37.488 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T09:23:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[75179]: cluster 2023-10-09T09:23:36.277704+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T09:23:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[75179]: cluster 2023-10-09T09:23:36.279227+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-09T09:23:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[75179]: cluster 2023-10-09T09:23:36.308700+0000 mgr.y (mgr.25421) 1394 : cluster 0 pgmap v1582: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:37.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[75179]: audit 2023-10-09T09:23:36.337699+0000 mon.a (mon.0) 1921 : audit 1 from='client.? 172.21.15.16:0/765712967' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-239143-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:23:37.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[77892]: cluster 2023-10-09T09:23:36.277704+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T09:23:37.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[77892]: cluster 2023-10-09T09:23:36.279227+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-09T09:23:37.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[77892]: cluster 2023-10-09T09:23:36.308700+0000 mgr.y (mgr.25421) 1394 : cluster 0 pgmap v1582: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:37.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:37 smithi016 bash[77892]: audit 2023-10-09T09:23:36.337699+0000 mon.a (mon.0) 1921 : audit 1 from='client.? 172.21.15.16:0/765712967' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-239143-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:23:37.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:37.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:37.653 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:37.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:37 smithi094 bash[59677]: cluster 2023-10-09T09:23:36.277704+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T09:23:37.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:37 smithi094 bash[59677]: cluster 2023-10-09T09:23:36.279227+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-09T09:23:37.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:37 smithi094 bash[59677]: cluster 2023-10-09T09:23:36.308700+0000 mgr.y (mgr.25421) 1394 : cluster 0 pgmap v1582: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:37.665 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:37 smithi094 bash[59677]: audit 2023-10-09T09:23:36.337699+0000 mon.a (mon.0) 1921 : audit 1 from='client.? 172.21.15.16:0/765712967' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-239143-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:23:37.684 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:37.754 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.copyup (267 ms) 2023-10-09T09:23:37.754 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T09:23:37.789 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.820 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:37.870 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.sparse_copyup (116 ms) 2023-10-09T09:23:37.870 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T09:23:37.883 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_and_set_id (13 ms) 2023-10-09T09:23:37.883 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T09:23:37.923 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:37.956 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:37.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:38.060 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:38.094 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:38.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:38.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:38.198 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:38.230 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:38.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:38.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:38.273 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.add_remove_child (391 ms) 2023-10-09T09:23:38.274 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T09:23:38.330 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.directory_methods (56 ms) 2023-10-09T09:23:38.330 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.create 2023-10-09T09:23:38.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:38.360 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.create (30 ms) 2023-10-09T09:23:38.360 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T09:23:38.362 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:38.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:38.365 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_features (5 ms) 2023-10-09T09:23:38.365 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T09:23:38.367 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T09:23:38.367 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T09:23:38.369 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T09:23:38.369 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T09:23:38.371 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-10-09T09:23:38.371 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T09:23:38.373 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T09:23:38.373 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T09:23:38.380 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_data_pool (7 ms) 2023-10-09T09:23:38.380 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T09:23:38.388 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_size (8 ms) 2023-10-09T09:23:38.389 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T09:23:38.396 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.set_size (8 ms) 2023-10-09T09:23:38.396 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T09:23:38.424 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.protection_status (29 ms) 2023-10-09T09:23:38.425 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T09:23:38.438 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapshot_limits (14 ms) 2023-10-09T09:23:38.439 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T09:23:38.452 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: cluster 2023-10-09T09:23:37.284285+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: cluster 2023-10-09T09:23:37.284610+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: audit 2023-10-09T09:23:37.286657+0000 mon.a (mon.0) 1922 : audit 1 from='client.? 172.21.15.16:0/765712967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-239143-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:23:38.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: cluster 2023-10-09T09:23:37.286745+0000 mon.a (mon.0) 1923 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-09T09:23:38.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: cluster 2023-10-09T09:23:37.287699+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: cluster 2023-10-09T09:23:37.288011+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: cluster 2023-10-09T09:23:37.288059+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.453 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[75179]: audit 2023-10-09T09:23:38.278872+0000 mon.a (mon.0) 1924 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: cluster 2023-10-09T09:23:37.284285+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: cluster 2023-10-09T09:23:37.284610+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: audit 2023-10-09T09:23:37.286657+0000 mon.a (mon.0) 1922 : audit 1 from='client.? 172.21.15.16:0/765712967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-239143-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: cluster 2023-10-09T09:23:37.286745+0000 mon.a (mon.0) 1923 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: cluster 2023-10-09T09:23:37.287699+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: cluster 2023-10-09T09:23:37.288011+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: cluster 2023-10-09T09:23:37.288059+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.454 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:38 smithi016 bash[77892]: audit 2023-10-09T09:23:38.278872+0000 mon.a (mon.0) 1924 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:23:38.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: cluster 2023-10-09T09:23:37.284285+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: cluster 2023-10-09T09:23:37.284610+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: audit 2023-10-09T09:23:37.286657+0000 mon.a (mon.0) 1922 : audit 1 from='client.? 172.21.15.16:0/765712967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-239143-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:23:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: cluster 2023-10-09T09:23:37.286745+0000 mon.a (mon.0) 1923 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-09T09:23:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: cluster 2023-10-09T09:23:37.287699+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: cluster 2023-10-09T09:23:37.288011+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: cluster 2023-10-09T09:23:37.288059+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T09:23:38.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:38 smithi094 bash[59677]: audit 2023-10-09T09:23:38.278872+0000 mon.a (mon.0) 1924 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:23:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:38.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:38.510 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.parents_v1 (71 ms) 2023-10-09T09:23:38.510 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T09:23:38.552 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.parents_v2 (42 ms) 2023-10-09T09:23:38.552 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T09:23:38.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:38.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:38.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:38.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:38.744 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapshots (192 ms) 2023-10-09T09:23:38.744 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T09:23:38.749 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-10-09T09:23:38.749 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T09:23:38.762 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.stripingv2 (14 ms) 2023-10-09T09:23:38.762 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T09:23:38.765 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-10-09T09:23:38.765 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T09:23:38.775 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_resize (10 ms) 2023-10-09T09:23:38.775 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T09:23:38.781 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_update (6 ms) 2023-10-09T09:23:38.781 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T09:23:38.781 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T09:23:38.781 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T09:23:38.788 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-10-09T09:23:38.788 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T09:23:38.795 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2023-10-09T09:23:38.795 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T09:23:38.803 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.flags (8 ms) 2023-10-09T09:23:38.803 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T09:23:38.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.846 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.metadata (44 ms) 2023-10-09T09:23:38.847 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T09:23:38.861 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.set_features (14 ms) 2023-10-09T09:23:38.861 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T09:23:38.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:38.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:38.914 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror (53 ms) 2023-10-09T09:23:38.914 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T09:23:38.939 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image (25 ms) 2023-10-09T09:23:38.940 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T09:23:38.979 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:39.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:39.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:39.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:39.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:39.244 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:39.274 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:39.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:39.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[77892]: cluster 2023-10-09T09:23:38.289306+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T09:23:39.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[77892]: cluster 2023-10-09T09:23:38.296266+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:39.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[77892]: cluster 2023-10-09T09:23:38.309231+0000 mgr.y (mgr.25421) 1395 : cluster 0 pgmap v1585: 105 pgs: 1 creating+activating, 19 unknown, 85 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:39.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[77892]: cluster 2023-10-09T09:23:38.368738+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:39.388 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[75179]: cluster 2023-10-09T09:23:38.289306+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T09:23:39.388 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[75179]: cluster 2023-10-09T09:23:38.296266+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:39.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[75179]: cluster 2023-10-09T09:23:38.309231+0000 mgr.y (mgr.25421) 1395 : cluster 0 pgmap v1585: 105 pgs: 1 creating+activating, 19 unknown, 85 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:39.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:39 smithi016 bash[75179]: cluster 2023-10-09T09:23:38.368738+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:39.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:39.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:39.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:39.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:39.636 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:39.648 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:39 smithi094 bash[59677]: cluster 2023-10-09T09:23:38.289306+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T09:23:39.648 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:39 smithi094 bash[59677]: cluster 2023-10-09T09:23:38.296266+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:39.648 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:39 smithi094 bash[59677]: cluster 2023-10-09T09:23:38.309231+0000 mgr.y (mgr.25421) 1395 : cluster 0 pgmap v1585: 105 pgs: 1 creating+activating, 19 unknown, 85 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:39.648 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:39 smithi094 bash[59677]: cluster 2023-10-09T09:23:38.368738+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:39.665 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:39.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:39.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:39.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:39.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:39.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:39.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:39.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:40.023 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:40.056 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:40.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:40.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:40.159 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:40.189 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:40.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:40.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:40.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:40.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:40.454 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:40.485 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:40.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:40.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:40.588 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:40.599 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:40 smithi094 bash[59677]: cluster 2023-10-09T09:23:38.331295+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:40.619 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:40.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:40.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:40.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:40.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:40 smithi016 bash[75179]: cluster 2023-10-09T09:23:38.331295+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:40.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:40 smithi016 bash[77892]: cluster 2023-10-09T09:23:38.331295+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T09:23:40.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:40.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:40.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:40.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:40.880 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:40.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:40.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:41.015 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:41.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:41.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:41.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:41.253 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:41.287 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:41.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:41.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:41.397 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:41 smithi016 bash[75179]: cluster 2023-10-09T09:23:40.309966+0000 mgr.y (mgr.25421) 1396 : cluster 0 pgmap v1586: 105 pgs: 1 creating+activating, 11 unknown, 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 37 KiB/s rd, 9.3 KiB/s wr, 56 op/s 2023-10-09T09:23:41.398 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:41 smithi016 bash[77892]: cluster 2023-10-09T09:23:40.309966+0000 mgr.y (mgr.25421) 1396 : cluster 0 pgmap v1586: 105 pgs: 1 creating+activating, 11 unknown, 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 37 KiB/s rd, 9.3 KiB/s wr, 56 op/s 2023-10-09T09:23:41.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:41.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:41.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:41.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:41.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:41.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:41.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:41 smithi094 bash[59677]: cluster 2023-10-09T09:23:40.309966+0000 mgr.y (mgr.25421) 1396 : cluster 0 pgmap v1586: 105 pgs: 1 creating+activating, 11 unknown, 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 37 KiB/s rd, 9.3 KiB/s wr, 56 op/s 2023-10-09T09:23:41.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:41.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:41.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:41.899 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:41.931 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:41.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:42.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:42.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:42.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:42.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:42.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:42.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:42.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:42.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:42.548 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:42.579 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:42.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:42.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:42.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:42.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:42.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:42.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:43.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:43.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:43.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:43.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:43.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:43.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:43.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:43.295 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image_status (4356 ms) 2023-10-09T09:23:43.295 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T09:23:43.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:43.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:43.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:43.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:43.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:43.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:43.492 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image_map (198 ms) 2023-10-09T09:23:43.493 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T09:23:43.506 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_instances (13 ms) 2023-10-09T09:23:43.506 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T09:23:43.522 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_snapshot (16 ms) 2023-10-09T09:23:43.522 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T09:23:43.526 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_list (4 ms) 2023-10-09T09:23:43.526 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T09:23:43.530 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-10-09T09:23:43.530 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T09:23:43.536 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.dir_add_already_existing (6 ms) 2023-10-09T09:23:43.536 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T09:23:43.545 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_rename (10 ms) 2023-10-09T09:23:43.546 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T09:23:43.552 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_remove (6 ms) 2023-10-09T09:23:43.552 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T09:23:43.559 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (7 ms) 2023-10-09T09:23:43.559 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T09:23:43.563 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_add (4 ms) 2023-10-09T09:23:43.563 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T09:23:43.568 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_remove (5 ms) 2023-10-09T09:23:43.568 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T09:23:43.571 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T09:23:43.571 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T09:23:43.577 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_clean (6 ms) 2023-10-09T09:23:43.577 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T09:23:43.577 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:43.580 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_add (4 ms) 2023-10-09T09:23:43.581 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T09:23:43.584 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T09:23:43.584 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T09:23:43.587 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T09:23:43.587 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T09:23:43.590 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-10-09T09:23:43.590 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T09:23:43.592 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T09:23:43.593 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T09:23:43.596 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2023-10-09T09:23:43.596 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T09:23:43.601 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (5 ms) 2023-10-09T09:23:43.601 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T09:23:43.603 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:43.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:43.655 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:43 smithi016 bash[75179]: cluster 2023-10-09T09:23:42.310659+0000 mgr.y (mgr.25421) 1397 : cluster 0 pgmap v1587: 105 pgs: 1 creating+activating, 104 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 698 KiB/s wr, 104 op/s 2023-10-09T09:23:43.655 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:43 smithi016 bash[77892]: cluster 2023-10-09T09:23:42.310659+0000 mgr.y (mgr.25421) 1397 : cluster 0 pgmap v1587: 105 pgs: 1 creating+activating, 104 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 698 KiB/s wr, 104 op/s 2023-10-09T09:23:43.663 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set (62 ms) 2023-10-09T09:23:43.663 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T09:23:43.669 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-10-09T09:23:43.669 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T09:23:43.692 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (24 ms) 2023-10-09T09:23:43.693 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T09:23:43.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:43.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:43.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:43.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:43 smithi094 bash[59677]: cluster 2023-10-09T09:23:42.310659+0000 mgr.y (mgr.25421) 1397 : cluster 0 pgmap v1587: 105 pgs: 1 creating+activating, 104 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 698 KiB/s wr, 104 op/s 2023-10-09T09:23:43.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:43.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:43.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:43.967 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (274 ms) 2023-10-09T09:23:43.967 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T09:23:43.973 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:43.974 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_remove (7 ms) 2023-10-09T09:23:43.974 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T09:23:43.977 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2023-10-09T09:23:43.977 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T09:23:43.986 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.trash_methods (10 ms) 2023-10-09T09:23:43.987 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T09:23:44.000 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.op_features (13 ms) 2023-10-09T09:23:44.000 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T09:23:44.022 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:44.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:44.094 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_parent (94 ms) 2023-10-09T09:23:44.094 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T09:23:44.109 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_parent_ns (15 ms) 2023-10-09T09:23:44.110 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T09:23:44.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:44.127 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_child (17 ms) 2023-10-09T09:23:44.127 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T09:23:44.138 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.namespace_methods (11 ms) 2023-10-09T09:23:44.138 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T09:23:44.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:44.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:44.154 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.migration (16 ms) 2023-10-09T09:23:44.154 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T09:23:44.165 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2023-10-09T09:23:44.165 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T09:23:44.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:44.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:44.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:44.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:44.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:44.513 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:44.539 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:44.642 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:44.676 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:44.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:44.779 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:44.805 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:44.909 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:44.933 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:44.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:45.036 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:45.067 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:45.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:45.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:45.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:45.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:45.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:45.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:45.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:45.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:45.384 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1219 ms) 2023-10-09T09:23:45.385 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T09:23:45.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:45.451 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.sparsify (68 ms) 2023-10-09T09:23:45.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:45.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:45.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:45.588 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:45.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:45.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:45.600 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[75179]: cluster 2023-10-09T09:23:44.311632+0000 mgr.y (mgr.25421) 1398 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 882 KiB/s wr, 977 op/s 2023-10-09T09:23:45.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[75179]: cluster 2023-10-09T09:23:44.373522+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T09:23:45.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[75179]: cluster 2023-10-09T09:23:44.375996+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T09:23:45.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[75179]: cluster 2023-10-09T09:23:44.380861+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T09:23:45.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[75179]: audit 2023-10-09T09:23:44.568218+0000 mon.a (mon.0) 1927 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:45.601 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[75179]: audit 2023-10-09T09:23:44.575043+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:45.692 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:45.704 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:45 smithi094 bash[59677]: cluster 2023-10-09T09:23:44.311632+0000 mgr.y (mgr.25421) 1398 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 882 KiB/s wr, 977 op/s 2023-10-09T09:23:45.705 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:45 smithi094 bash[59677]: cluster 2023-10-09T09:23:44.373522+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T09:23:45.705 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:45 smithi094 bash[59677]: cluster 2023-10-09T09:23:44.375996+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T09:23:45.705 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:45 smithi094 bash[59677]: cluster 2023-10-09T09:23:44.380861+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T09:23:45.705 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:45 smithi094 bash[59677]: audit 2023-10-09T09:23:44.568218+0000 mon.a (mon.0) 1927 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:45.705 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:45 smithi094 bash[59677]: audit 2023-10-09T09:23:44.575043+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:45.724 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:45.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:45.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:45.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[77892]: cluster 2023-10-09T09:23:44.311632+0000 mgr.y (mgr.25421) 1398 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 882 KiB/s wr, 977 op/s 2023-10-09T09:23:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[77892]: cluster 2023-10-09T09:23:44.373522+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T09:23:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[77892]: cluster 2023-10-09T09:23:44.375996+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T09:23:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[77892]: cluster 2023-10-09T09:23:44.380861+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T09:23:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[77892]: audit 2023-10-09T09:23:44.568218+0000 mon.a (mon.0) 1927 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:45.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[77892]: audit 2023-10-09T09:23:44.575043+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:45.828 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:45.858 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:45.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:45.961 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:45.997 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:46.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:23:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:23:45] "GET /metrics HTTP/1.1" 200 34043 "" "Prometheus/2.33.4" 2023-10-09T09:23:46.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:46.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:46.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:46.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:46.358 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:46.393 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:46.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:46.399 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] 70 tests from TestClsRbd (8162 ms total) 2023-10-09T09:23:46.399 INFO:tasks.workunit.client.0.smithi016.stdout: 2023-10-09T09:23:46.399 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] Global test environment tear-down 2023-10-09T09:23:46.400 INFO:tasks.workunit.client.0.smithi016.stdout:[==========] 70 tests from 1 test suite ran. (10770 ms total) 2023-10-09T09:23:46.400 INFO:tasks.workunit.client.0.smithi016.stdout:[ PASSED ] 70 tests. 2023-10-09T09:23:46.400 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T09:23:46.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T09:23:46.411 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T09:23:46.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:23:46.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:46.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:46.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:46.645 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:46 smithi094 bash[59677]: cluster 2023-10-09T09:23:45.381508+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:46 smithi094 bash[59677]: cluster 2023-10-09T09:23:45.382958+0000 mon.a (mon.0) 1929 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T09:23:46.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:46 smithi094 bash[59677]: cluster 2023-10-09T09:23:45.386109+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.646 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:46 smithi094 bash[59677]: cluster 2023-10-09T09:23:45.601847+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.664 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:46.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[77892]: cluster 2023-10-09T09:23:45.381508+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[77892]: cluster 2023-10-09T09:23:45.382958+0000 mon.a (mon.0) 1929 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T09:23:46.688 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[77892]: cluster 2023-10-09T09:23:45.386109+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.688 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[77892]: cluster 2023-10-09T09:23:45.601847+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.688 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[75179]: cluster 2023-10-09T09:23:45.381508+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.688 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[75179]: cluster 2023-10-09T09:23:45.382958+0000 mon.a (mon.0) 1929 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T09:23:46.688 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[75179]: cluster 2023-10-09T09:23:45.386109+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.688 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:46 smithi016 bash[75179]: cluster 2023-10-09T09:23:45.601847+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:46.689 DEBUG:teuthology.parallel:result is None 2023-10-09T09:23:46.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:23:46.699 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:23:46.699 DEBUG:teuthology.orchestra.run.smithi016:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:23:46.748 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:23:46.748 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:23:46.748 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T09:23:46.748 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T09:23:46.749 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T09:23:46.749 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T09:23:46.749 DEBUG:teuthology.orchestra.run.smithi016:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:23:46.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:46.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.798 INFO:teuthology.orchestra.run.smithi016.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T09:23:46.799 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:23:46.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:46.820 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T09:23:46.821 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T09:23:46.867 INFO:tasks.workunit:timeout=3h 2023-10-09T09:23:46.867 INFO:tasks.workunit:cleanup=True 2023-10-09T09:23:46.868 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:23:46.916 INFO:tasks.workunit.client.0.smithi016.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T09:23:46.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:46.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:46.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:47.050 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:47.082 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:47.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:47.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:47.222 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:47.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:47.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:47.325 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:47.359 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:47.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:47.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:47.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:47.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:47.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:47.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:47.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:47.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:47.712 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:47.724 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:47 smithi094 bash[59677]: cluster 2023-10-09T09:23:45.601688+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:47.724 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:47 smithi094 bash[59677]: cluster 2023-10-09T09:23:46.312464+0000 mgr.y (mgr.25421) 1399 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 891 KiB/s wr, 998 op/s 2023-10-09T09:23:47.724 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:47 smithi094 bash[59677]: cluster 2023-10-09T09:23:46.391908+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T09:23:47.724 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:47 smithi094 bash[59677]: cluster 2023-10-09T09:23:46.392004+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:47.745 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:47.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:47.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[77892]: cluster 2023-10-09T09:23:45.601688+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:47.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[77892]: cluster 2023-10-09T09:23:46.312464+0000 mgr.y (mgr.25421) 1399 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 891 KiB/s wr, 998 op/s 2023-10-09T09:23:47.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[77892]: cluster 2023-10-09T09:23:46.391908+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T09:23:47.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[77892]: cluster 2023-10-09T09:23:46.392004+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:47.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[75179]: cluster 2023-10-09T09:23:45.601688+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T09:23:47.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[75179]: cluster 2023-10-09T09:23:46.312464+0000 mgr.y (mgr.25421) 1399 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 891 KiB/s wr, 998 op/s 2023-10-09T09:23:47.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[75179]: cluster 2023-10-09T09:23:46.391908+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T09:23:47.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:47 smithi016 bash[75179]: cluster 2023-10-09T09:23:46.392004+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:47.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:47.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:47.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:47.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:47.981 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:48.010 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:48.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:48.114 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:48.147 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:48.251 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:48.284 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:48.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:48.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:48.514 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:48.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:48 smithi094 bash[59677]: cluster 2023-10-09T09:23:47.753131+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:48.525 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:48 smithi094 bash[59677]: cluster 2023-10-09T09:23:47.978849+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:48.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:48.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:48.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:48.659 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:48 smithi016 bash[75179]: cluster 2023-10-09T09:23:47.753131+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:48.659 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:48 smithi016 bash[75179]: cluster 2023-10-09T09:23:47.978849+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:48.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:48 smithi016 bash[77892]: cluster 2023-10-09T09:23:47.753131+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:48.660 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:48 smithi016 bash[77892]: cluster 2023-10-09T09:23:47.978849+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:48.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:48.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:48.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:48.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:48.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:48.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:48.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:49.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:49.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:49.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:49.137 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:23:49.137 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:23:49.137 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:23:49.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:49.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:49.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:49.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:49.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:49.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:49.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:49.425 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:49.428 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:23:49.429 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-10-09T09:23:49.429 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-10-09T09:23:49.429 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-10-09T09:23:49.429 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:23:49.453 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:49.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:49.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:49.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:49 smithi016 bash[75179]: cluster 2023-10-09T09:23:47.497113+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:49.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:49 smithi016 bash[75179]: cluster 2023-10-09T09:23:48.313266+0000 mgr.y (mgr.25421) 1400 : cluster 0 pgmap v1593: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:49.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:49 smithi016 bash[75179]: audit 2023-10-09T09:23:49.265405+0000 mon.a (mon.0) 1931 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:49.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:49 smithi016 bash[77892]: cluster 2023-10-09T09:23:47.497113+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:49.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:49 smithi016 bash[77892]: cluster 2023-10-09T09:23:48.313266+0000 mgr.y (mgr.25421) 1400 : cluster 0 pgmap v1593: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:49.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:49 smithi016 bash[77892]: audit 2023-10-09T09:23:49.265405+0000 mon.a (mon.0) 1931 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:49.556 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:49.586 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:49.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:49.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:49.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:49.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:49.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:49.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:49 smithi094 bash[59677]: cluster 2023-10-09T09:23:47.497113+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T09:23:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:49 smithi094 bash[59677]: cluster 2023-10-09T09:23:48.313266+0000 mgr.y (mgr.25421) 1400 : cluster 0 pgmap v1593: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:49 smithi094 bash[59677]: audit 2023-10-09T09:23:49.265405+0000 mon.a (mon.0) 1931 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:49.836 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:49.869 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:49.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:49.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:50.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:50.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:50.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:50.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:50.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:50.226 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:50.261 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:50.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:50.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:50.364 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:50.396 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:50.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:50.500 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:50.530 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:50.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:50.634 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:50.668 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:50.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:50.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:50.772 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:50.805 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:50.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:50.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:50.908 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:50.943 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:51.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:51.141 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:51.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:50 smithi094 bash[59677]: audit 2023-10-09T09:23:49.929639+0000 mon.a (mon.0) 1932 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:50 smithi094 bash[59677]: audit 2023-10-09T09:23:49.936542+0000 mon.a (mon.0) 1933 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:50 smithi094 bash[59677]: cluster 2023-10-09T09:23:50.313830+0000 mgr.y (mgr.25421) 1401 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:51.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:50 smithi094 bash[59677]: audit 2023-10-09T09:23:50.417046+0000 mon.a (mon.0) 1934 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:23:51.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:50 smithi094 bash[59677]: audit 2023-10-09T09:23:50.418144+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:23:51.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:50 smithi094 bash[59677]: audit 2023-10-09T09:23:50.424418+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.206 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:50 smithi094 bash[59677]: audit 2023-10-09T09:23:50.532414+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.225 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:51.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:51.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[75179]: audit 2023-10-09T09:23:49.929639+0000 mon.a (mon.0) 1932 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.522 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[75179]: audit 2023-10-09T09:23:49.936542+0000 mon.a (mon.0) 1933 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.522 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[75179]: cluster 2023-10-09T09:23:50.313830+0000 mgr.y (mgr.25421) 1401 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:51.522 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[75179]: audit 2023-10-09T09:23:50.417046+0000 mon.a (mon.0) 1934 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:23:51.522 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[75179]: audit 2023-10-09T09:23:50.418144+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:23:51.522 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[75179]: audit 2023-10-09T09:23:50.424418+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.522 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[75179]: audit 2023-10-09T09:23:50.532414+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[77892]: audit 2023-10-09T09:23:49.929639+0000 mon.a (mon.0) 1932 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.523 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[77892]: audit 2023-10-09T09:23:49.936542+0000 mon.a (mon.0) 1933 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.523 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[77892]: cluster 2023-10-09T09:23:50.313830+0000 mgr.y (mgr.25421) 1401 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:51.523 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[77892]: audit 2023-10-09T09:23:50.417046+0000 mon.a (mon.0) 1934 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:23:51.523 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[77892]: audit 2023-10-09T09:23:50.418144+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:23:51.523 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[77892]: audit 2023-10-09T09:23:50.424418+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.523 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:50 smithi016 bash[77892]: audit 2023-10-09T09:23:50.532414+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:23:51.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:51.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:51.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:51.942 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:51.975 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:51.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:52.079 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:52.110 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:52.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:52.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:52.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:52.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:52.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:52.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:52.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:52.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:52.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:52.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:52.550 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:52.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:52.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:52.654 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:52.685 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:52.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:52.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:52.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:52.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:52.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:52.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:52.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:52.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:52.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:53.041 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:53.071 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:53.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:53.079 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T09:23:53.086 INFO:tasks.workunit.client.0.smithi016.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T09:23:53.087 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:23:53.087 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T09:23:53.175 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:53.209 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:53.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:53.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:53.248 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T09:23:53.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:53.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:53.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:53.441 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:53.472 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T09:23:53.476 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:53.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:53.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:53.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:53.614 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:53.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:53.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:53.627 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:23:53.630 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:23:53.630 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T09:23:53.684 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:23:53.687 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-10-09T09:23:53.687 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T09:23:53.694 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T09:23:53.695 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T09:23:53.695 DEBUG:teuthology.orchestra.run.smithi016:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T09:23:53.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:53.730 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:53 smithi094 bash[59677]: cluster 2023-10-09T09:23:52.314411+0000 mgr.y (mgr.25421) 1402 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:53.746 INFO:tasks.workunit.client.0.smithi016.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T09:23:53.747 INFO:tasks.workunit.client.0.smithi016.stderr:+ TMPDIR=/tmp/rbd_import_export_240707 2023-10-09T09:23:53.747 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm -rf /tmp/rbd_import_export_240707 2023-10-09T09:23:53.747 INFO:tasks.workunit.client.0.smithi016.stderr:+ mkdir /tmp/rbd_import_export_240707 2023-10-09T09:23:53.748 INFO:tasks.workunit.client.0.smithi016.stderr:+ trap rm -rf /tmp/rbd_import_export_240707 INT TERM EXIT 2023-10-09T09:23:53.748 INFO:tasks.workunit.client.0.smithi016.stderr:+ mkdir foo.240707 2023-10-09T09:23:53.749 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:53.749 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import foo.240707 foo.dir 2023-10-09T09:23:53.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:53 smithi016 bash[75179]: cluster 2023-10-09T09:23:52.314411+0000 mgr.y (mgr.25421) 1402 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:53.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:53 smithi016 bash[77892]: cluster 2023-10-09T09:23:52.314411+0000 mgr.y (mgr.25421) 1402 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:53.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:53.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:53.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:53.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:53.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:53.982 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:54.013 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:54.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:23:54.117 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:54.149 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:54.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:54.253 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:54.286 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:23:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:23:54.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:54.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:54.515 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:54.537 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:23:54.549 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:54.563 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:23:54.587 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:23:54.612 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: cannot import a directory 2023-10-09T09:23:54.613 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 0% complete...failed. 2023-10-09T09:23:54.613 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: import failed: (21) Is a directory 2023-10-09T09:23:54.619 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:23:54.628 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-10-09T09:23:54.629 INFO:tasks.workunit.client.0.smithi016.stderr:+ rmdir foo.240707 2023-10-09T09:23:54.629 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_240707/img bs=1k count=1 seek=10 2023-10-09T09:23:54.630 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-10-09T09:23:54.630 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-10-09T09:23:54.630 INFO:tasks.workunit.client.0.smithi016.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 8.6232e-05 s, 11.9 MB/s 2023-10-09T09:23:54.630 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_240707/img bs=1k count=10 seek=100 2023-10-09T09:23:54.631 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records in 2023-10-09T09:23:54.631 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records out 2023-10-09T09:23:54.631 INFO:tasks.workunit.client.0.smithi016.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000215562 s, 47.5 MB/s 2023-10-09T09:23:54.631 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_240707/img bs=1k count=100 seek=1000 2023-10-09T09:23:54.633 INFO:tasks.workunit.client.0.smithi016.stderr:70+1 records in 2023-10-09T09:23:54.633 INFO:tasks.workunit.client.0.smithi016.stderr:70+1 records out 2023-10-09T09:23:54.633 INFO:tasks.workunit.client.0.smithi016.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000638378 s, 113 MB/s 2023-10-09T09:23:54.633 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_240707/img bs=1k seek=10000 2023-10-09T09:23:54.634 INFO:tasks.workunit.client.0.smithi016.stderr:138+1 records in 2023-10-09T09:23:54.634 INFO:tasks.workunit.client.0.smithi016.stderr:138+1 records out 2023-10-09T09:23:54.634 INFO:tasks.workunit.client.0.smithi016.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000488397 s, 291 MB/s 2023-10-09T09:23:54.634 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_240707/img bs=1k seek=100000 2023-10-09T09:23:54.647 INFO:tasks.workunit.client.0.smithi016.stderr:74+1 records in 2023-10-09T09:23:54.647 INFO:tasks.workunit.client.0.smithi016.stderr:74+1 records out 2023-10-09T09:23:54.648 INFO:tasks.workunit.client.0.smithi016.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.0124687 s, 6.1 MB/s 2023-10-09T09:23:54.648 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_240707/img bs=1k seek=1000000 2023-10-09T09:23:54.649 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:23:54.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:54.653 INFO:tasks.workunit.client.0.smithi016.stderr:194+1 records in 2023-10-09T09:23:54.653 INFO:tasks.workunit.client.0.smithi016.stderr:194+1 records out 2023-10-09T09:23:54.653 INFO:tasks.workunit.client.0.smithi016.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.00480416 s, 41.5 MB/s 2023-10-09T09:23:54.654 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-10-09T09:23:54.676 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:23:54.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:54.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:54.702 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:23:54.720 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 0% complete...failed. 2023-10-09T09:23:54.720 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T09:23:54.725 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-10-09T09:23:54.725 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format /tmp/rbd_import_export_240707/img testimg 2023-10-09T09:23:54.729 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:23:54.731 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.731 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.732 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:23:54.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:54.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:54.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:54.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:54.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:55.050 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:55.085 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:55.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:55.189 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:55.221 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:55.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:55.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:55.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:55.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:55.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:55.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:55.386 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-10-09T09:23:55.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:55.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:55.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:55.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:55.592 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[75179]: cluster 2023-10-09T09:23:54.314941+0000 mgr.y (mgr.25421) 1403 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:55.592 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[75179]: audit 2023-10-09T09:23:54.508388+0000 mon.a (mon.0) 1938 : audit 0 from='client.? 172.21.15.16:0/359505959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:55.592 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[75179]: audit 2023-10-09T09:23:54.572076+0000 mon.c (mon.2) 126 : audit 1 from='client.? 172.21.15.16:0/3217261434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:55.593 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[75179]: audit 2023-10-09T09:23:54.572658+0000 mon.a (mon.0) 1939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:55.593 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[77892]: cluster 2023-10-09T09:23:54.314941+0000 mgr.y (mgr.25421) 1403 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:55.593 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[77892]: audit 2023-10-09T09:23:54.508388+0000 mon.a (mon.0) 1938 : audit 0 from='client.? 172.21.15.16:0/359505959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:55.593 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[77892]: audit 2023-10-09T09:23:54.572076+0000 mon.c (mon.2) 126 : audit 1 from='client.? 172.21.15.16:0/3217261434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:55.593 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[77892]: audit 2023-10-09T09:23:54.572658+0000 mon.a (mon.0) 1939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:55.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:55.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:55.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:55.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:55.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:55 smithi094 bash[59677]: cluster 2023-10-09T09:23:54.314941+0000 mgr.y (mgr.25421) 1403 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:23:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:55 smithi094 bash[59677]: audit 2023-10-09T09:23:54.508388+0000 mon.a (mon.0) 1938 : audit 0 from='client.? 172.21.15.16:0/359505959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:23:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:55 smithi094 bash[59677]: audit 2023-10-09T09:23:54.572076+0000 mon.c (mon.2) 126 : audit 1 from='client.? 172.21.15.16:0/3217261434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:55 smithi094 bash[59677]: audit 2023-10-09T09:23:54.572658+0000 mon.a (mon.0) 1939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:23:55.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:55.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:55.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:55.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:55.878 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:23:55.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:55.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:23:55.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:55.996 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:23:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:23:55] "GET /metrics HTTP/1.1" 200 34047 "" "Prometheus/2.33.4" 2023-10-09T09:23:56.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:23:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:56.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:23:56.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:56.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:23:56.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:56.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:23:56.249 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:56.280 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:56.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:23:56.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:56.417 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:23:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:56.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:23:56.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:56.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:56.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:56.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:58.392 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[77892]: cluster 2023-10-09T09:23:55.370561+0000 mon.a (mon.0) 1940 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:23:58.858 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[77892]: cluster 2023-10-09T09:23:55.374196+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[77892]: cluster 2023-10-09T09:23:55.375293+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[77892]: audit 2023-10-09T09:23:55.375327+0000 mon.a (mon.0) 1941 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:23:58.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[77892]: cluster 2023-10-09T09:23:55.375496+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T09:23:58.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[77892]: cluster 2023-10-09T09:23:55.380726+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[77892]: cluster 2023-10-09T09:23:56.108307+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.859 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[75179]: cluster 2023-10-09T09:23:55.370561+0000 mon.a (mon.0) 1940 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:23:58.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[75179]: cluster 2023-10-09T09:23:55.374196+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[75179]: cluster 2023-10-09T09:23:55.375293+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[75179]: audit 2023-10-09T09:23:55.375327+0000 mon.a (mon.0) 1941 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:23:58.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[75179]: cluster 2023-10-09T09:23:55.375496+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T09:23:58.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[75179]: cluster 2023-10-09T09:23:55.380726+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:56 smithi016 bash[75179]: cluster 2023-10-09T09:23:56.108307+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.862 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 6, 5, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:23:58.862 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:23:58.863 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T09:23:58.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:23:58.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:56 smithi094 bash[59677]: cluster 2023-10-09T09:23:55.370561+0000 mon.a (mon.0) 1940 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:56 smithi094 bash[59677]: cluster 2023-10-09T09:23:55.374196+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:56 smithi094 bash[59677]: cluster 2023-10-09T09:23:55.375293+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:56 smithi094 bash[59677]: audit 2023-10-09T09:23:55.375327+0000 mon.a (mon.0) 1941 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:56 smithi094 bash[59677]: cluster 2023-10-09T09:23:55.375496+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:56 smithi094 bash[59677]: cluster 2023-10-09T09:23:55.380726+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:56 smithi094 bash[59677]: cluster 2023-10-09T09:23:56.108307+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:57 smithi094 bash[59677]: cluster 2023-10-09T09:23:56.315745+0000 mgr.y (mgr.25421) 1404 : cluster 0 pgmap v1598: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 103 B/s wr, 0 op/s 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:57 smithi094 bash[59677]: cluster 2023-10-09T09:23:56.393265+0000 mon.a (mon.0) 1943 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T09:23:58.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:57 smithi094 bash[59677]: cluster 2023-10-09T09:23:56.396346+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T09:23:58.871 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T09:23:58.964 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:23:58.964 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg /tmp/rbd_import_export_240707/img2 2023-10-09T09:23:58.965 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:23:58.965 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg - 2023-10-09T09:23:58.965 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:57 smithi016 bash[77892]: cluster 2023-10-09T09:23:56.315745+0000 mgr.y (mgr.25421) 1404 : cluster 0 pgmap v1598: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 103 B/s wr, 0 op/s 2023-10-09T09:23:58.965 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:57 smithi016 bash[77892]: cluster 2023-10-09T09:23:56.393265+0000 mon.a (mon.0) 1943 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T09:23:58.965 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:57 smithi016 bash[77892]: cluster 2023-10-09T09:23:56.396346+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T09:23:58.966 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:57 smithi016 bash[75179]: cluster 2023-10-09T09:23:56.315745+0000 mgr.y (mgr.25421) 1404 : cluster 0 pgmap v1598: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 103 B/s wr, 0 op/s 2023-10-09T09:23:58.966 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:57 smithi016 bash[75179]: cluster 2023-10-09T09:23:56.393265+0000 mon.a (mon.0) 1943 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T09:23:58.966 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:57 smithi016 bash[75179]: cluster 2023-10-09T09:23:56.396346+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T09:23:58.976 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:23:59.270 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-10-09T09:23:59.270 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T09:23:59.270 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_240707/img /tmp/rbd_import_export_240707/img2 2023-10-09T09:23:59.309 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_240707/img /tmp/rbd_import_export_240707/img3 2023-10-09T09:23:59.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:23:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:59.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:23:59.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:59.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:23:59 smithi094 bash[59677]: cluster 2023-10-09T09:23:58.316517+0000 mgr.y (mgr.25421) 1405 : cluster 0 pgmap v1600: 73 pgs: 73 active+clean; 1.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 25 KiB/s wr, 0 op/s 2023-10-09T09:23:59.745 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:23:59 smithi016 bash[75179]: cluster 2023-10-09T09:23:58.316517+0000 mgr.y (mgr.25421) 1405 : cluster 0 pgmap v1600: 73 pgs: 73 active+clean; 1.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 25 KiB/s wr, 0 op/s 2023-10-09T09:23:59.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:23:59 smithi016 bash[77892]: cluster 2023-10-09T09:23:58.316517+0000 mgr.y (mgr.25421) 1405 : cluster 0 pgmap v1600: 73 pgs: 73 active+clean; 1.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 25 KiB/s wr, 0 op/s 2023-10-09T09:23:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:59.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:23:59.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:23:59.813 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img2 /tmp/rbd_import_export_240707/img3 2023-10-09T09:23:59.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:59.919 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:23:59.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:23:59.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:00.026 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:00.072 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:00.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:00.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:00.147 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format - testimg 2023-10-09T09:24:00.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:00.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:00.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:00.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:00.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:00.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:00.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:00.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:00.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:00.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:00.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:00.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:00.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:00.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:00.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:00.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:00.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:00.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:00.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:00.793 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:00.795 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:00.800 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg /tmp/rbd_import_export_240707/img2 2023-10-09T09:24:00.850 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:00.879 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:00.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:00.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:00 smithi016 bash[77892]: cluster 2023-10-09T09:24:00.317168+0000 mgr.y (mgr.25421) 1406 : cluster 0 pgmap v1601: 73 pgs: 73 active+clean; 1.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 26 KiB/s wr, 11 op/s 2023-10-09T09:24:00.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:01.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:01.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:01.117 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:01.128 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:00 smithi094 bash[59677]: cluster 2023-10-09T09:24:00.317168+0000 mgr.y (mgr.25421) 1406 : cluster 0 pgmap v1601: 73 pgs: 73 active+clean; 1.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 26 KiB/s wr, 11 op/s 2023-10-09T09:24:01.146 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:01.217 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:01.224 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg - 2023-10-09T09:24:01.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:00 smithi016 bash[75179]: cluster 2023-10-09T09:24:00.317168+0000 mgr.y (mgr.25421) 1406 : cluster 0 pgmap v1601: 73 pgs: 73 active+clean; 1.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 26 KiB/s wr, 11 op/s 2023-10-09T09:24:01.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:01.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:01.384 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:01.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:01.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:01.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:01.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:01.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:01.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:24:01.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:01.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:01.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:01.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:01.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:01.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:01.965 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:01.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:02.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:02.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:24:02.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:02.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:02.197 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:02.223 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:02.226 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:02.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:02.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:02.230 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-10-09T09:24:02.329 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:02.361 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:02.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:02.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:02.378 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T09:24:02.383 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_240707/img /tmp/rbd_import_export_240707/img2 2023-10-09T09:24:02.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:02.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:02.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:02.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:02.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:02.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:02.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:02.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:02.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:02.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:02.865 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:02.895 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:02.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:02.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:02.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:03.034 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:03.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:03.045 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:03.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:03.139 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_240707/img /tmp/rbd_import_export_240707/img3 2023-10-09T09:24:03.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:03.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:03.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:03.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:03.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:03.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:03.541 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:03.575 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:03.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:03.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:03.687 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img /tmp/rbd_import_export_240707/img2 /tmp/rbd_import_export_240707/img3 2023-10-09T09:24:03.691 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:03 smithi016 bash[75179]: cluster 2023-10-09T09:24:02.317850+0000 mgr.y (mgr.25421) 1407 : cluster 0 pgmap v1602: 73 pgs: 73 active+clean; 4.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 123 KiB/s rd, 96 KiB/s wr, 66 op/s 2023-10-09T09:24:03.691 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:03 smithi016 bash[77892]: cluster 2023-10-09T09:24:02.317850+0000 mgr.y (mgr.25421) 1407 : cluster 0 pgmap v1602: 73 pgs: 73 active+clean; 4.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 123 KiB/s rd, 96 KiB/s wr, 66 op/s 2023-10-09T09:24:03.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:03.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:03 smithi094 bash[59677]: cluster 2023-10-09T09:24:02.317850+0000 mgr.y (mgr.25421) 1407 : cluster 0 pgmap v1602: 73 pgs: 73 active+clean; 4.3 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 123 KiB/s rd, 96 KiB/s wr, 66 op/s 2023-10-09T09:24:03.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:03.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:03.952 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:03.985 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:03.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:03.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:04.064 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:04.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:04.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:04.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:04.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:04.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:04.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:04.350 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:04.386 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:04.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:04.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:04.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:04.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:04.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:04.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:04.623 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:04.635 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:04 smithi094 bash[59677]: audit 2023-10-09T09:24:04.265732+0000 mon.a (mon.0) 1944 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:04.656 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:04.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:04.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:04 smithi016 bash[75179]: audit 2023-10-09T09:24:04.265732+0000 mon.a (mon.0) 1944 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:04.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:04 smithi016 bash[77892]: audit 2023-10-09T09:24:04.265732+0000 mon.a (mon.0) 1944 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:04.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:04.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:04.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:04.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:04.897 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:04.925 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:04.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:05.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:05.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:05.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:05.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:05.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:05.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:05.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:05.291 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:05.324 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:05.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:05.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:05.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:05.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:05.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:05.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:05.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:05.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:05.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:05.696 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:05.708 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:05 smithi094 bash[59677]: cluster 2023-10-09T09:24:04.318821+0000 mgr.y (mgr.25421) 1408 : cluster 0 pgmap v1603: 73 pgs: 73 active+clean; 3.5 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 216 KiB/s rd, 117 KiB/s wr, 63 op/s 2023-10-09T09:24:05.708 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:05 smithi094 bash[59677]: cluster 2023-10-09T09:24:04.906097+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:05.709 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:05 smithi094 bash[59677]: cluster 2023-10-09T09:24:04.909298+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T09:24:05.726 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:05.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:05.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:05.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:05 smithi016 bash[75179]: cluster 2023-10-09T09:24:04.318821+0000 mgr.y (mgr.25421) 1408 : cluster 0 pgmap v1603: 73 pgs: 73 active+clean; 3.5 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 216 KiB/s rd, 117 KiB/s wr, 63 op/s 2023-10-09T09:24:05.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:05 smithi016 bash[75179]: cluster 2023-10-09T09:24:04.906097+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:05.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:05 smithi016 bash[75179]: cluster 2023-10-09T09:24:04.909298+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T09:24:05.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:05 smithi016 bash[77892]: cluster 2023-10-09T09:24:04.318821+0000 mgr.y (mgr.25421) 1408 : cluster 0 pgmap v1603: 73 pgs: 73 active+clean; 3.5 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 216 KiB/s rd, 117 KiB/s wr, 63 op/s 2023-10-09T09:24:05.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:05 smithi016 bash[77892]: cluster 2023-10-09T09:24:04.906097+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:05.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:05 smithi016 bash[77892]: cluster 2023-10-09T09:24:04.909298+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T09:24:05.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:05.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:05.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:05.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:05.964 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:05.996 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:24:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:24:05] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.33.4" 2023-10-09T09:24:06.092 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:06.093 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:06.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:06.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:06.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:06.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:06.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:06.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:06.624 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:06.652 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:06.654 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:06 smithi094 bash[59677]: cluster 2023-10-09T09:24:05.351412+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.654 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:06 smithi094 bash[59677]: cluster 2023-10-09T09:24:05.354332+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.654 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:06 smithi094 bash[59677]: cluster 2023-10-09T09:24:06.222678+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.654 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:06 smithi094 bash[59677]: cluster 2023-10-09T09:24:06.225378+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[77892]: cluster 2023-10-09T09:24:05.351412+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[77892]: cluster 2023-10-09T09:24:05.354332+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[77892]: cluster 2023-10-09T09:24:06.222678+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[77892]: cluster 2023-10-09T09:24:06.225378+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[75179]: cluster 2023-10-09T09:24:05.351412+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[75179]: cluster 2023-10-09T09:24:05.354332+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[75179]: cluster 2023-10-09T09:24:06.222678+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:06 smithi016 bash[75179]: cluster 2023-10-09T09:24:06.225378+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T09:24:06.756 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:06.787 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:06.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:06.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:06.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:07.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:07.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:07.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:07.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:07.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:07.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:07.296 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:07.328 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:07.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:07.433 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:07.468 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:07.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:07.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:07.501 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:07 smithi016 bash[75179]: cluster 2023-10-09T09:24:06.319645+0000 mgr.y (mgr.25421) 1409 : cluster 0 pgmap v1605: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 258 KiB/s rd, 106 KiB/s wr, 80 op/s 2023-10-09T09:24:07.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:07 smithi016 bash[77892]: cluster 2023-10-09T09:24:06.319645+0000 mgr.y (mgr.25421) 1409 : cluster 0 pgmap v1605: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 258 KiB/s rd, 106 KiB/s wr, 80 op/s 2023-10-09T09:24:07.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:07.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:07.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:07.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:07.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:07.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:07.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:07.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:07.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:07 smithi094 bash[59677]: cluster 2023-10-09T09:24:06.319645+0000 mgr.y (mgr.25421) 1409 : cluster 0 pgmap v1605: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 258 KiB/s rd, 106 KiB/s wr, 80 op/s 2023-10-09T09:24:07.839 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:07.873 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:07.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:07.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:07.977 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:08.009 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:08.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:08.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:08.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:08.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:08.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:08.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:08.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:08.413 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:08.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:08.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:08.547 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:08.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:08.687 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:08.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:08.792 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:08.821 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:08.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:08.925 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:08.951 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:08.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:08.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:09.054 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:09.089 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:09.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:09.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:09.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:09.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:09.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:09.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:09.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:09.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:09.435 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd help+ export 2023-10-09T09:24:09.435 INFO:tasks.workunit.client.0.smithi016.stderr:grep -q export-format 2023-10-09T09:24:09.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:09.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:09 smithi016 bash[75179]: cluster 2023-10-09T09:24:08.320678+0000 mgr.y (mgr.25421) 1410 : cluster 0 pgmap v1606: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 256 KiB/s rd, 86 KiB/s wr, 79 op/s 2023-10-09T09:24:09.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:09 smithi016 bash[77892]: cluster 2023-10-09T09:24:08.320678+0000 mgr.y (mgr.25421) 1410 : cluster 0 pgmap v1606: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 256 KiB/s rd, 86 KiB/s wr, 79 op/s 2023-10-09T09:24:09.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:09.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:09.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:09.547 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_240707/img bs=1k count=10 seek=100 2023-10-09T09:24:09.551 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records in 2023-10-09T09:24:09.552 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records out 2023-10-09T09:24:09.552 INFO:tasks.workunit.client.0.smithi016.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000223034 s, 45.9 MB/s 2023-10-09T09:24:09.552 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format /tmp/rbd_import_export_240707/img testimg 2023-10-09T09:24:09.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:09.610 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:09.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:09.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:09.623 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:09.624 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:09.627 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap create testimg@snap 2023-10-09T09:24:09.713 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:09.725 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:09 smithi094 bash[59677]: cluster 2023-10-09T09:24:08.320678+0000 mgr.y (mgr.25421) 1410 : cluster 0 pgmap v1606: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 256 KiB/s rd, 86 KiB/s wr, 79 op/s 2023-10-09T09:24:09.744 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:09.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:09.848 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:09.878 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:09.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:09.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:09.983 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:10.016 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:10.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:10.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:10.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:10.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:10.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:10.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:10.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:10.420 INFO:tasks.workunit.client.0.smithi016.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T09:24:10.430 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T09:24:10.495 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd image-meta list testimg 2023-10-09T09:24:10.511 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:10.543 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:10.544 INFO:tasks.workunit.client.0.smithi016.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-09T09:24:10.544 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-10-09T09:24:10.544 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-10-09T09:24:10.544 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 2023-10-09T09:24:10.545 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:10.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:10.591 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:10.594 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_240707/img_v2 testimg_import 2023-10-09T09:24:10.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:10.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:10.780 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:10.811 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:10.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:10.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:10.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:11.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:11.082 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:11.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:11.186 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:11.217 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:11.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:11.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:11.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:11.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:11.438 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:24:11.443 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-10-09T09:24:11.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:11.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:11.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:11.492 INFO:tasks.workunit.client.0.smithi016.stdout:rbd image 'testimg_import': 2023-10-09T09:24:11.492 INFO:tasks.workunit.client.0.smithi016.stdout: size 110 KiB in 1 objects 2023-10-09T09:24:11.492 INFO:tasks.workunit.client.0.smithi016.stdout: order 22 (4 MiB objects) 2023-10-09T09:24:11.492 INFO:tasks.workunit.client.0.smithi016.stdout: snapshot_count: 1 2023-10-09T09:24:11.493 INFO:tasks.workunit.client.0.smithi016.stdout: id: 8d33d8220ce3 2023-10-09T09:24:11.493 INFO:tasks.workunit.client.0.smithi016.stdout: block_name_prefix: rbd_data.8d33d8220ce3 2023-10-09T09:24:11.493 INFO:tasks.workunit.client.0.smithi016.stdout: format: 2 2023-10-09T09:24:11.493 INFO:tasks.workunit.client.0.smithi016.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T09:24:11.493 INFO:tasks.workunit.client.0.smithi016.stdout: op_features: 2023-10-09T09:24:11.494 INFO:tasks.workunit.client.0.smithi016.stdout: flags: 2023-10-09T09:24:11.494 INFO:tasks.workunit.client.0.smithi016.stdout: create_timestamp: Mon Oct 9 09:24:10 2023 2023-10-09T09:24:11.494 INFO:tasks.workunit.client.0.smithi016.stdout: access_timestamp: Mon Oct 9 09:24:10 2023 2023-10-09T09:24:11.494 INFO:tasks.workunit.client.0.smithi016.stdout: modify_timestamp: Mon Oct 9 09:24:10 2023 2023-10-09T09:24:11.498 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import@snap 2023-10-09T09:24:11.546 INFO:tasks.workunit.client.0.smithi016.stdout:rbd image 'testimg_import': 2023-10-09T09:24:11.546 INFO:tasks.workunit.client.0.smithi016.stdout: size 110 KiB in 1 objects 2023-10-09T09:24:11.546 INFO:tasks.workunit.client.0.smithi016.stdout: order 22 (4 MiB objects) 2023-10-09T09:24:11.546 INFO:tasks.workunit.client.0.smithi016.stdout: snapshot_count: 1 2023-10-09T09:24:11.546 INFO:tasks.workunit.client.0.smithi016.stdout: id: 8d33d8220ce3 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: block_name_prefix: rbd_data.8d33d8220ce3 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: format: 2 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: op_features: 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: flags: 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: create_timestamp: Mon Oct 9 09:24:10 2023 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: access_timestamp: Mon Oct 9 09:24:10 2023 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: modify_timestamp: Mon Oct 9 09:24:10 2023 2023-10-09T09:24:11.547 INFO:tasks.workunit.client.0.smithi016.stdout: protected: False 2023-10-09T09:24:11.551 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd image-meta list testimg_import 2023-10-09T09:24:11.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:11.595 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[75179]: cluster 2023-10-09T09:24:10.321542+0000 mgr.y (mgr.25421) 1411 : cluster 0 pgmap v1607: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 251 KiB/s rd, 84 KiB/s wr, 71 op/s 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[75179]: cluster 2023-10-09T09:24:10.402489+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[75179]: cluster 2023-10-09T09:24:10.402849+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[75179]: cluster 2023-10-09T09:24:10.414690+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[75179]: cluster 2023-10-09T09:24:10.414715+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[77892]: cluster 2023-10-09T09:24:10.321542+0000 mgr.y (mgr.25421) 1411 : cluster 0 pgmap v1607: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 251 KiB/s rd, 84 KiB/s wr, 71 op/s 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[77892]: cluster 2023-10-09T09:24:10.402489+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[77892]: cluster 2023-10-09T09:24:10.402849+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[77892]: cluster 2023-10-09T09:24:10.414690+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.596 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:11 smithi016 bash[77892]: cluster 2023-10-09T09:24:10.414715+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.608 INFO:tasks.workunit.client.0.smithi016.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-09T09:24:11.608 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-10-09T09:24:11.608 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-10-09T09:24:11.608 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 2023-10-09T09:24:11.608 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ There is 1 metadatum on this image: 2023-10-09T09:24:11.608 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-10-09T09:24:11.608 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-10-09T09:24:11.609 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-09T09:24:11.609 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-10-09T09:24:11.609 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-10-09T09:24:11.609 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 ] 2023-10-09T09:24:11.609 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg_import /tmp/rbd_import_export_240707/img_import 2023-10-09T09:24:11.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:11.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:11.652 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-10-09T09:24:11.658 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_240707/img /tmp/rbd_import_export_240707/img_import 2023-10-09T09:24:11.658 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_240707/img /tmp/rbd_import_export_240707/img_import 2023-10-09T09:24:11.659 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/img --format %b 2023-10-09T09:24:11.674 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=24 2023-10-09T09:24:11.674 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/img_import --format %b 2023-10-09T09:24:11.675 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=224 2023-10-09T09:24:11.675 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=200 2023-10-09T09:24:11.676 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=200 2023-10-09T09:24:11.676 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T09:24:11.676 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_240707/img_snap 2023-10-09T09:24:11.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:11.728 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-10-09T09:24:11.732 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_240707/img_snap_import 2023-10-09T09:24:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:11 smithi094 bash[59677]: cluster 2023-10-09T09:24:10.321542+0000 mgr.y (mgr.25421) 1411 : cluster 0 pgmap v1607: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 251 KiB/s rd, 84 KiB/s wr, 71 op/s 2023-10-09T09:24:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:11 smithi094 bash[59677]: cluster 2023-10-09T09:24:10.402489+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:11 smithi094 bash[59677]: cluster 2023-10-09T09:24:10.402849+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T09:24:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:11 smithi094 bash[59677]: cluster 2023-10-09T09:24:10.414690+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:11 smithi094 bash[59677]: cluster 2023-10-09T09:24:10.414715+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T09:24:11.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:11.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:11.779 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-10-09T09:24:11.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_240707/img_snap /tmp/rbd_import_export_240707/img_snap_import 2023-10-09T09:24:11.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_240707/img_snap /tmp/rbd_import_export_240707/img_snap_import 2023-10-09T09:24:11.785 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/img_snap --format %b 2023-10-09T09:24:11.787 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=224 2023-10-09T09:24:11.787 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/img_snap_import --format %b 2023-10-09T09:24:11.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=224 2023-10-09T09:24:11.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:11.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:11.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:24:11.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:11.789 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img_import 2023-10-09T09:24:11.790 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img_snap 2023-10-09T09:24:11.791 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img_snap_import 2023-10-09T09:24:11.794 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T09:24:11.847 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:11.877 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:11.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:11.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:11.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:12.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:12.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:12.111 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:12.143 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:12.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:12.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:12.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:12.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:12.429 INFO:tasks.workunit.client.0.smithi016.stderr: Removing snap: 100% complete...done. 2023-10-09T09:24:12.439 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-10-09T09:24:12.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:12.510 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[75179]: cluster 2023-10-09T09:24:11.411180+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.510 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[75179]: cluster 2023-10-09T09:24:11.411432+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[75179]: cluster 2023-10-09T09:24:11.412869+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[75179]: cluster 2023-10-09T09:24:11.419872+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[75179]: cluster 2023-10-09T09:24:11.424413+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[77892]: cluster 2023-10-09T09:24:11.411180+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[77892]: cluster 2023-10-09T09:24:11.411432+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[77892]: cluster 2023-10-09T09:24:11.412869+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[77892]: cluster 2023-10-09T09:24:11.419872+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:12 smithi016 bash[77892]: cluster 2023-10-09T09:24:11.424413+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:12.571 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:12.576 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap rm testimg@snap 2023-10-09T09:24:12.632 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:12.661 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:12 smithi094 bash[59677]: cluster 2023-10-09T09:24:11.411180+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:12 smithi094 bash[59677]: cluster 2023-10-09T09:24:11.411432+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:12 smithi094 bash[59677]: cluster 2023-10-09T09:24:11.412869+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T09:24:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:12 smithi094 bash[59677]: cluster 2023-10-09T09:24:11.419872+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:12 smithi094 bash[59677]: cluster 2023-10-09T09:24:11.424413+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T09:24:12.764 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:12.795 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:12.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:12.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:12.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:12.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:12.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:13.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:13.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:13.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:13.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:13.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:13.411 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:13.438 INFO:tasks.workunit.client.0.smithi016.stderr: Removing snap: 100% complete...done. 2023-10-09T09:24:13.444 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:13.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:13.447 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-10-09T09:24:13.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:13.587 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:13.592 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --order 20 /tmp/rbd_import_export_240707/img testimg 2023-10-09T09:24:13.672 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:13.672 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:24:13.675 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:13.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:13.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[77892]: cluster 2023-10-09T09:24:12.322073+0000 mgr.y (mgr.25421) 1412 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 90 KiB/s rd, 3.4 KiB/s wr, 37 op/s 2023-10-09T09:24:13.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[77892]: cluster 2023-10-09T09:24:12.419435+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[77892]: cluster 2023-10-09T09:24:12.419824+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[77892]: cluster 2023-10-09T09:24:12.420811+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[77892]: cluster 2023-10-09T09:24:12.427137+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[77892]: cluster 2023-10-09T09:24:12.427257+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[75179]: cluster 2023-10-09T09:24:12.322073+0000 mgr.y (mgr.25421) 1412 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 90 KiB/s rd, 3.4 KiB/s wr, 37 op/s 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[75179]: cluster 2023-10-09T09:24:12.419435+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[75179]: cluster 2023-10-09T09:24:12.419824+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[75179]: cluster 2023-10-09T09:24:12.420811+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[75179]: cluster 2023-10-09T09:24:12.427137+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.687 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:13 smithi016 bash[75179]: cluster 2023-10-09T09:24:12.427257+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:13.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:13.730 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:13.734 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_240707/img_v2 testimg_import 2023-10-09T09:24:13.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:13 smithi094 bash[59677]: cluster 2023-10-09T09:24:12.322073+0000 mgr.y (mgr.25421) 1412 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 90 KiB/s rd, 3.4 KiB/s wr, 37 op/s 2023-10-09T09:24:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:13 smithi094 bash[59677]: cluster 2023-10-09T09:24:12.419435+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:13 smithi094 bash[59677]: cluster 2023-10-09T09:24:12.419824+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:13 smithi094 bash[59677]: cluster 2023-10-09T09:24:12.420811+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T09:24:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:13 smithi094 bash[59677]: cluster 2023-10-09T09:24:12.427137+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:13 smithi094 bash[59677]: cluster 2023-10-09T09:24:12.427257+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T09:24:13.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.818 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:24:13.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:13.825 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-10-09T09:24:13.825 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep order 2023-10-09T09:24:13.825 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $2} 2023-10-09T09:24:13.825 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep 20 2023-10-09T09:24:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:13.927 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:13.960 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:13.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:13.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:14.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:14.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:14.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:14.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:14.192 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:14.225 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:14.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:14.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:14.316 INFO:tasks.workunit.client.0.smithi016.stdout:20 2023-10-09T09:24:14.316 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:14.317 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-10-09T09:24:14.330 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:14.365 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:14.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:14.466 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:14.468 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:14.471 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-10-09T09:24:14.480 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:14 smithi094 bash[59677]: cluster 2023-10-09T09:24:13.421932+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.480 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:14 smithi094 bash[59677]: cluster 2023-10-09T09:24:13.422112+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.480 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:14 smithi094 bash[59677]: cluster 2023-10-09T09:24:13.424681+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T09:24:14.480 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:14 smithi094 bash[59677]: cluster 2023-10-09T09:24:13.562339+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.499 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[75179]: cluster 2023-10-09T09:24:13.421932+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[75179]: cluster 2023-10-09T09:24:13.422112+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[75179]: cluster 2023-10-09T09:24:13.424681+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[75179]: cluster 2023-10-09T09:24:13.562339+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[77892]: cluster 2023-10-09T09:24:13.421932+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[77892]: cluster 2023-10-09T09:24:13.422112+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[77892]: cluster 2023-10-09T09:24:13.424681+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T09:24:14.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:14 smithi016 bash[77892]: cluster 2023-10-09T09:24:13.562339+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T09:24:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:14.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:14.604 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:14.614 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:14.618 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_240707/img testimg 2023-10-09T09:24:14.624 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:14.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:14.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:14.684 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:14.689 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg 2023-10-09T09:24:14.689 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep features 2023-10-09T09:24:14.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:14.738 INFO:tasks.workunit.client.0.smithi016.stderr:+ FEATURES_BEFORE= features: layering 2023-10-09T09:24:14.738 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: 2023-10-09T09:24:14.738 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:14.753 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:14.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:14.784 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:14.789 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_240707/img_v2 testimg_import 2023-10-09T09:24:14.856 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:24:14.857 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:14.862 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-10-09T09:24:14.862 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep features 2023-10-09T09:24:14.884 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:14.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:14.906 INFO:tasks.workunit.client.0.smithi016.stderr:+ FEATURES_AFTER= features: layering 2023-10-09T09:24:14.906 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: 2023-10-09T09:24:14.906 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ features: layering 2023-10-09T09:24:14.906 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: != features: layering 2023-10-09T09:24:14.906 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: ] 2023-10-09T09:24:14.906 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:14.907 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-10-09T09:24:14.987 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:15.009 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:15.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-10-09T09:24:15.017 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:15.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:15.110 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:15.115 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_240707/img testimg 2023-10-09T09:24:15.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:15.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:15.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:15.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:15.196 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:15.200 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:15.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:15.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:15.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:15.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:15.289 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:15.292 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_240707/img_v2 testimg_import 2023-10-09T09:24:15.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:15.379 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:24:15.383 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-10-09T09:24:15.383 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep stripe unit 2023-10-09T09:24:15.383 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep -Ei (4 KiB|4096) 2023-10-09T09:24:15.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:15.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:15.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:15.432 INFO:tasks.workunit.client.0.smithi016.stdout: stripe unit: 4 KiB 2023-10-09T09:24:15.433 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-10-09T09:24:15.433 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep stripe count 2023-10-09T09:24:15.433 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $3} 2023-10-09T09:24:15.433 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep 1000 2023-10-09T09:24:15.484 INFO:tasks.workunit.client.0.smithi016.stdout:1000 2023-10-09T09:24:15.485 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/img_v2 2023-10-09T09:24:15.493 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:15.525 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:15.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:15.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:15.542 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-10-09T09:24:15.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:16.090 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:15 smithi094 bash[59677]: cluster 2023-10-09T09:24:14.322564+0000 mgr.y (mgr.25421) 1413 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 154 KiB/s rd, 12 KiB/s wr, 194 op/s 2023-10-09T09:24:16.092 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T09:24:16.092 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-10-09T09:24:16.092 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:15 smithi016 bash[75179]: cluster 2023-10-09T09:24:14.322564+0000 mgr.y (mgr.25421) 1413 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 154 KiB/s rd, 12 KiB/s wr, 194 op/s 2023-10-09T09:24:16.092 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:15 smithi016 bash[77892]: cluster 2023-10-09T09:24:14.322564+0000 mgr.y (mgr.25421) 1413 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 154 KiB/s rd, 12 KiB/s wr, 194 op/s 2023-10-09T09:24:16.094 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T09:24:16.094 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_240707/img testimg 2023-10-09T09:24:16.094 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:16.094 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap create testimg@snap1 2023-10-09T09:24:16.094 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:24:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:24:15] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.33.4" 2023-10-09T09:24:16.120 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:16.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:16.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:16.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:16.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:16.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:16.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:16.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:16.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:16.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:16.458 INFO:tasks.workunit.client.0.smithi016.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T09:24:16.468 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap create testimg@snap2 2023-10-09T09:24:16.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:16.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:16.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:16.968 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:17.074 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:17.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:17.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:17.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:17.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:17.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:17.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:17.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:17.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:17.439 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:17.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[77892]: cluster 2023-10-09T09:24:16.323427+0000 mgr.y (mgr.25421) 1414 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 213 KiB/s rd, 25 KiB/s wr, 269 op/s 2023-10-09T09:24:17.618 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[77892]: cluster 2023-10-09T09:24:16.445691+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.618 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[77892]: cluster 2023-10-09T09:24:16.447750+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-09T09:24:17.618 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[77892]: cluster 2023-10-09T09:24:16.450962+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.618 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[77892]: cluster 2023-10-09T09:24:16.455994+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.619 INFO:tasks.workunit.client.0.smithi016.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T09:24:17.619 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap protect testimg@snap2 2023-10-09T09:24:17.619 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_240707/snap_protect 2023-10-09T09:24:17.619 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:17.619 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_240707/snap_protect testimg_import 2023-10-09T09:24:17.652 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:17.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:17 smithi094 bash[59677]: cluster 2023-10-09T09:24:16.323427+0000 mgr.y (mgr.25421) 1414 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 213 KiB/s rd, 25 KiB/s wr, 269 op/s 2023-10-09T09:24:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:17 smithi094 bash[59677]: cluster 2023-10-09T09:24:16.445691+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:17 smithi094 bash[59677]: cluster 2023-10-09T09:24:16.447750+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-09T09:24:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:17 smithi094 bash[59677]: cluster 2023-10-09T09:24:16.450962+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:17 smithi094 bash[59677]: cluster 2023-10-09T09:24:16.455994+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[75179]: cluster 2023-10-09T09:24:16.323427+0000 mgr.y (mgr.25421) 1414 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 213 KiB/s rd, 25 KiB/s wr, 269 op/s 2023-10-09T09:24:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[75179]: cluster 2023-10-09T09:24:16.445691+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[75179]: cluster 2023-10-09T09:24:16.447750+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-09T09:24:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[75179]: cluster 2023-10-09T09:24:16.450962+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:17 smithi016 bash[75179]: cluster 2023-10-09T09:24:16.455994+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:17.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:17.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:17.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:17.887 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:17.917 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:17.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:17.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:18.021 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:18.053 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:18.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:18.157 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:18.189 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:18.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:18.293 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:18.327 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:18.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:18.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:18.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:18.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:18.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:18.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:18.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:18.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:18.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[77892]: cluster 2023-10-09T09:24:16.508292+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:18.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[77892]: cluster 2023-10-09T09:24:17.452311+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-09T09:24:18.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[77892]: cluster 2023-10-09T09:24:17.455809+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[77892]: cluster 2023-10-09T09:24:17.455932+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[77892]: cluster 2023-10-09T09:24:17.459526+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[75179]: cluster 2023-10-09T09:24:16.508292+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:18.701 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[75179]: cluster 2023-10-09T09:24:17.452311+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-09T09:24:18.702 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[75179]: cluster 2023-10-09T09:24:17.455809+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.702 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[75179]: cluster 2023-10-09T09:24:17.455932+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.702 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:18 smithi016 bash[75179]: cluster 2023-10-09T09:24:17.459526+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:18 smithi094 bash[59677]: cluster 2023-10-09T09:24:16.508292+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T09:24:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:18 smithi094 bash[59677]: cluster 2023-10-09T09:24:17.452311+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-09T09:24:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:18 smithi094 bash[59677]: cluster 2023-10-09T09:24:17.455809+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:18 smithi094 bash[59677]: cluster 2023-10-09T09:24:17.455932+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:18 smithi094 bash[59677]: cluster 2023-10-09T09:24:17.459526+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T09:24:18.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:18.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:18.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:18.896 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:18.926 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:18.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:18.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:19.030 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:19.063 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:19.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:19.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:19.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:19.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:19.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:19.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:19.424 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:19.460 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:19.511 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:24:19.515 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import@snap1 2023-10-09T09:24:19.516 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep protected: False 2023-10-09T09:24:19.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:19.573 INFO:tasks.workunit.client.0.smithi016.stdout: protected: False 2023-10-09T09:24:19.573 INFO:tasks.workunit.client.0.smithi016.stderr:+ + rbd info testimg_import@snap2 2023-10-09T09:24:19.573 INFO:tasks.workunit.client.0.smithi016.stderr:grep protected: True 2023-10-09T09:24:19.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:19.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:19.629 INFO:tasks.workunit.client.0.smithi016.stdout: protected: True 2023-10-09T09:24:19.629 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/snap_protect 2023-10-09T09:24:19.629 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T09:24:19.692 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T09:24:19.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:19.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:19.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[75179]: cluster 2023-10-09T09:24:18.324232+0000 mgr.y (mgr.25421) 1415 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 257 KiB/s rd, 30 KiB/s wr, 339 op/s 2023-10-09T09:24:19.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[75179]: cluster 2023-10-09T09:24:18.464011+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T09:24:19.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[75179]: cluster 2023-10-09T09:24:18.467241+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-09T09:24:19.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[75179]: audit 2023-10-09T09:24:19.266253+0000 mon.a (mon.0) 1953 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:19.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[77892]: cluster 2023-10-09T09:24:18.324232+0000 mgr.y (mgr.25421) 1415 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 257 KiB/s rd, 30 KiB/s wr, 339 op/s 2023-10-09T09:24:19.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[77892]: cluster 2023-10-09T09:24:18.464011+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T09:24:19.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[77892]: cluster 2023-10-09T09:24:18.467241+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-09T09:24:19.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:19 smithi016 bash[77892]: audit 2023-10-09T09:24:19.266253+0000 mon.a (mon.0) 1953 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:19.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:19.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:19 smithi094 bash[59677]: cluster 2023-10-09T09:24:18.324232+0000 mgr.y (mgr.25421) 1415 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 257 KiB/s rd, 30 KiB/s wr, 339 op/s 2023-10-09T09:24:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:19 smithi094 bash[59677]: cluster 2023-10-09T09:24:18.464011+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T09:24:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:19 smithi094 bash[59677]: cluster 2023-10-09T09:24:18.467241+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-09T09:24:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:19 smithi094 bash[59677]: audit 2023-10-09T09:24:19.266253+0000 mon.a (mon.0) 1953 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:19.752 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap purge testimg 2023-10-09T09:24:19.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:19.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:19.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:19.982 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:19.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:20.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:20.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:20.216 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:20.248 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:20.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:20.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:20.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:20.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:20.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:18.475076+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T09:24:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.471018+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.471921+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-09T09:24:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.474623+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.910002+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.912653+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-09T09:24:20.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.916128+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.492 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:20 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.930358+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:20.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:20.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.614 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:18.475076+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T09:24:20.614 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.471018+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:20.614 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.471921+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-09T09:24:20.614 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.474623+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:20.614 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.910002+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.912653+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.916128+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.930358+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:18.475076+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.471018+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.471921+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.474623+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.910002+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.912653+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-09T09:24:20.615 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.916128+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.616 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:20 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.930358+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:20.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:20.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:20.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.758 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:20.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:20.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:20.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:20.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:20.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:20.930 INFO:tasks.workunit.client.0.smithi016.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T09:24:20.943 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap purge testimg_import 2023-10-09T09:24:20.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:21.018 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:21.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:21.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:21.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:21.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:21.359 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:24:21.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:21.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:21.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:21.397 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:24:21.435 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:24:21.466 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:24:21.494 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:21.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:24:21.521 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:21.532 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:24:21.534 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:24:21.535 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:24:21.562 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:24:21.592 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:24:21.594 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.595 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:24:21.625 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.471262+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:19.921657+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:20.324836+0000 mgr.y (mgr.25421) 1416 : cluster 0 pgmap v1621: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 191 KiB/s rd, 19 KiB/s wr, 270 op/s 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:20.918322+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:20.919509+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:20.928412+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:20.931209+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: cluster 2023-10-09T09:24:20.987974+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[75179]: audit 2023-10-09T09:24:21.337128+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.16:0/1997023007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.471262+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:19.921657+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:20.324836+0000 mgr.y (mgr.25421) 1416 : cluster 0 pgmap v1621: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 191 KiB/s rd, 19 KiB/s wr, 270 op/s 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:20.918322+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:20.919509+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:20.928412+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:20.931209+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: cluster 2023-10-09T09:24:20.987974+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.650 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:21 smithi016 bash[77892]: audit 2023-10-09T09:24:21.337128+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.16:0/1997023007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:24:21.656 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.471262+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:19.921657+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:20.324836+0000 mgr.y (mgr.25421) 1416 : cluster 0 pgmap v1621: 73 pgs: 73 active+clean; 700 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 191 KiB/s rd, 19 KiB/s wr, 270 op/s 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:20.918322+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:20.919509+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:20.928412+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:20.931209+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: cluster 2023-10-09T09:24:20.987974+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T09:24:21.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:21 smithi094 bash[59677]: audit 2023-10-09T09:24:21.337128+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.16:0/1997023007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:24:21.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:21.766 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169612 11.0 e326/294 11:0d8b54c8:::benchmark_data_smithi016_218422_object76474:head v 326'10741, mlcod=326'10739)", 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.730455+0000", 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.032245034, 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23246093500000001, 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.767 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730455+0000", 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730455+0000", 2023-10-09T09:24:21.768 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730454+0000", 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730463+0000", 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9239999999999996e-06 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730475+0000", 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1871e-05 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730483+0000", 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1680000000000004e-06 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730552+0000", 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8343000000000006e-05 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.730574+0000", 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1951e-05 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.962865+0000", 2023-10-09T09:24:21.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23229182900000001 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.962916+0000", 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0776999999999999e-05 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169615 11.0 e326/294 11:01554223:::benchmark_data_smithi016_218422_object76477:head v 326'10742, mlcod=326'10739)", 2023-10-09T09:24:21.772 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.736980+0000", 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.025720154, 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23734126999999999, 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.736980+0000", 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.773 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.736980+0000", 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.736979+0000", 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.736988+0000", 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8619999999999992e-06 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.736999+0000", 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1473e-05 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.737007+0000", 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7810000000000005e-06 2023-10-09T09:24:21.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.744190+0000", 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0071835110000000001 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.744213+0000", 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2773999999999999e-05 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974291+0000", 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.230077632 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974321+0000", 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0584999999999999e-05 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169623 11.8 e326/294 11:18d61715:::benchmark_data_smithi016_218422_object76485:head v 326'10520, mlcod=326'10518)", 2023-10-09T09:24:21.777 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.737097+0000", 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.025603182, 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23718899600000001, 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.737097+0000", 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.737097+0000", 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.737097+0000", 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.737100+0000", 2023-10-09T09:24:21.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.41e-06 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.737106+0000", 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.118e-06 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.737109+0000", 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9709999999999998e-06 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.740652+0000", 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035425040000000001 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.740672+0000", 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0285e-05 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974241+0000", 2023-10-09T09:24:21.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.233568835 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974286+0000", 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5077999999999998e-05 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.782 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169617 11.0 e326/294 11:0bc198be:::benchmark_data_smithi016_218422_object76479:head v 326'10743, mlcod=326'10739)", 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.743454+0000", 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.019246426, 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23088735799999999, 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.743454+0000", 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.743454+0000", 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.743452+0000", 2023-10-09T09:24:21.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.743462+0000", 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3589999999999997e-06 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.743474+0000", 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1822e-05 2023-10-09T09:24:21.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.743483+0000", 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1360000000000003e-06 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749254+0000", 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0057712390000000001 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749273+0000", 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8708000000000001e-05 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974324+0000", 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.225050899 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974341+0000", 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7655e-05 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169620 11.0 e326/294 11:00df34e6:::benchmark_data_smithi016_218422_object76482:head v 326'10744, mlcod=326'10739)", 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.747972+0000", 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.01472835300001, 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.226388532, 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.788 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.747972+0000", 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.747972+0000", 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.747971+0000", 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.747982+0000", 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1195999999999999e-05 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.747995+0000", 2023-10-09T09:24:21.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2866e-05 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748004+0000", 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.901e-06 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954625+0000", 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.206620738 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954698+0000", 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3181000000000003e-05 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974343+0000", 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.019645587999999999 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.974361+0000", 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7155000000000001e-05 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169628 11.5 e326/294 11:af1f3843:::benchmark_data_smithi016_218422_object76490:head v 326'10390, mlcod=326'10388)", 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.748672+0000", 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.01402874600001, 2023-10-09T09:24:21.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.206197203, 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748672+0000", 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748672+0000", 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748670+0000", 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748680+0000", 2023-10-09T09:24:21.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1560000000000001e-06 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748692+0000", 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2714999999999999e-05 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748700+0000", 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9810000000000003e-06 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.750505+0000", 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018042990000000001 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.750525+0000", 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0200999999999999e-05 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954723+0000", 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.204197931 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954869+0000", 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014605099999999999 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.798 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169627 11.9 e326/294 11:95e1ff51:::benchmark_data_smithi016_218422_object76489:head v 326'10738, mlcod=326'10736)", 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.748716+0000", 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.01398405200001, 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20594959400000001, 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748716+0000", 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748716+0000", 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748716+0000", 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748722+0000", 2023-10-09T09:24:21.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5199999999999997e-06 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748728+0000", 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2079999999999997e-06 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.748733+0000", 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2660000000000001e-06 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749625+0000", 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000892489 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749646+0000", 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0846999999999999e-05 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.758825+0000", 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0091790569999999991 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954666+0000", 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.19584052199999999 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.803 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169630 11.2 e326/309 11:4edc6ef2:::benchmark_data_smithi016_218422_object76492:head v 326'10608, mlcod=326'10606)", 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.749180+0000", 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.013520032, 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.205634185, 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749180+0000", 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749180+0000", 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749180+0000", 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749186+0000", 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4999999999999996e-06 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749197+0000", 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1096e-05 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749204+0000", 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7819999999999996e-06 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749239+0000", 2023-10-09T09:24:21.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4703000000000003e-05 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.749258+0000", 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9085000000000001e-05 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954736+0000", 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20547831999999999 2023-10-09T09:24:21.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954815+0000", 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8431999999999997e-05 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169629 11.b e326/294 11:d5cc9e0b:::benchmark_data_smithi016_218422_object76491:head v 326'10388, mlcod=326'10386)", 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.756197+0000", 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.00650339000001, 2023-10-09T09:24:21.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20289927399999999, 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.756197+0000", 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.809 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.756197+0000", 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.756196+0000", 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.756205+0000", 2023-10-09T09:24:21.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1109999999999997e-06 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954594+0000", 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.19838857300000001 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954610+0000", 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6252e-05 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954690+0000", 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0264000000000003e-05 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954709+0000", 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.853e-05 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.959050+0000", 2023-10-09T09:24:21.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043411760000000004 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.959096+0000", 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6372999999999998e-05 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:169636 11.5 e326/294 11:af06a717:::benchmark_data_smithi016_218422_object76498:head v 326'10391, mlcod=326'10388)", 2023-10-09T09:24:21.813 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:21:59.758369+0000", 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 142.00433131899999, 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20553606799999999, 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.758369+0000", 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.814 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.758369+0000", 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.758368+0000", 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.758375+0000", 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3350000000000003e-06 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.815 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954642+0000", 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.196267415 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.954652+0000", 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7410000000000004e-06 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.816 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.955011+0000", 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00035928300000000001 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.955046+0000", 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5397e-05 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.963871+0000", 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0088247259999999998 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.817 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:21:59.963905+0000", 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3905000000000001e-05 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171593 11.9 e327/294 11:9357d747:::benchmark_data_smithi016_218422_object78455:head v 327'10855, mlcod=327'10852)", 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.893260+0000", 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.86944013499999, 2023-10-09T09:24:21.818 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.468307113, 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.893260+0000", 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.893260+0000", 2023-10-09T09:24:21.819 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.893259+0000", 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.893271+0000", 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2714999999999999e-05 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.820 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.893286+0000", 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4351e-05 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.893295+0000", 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2429999999999994e-06 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.894499+0000", 2023-10-09T09:24:21.821 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001204028 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.894549+0000", 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0076e-05 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.361567+0000", 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.46701817800000001 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.822 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171596 11.9 e327/294 11:9905b267:::benchmark_data_smithi016_218422_object78458:head v 327'10856, mlcod=327'10852)", 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.897379+0000", 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.86532152699999, 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.46430855399999998, 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.823 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.897379+0000", 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.897379+0000", 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.897377+0000", 2023-10-09T09:24:21.824 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.897389+0000", 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2104e-05 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.897410+0000", 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1077000000000002e-05 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.825 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.897419+0000", 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3100000000000006e-06 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.899857+0000", 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024376549999999999 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.826 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.899914+0000", 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6969000000000003e-05 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.361687+0000", 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.46177332900000001 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.827 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171595 11.2 e327/309 11:472561ad:::benchmark_data_smithi016_218422_object78457:head v 327'10714, mlcod=327'10711)", 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.903329+0000", 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.85937171699999, 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.45075432700000001, 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903329+0000", 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.828 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903329+0000", 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903328+0000", 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903336+0000", 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9619999999999993e-06 2023-10-09T09:24:21.829 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903351+0000", 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4809e-05 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903359+0000", 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8459999999999997e-06 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.830 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903520+0000", 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000161108 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.903547+0000", 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6539000000000001e-05 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.354083+0000", 2023-10-09T09:24:21.831 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.45053593600000003 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171601 11.8 e327/294)", 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.914764+0000", 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.84793593099999, 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.429092474, 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.832 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914764+0000", 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914764+0000", 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.833 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914763+0000", 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914779+0000", 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6073e-05 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.834 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914795+0000", 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5865000000000001e-05 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914804+0000", 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0890000000000006e-06 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.343778+0000", 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.428973883 2023-10-09T09:24:21.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.343857+0000", 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8888999999999996e-05 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171598 11.2 e327/309)", 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.914818+0000", 2023-10-09T09:24:21.836 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.84788282, 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.43393184499999998, 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914818+0000", 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914818+0000", 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914817+0000", 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914822+0000", 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5379999999999996e-06 2023-10-09T09:24:21.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914828+0000", 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2330000000000003e-06 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.914833+0000", 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5279999999999997e-06 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.348727+0000", 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.433894897 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.348749+0000", 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1963999999999998e-05 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171604 11.2 e327/309)", 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.927357+0000", 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.835343752, 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.42159186599999998, 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927357+0000", 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927357+0000", 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927355+0000", 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927366+0000", 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.048e-05 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927385+0000", 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9315999999999999e-05 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927395+0000", 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4399999999999994e-06 2023-10-09T09:24:21.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.348934+0000", 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.42153923999999998 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.348948+0000", 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4517e-05 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171606 11.a e327/294 11:57183944:::benchmark_data_smithi016_218422_object78468:head v 327'10833, mlcod=327'10831)", 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.927418+0000", 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.83528263900001, 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.46257502499999997, 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927418+0000", 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927418+0000", 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927417+0000", 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927423+0000", 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5690000000000004e-06 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927430+0000", 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9809999999999997e-06 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927435+0000", 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9339999999999997e-06 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.348967+0000", 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.421531928 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.348989+0000", 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2599000000000002e-05 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.389878+0000", 2023-10-09T09:24:21.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.040888485000000002 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.389993+0000", 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011500799999999999 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171605 11.9 e327/294)", 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.927439+0000", 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.83526101499999, 2023-10-09T09:24:21.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.44744657399999999, 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927439+0000", 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927439+0000", 2023-10-09T09:24:21.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927439+0000", 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927443+0000", 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8970000000000001e-06 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927448+0000", 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7639999999999996e-06 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927452+0000", 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.941e-06 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.852 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.374860+0000", 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.44740775700000002 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.374886+0000", 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6412000000000001e-05 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171607 11.9 e327/294)", 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.927456+0000", 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.835244652, 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.447477293, 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927456+0000", 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927456+0000", 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927456+0000", 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927461+0000", 2023-10-09T09:24:21.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8170000000000004e-06 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927466+0000", 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6999999999999999e-06 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927470+0000", 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1930000000000003e-06 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.374916+0000", 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.447445802 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.374933+0000", 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6948000000000001e-05 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.35859.0:171603 11.0 e327/294)", 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-10-09T09:22:02.927606+0000", 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 138.83509420300001, 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.41636951, 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-10-09T09:24:21.858 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927606+0000", 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927606+0000", 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927606+0000", 2023-10-09T09:24:21.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927610+0000", 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5059999999999997e-06 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927622+0000", 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1479e-05 2023-10-09T09:24:21.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:02.927628+0000", 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2360000000000001e-06 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.343885+0000", 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.41625701399999998 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-10-09T09:24:21.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-10-09T09:22:03.343976+0000", 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0544999999999995e-05 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-10-09T09:24:21.862 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-10-09T09:24:21.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.863 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:21.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:21.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:21.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:21.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:22.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:23.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: audit 2023-10-09T09:24:21.519499+0000 mgr.y (mgr.25421) 1417 : audit 0 from='client.46265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:23.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: cluster 2023-10-09T09:24:21.926733+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: cluster 2023-10-09T09:24:21.927640+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-09T09:24:23.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: cluster 2023-10-09T09:24:21.930182+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: cluster 2023-10-09T09:24:21.942507+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.503 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: cluster 2023-10-09T09:24:21.952599+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: audit 2023-10-09T09:24:22.026123+0000 mon.c (mon.2) 128 : audit 1 from='client.? 172.21.15.16:0/1222384256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:23.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[75179]: audit 2023-10-09T09:24:22.026801+0000 mon.a (mon.0) 1958 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:23.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: audit 2023-10-09T09:24:21.519499+0000 mgr.y (mgr.25421) 1417 : audit 0 from='client.46265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:23.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: cluster 2023-10-09T09:24:21.926733+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: cluster 2023-10-09T09:24:21.927640+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-09T09:24:23.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: cluster 2023-10-09T09:24:21.930182+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: cluster 2023-10-09T09:24:21.942507+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: cluster 2023-10-09T09:24:21.952599+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: audit 2023-10-09T09:24:22.026123+0000 mon.c (mon.2) 128 : audit 1 from='client.? 172.21.15.16:0/1222384256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:23.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:22 smithi016 bash[77892]: audit 2023-10-09T09:24:22.026801+0000 mon.a (mon.0) 1958 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:24:23.506 INFO:tasks.workunit.client.0.smithi016.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T09:24:23.602 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-10-09T09:24:23.605 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:23.605 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-10-09T09:24:23.605 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:23.605 INFO:tasks.workunit.client.0.smithi016.stderr:+ tiered=0 2023-10-09T09:24:23.606 INFO:tasks.workunit.client.0.smithi016.stderr:+ ceph osd dump 2023-10-09T09:24:23.606 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep 'rbd' 2023-10-09T09:24:23.606 INFO:tasks.workunit.client.0.smithi016.stderr:grep ^pool 2023-10-09T09:24:23.606 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep tier 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: audit 2023-10-09T09:24:21.519499+0000 mgr.y (mgr.25421) 1417 : audit 0 from='client.46265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: cluster 2023-10-09T09:24:21.926733+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: cluster 2023-10-09T09:24:21.927640+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: cluster 2023-10-09T09:24:21.930182+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: cluster 2023-10-09T09:24:21.942507+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: cluster 2023-10-09T09:24:21.952599+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: audit 2023-10-09T09:24:22.026123+0000 mon.c (mon.2) 128 : audit 1 from='client.? 172.21.15.16:0/1222384256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:23.607 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:22 smithi094 bash[59677]: audit 2023-10-09T09:24:22.026801+0000 mon.a (mon.0) 1958 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:23.628 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:23.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:23 smithi094 bash[59677]: cluster 2023-10-09T09:24:22.325440+0000 mgr.y (mgr.25421) 1418 : cluster 0 pgmap v1624: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 163 op/s 2023-10-09T09:24:23.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:23 smithi094 bash[59677]: cluster 2023-10-09T09:24:22.925586+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:24:23.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:23 smithi094 bash[59677]: cluster 2023-10-09T09:24:22.934891+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:23 smithi094 bash[59677]: audit 2023-10-09T09:24:22.936018+0000 mon.a (mon.0) 1960 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:24:23.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:23 smithi094 bash[59677]: cluster 2023-10-09T09:24:22.936132+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-09T09:24:23.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:23 smithi094 bash[59677]: cluster 2023-10-09T09:24:22.939581+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:23 smithi094 bash[59677]: cluster 2023-10-09T09:24:22.952579+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[77892]: cluster 2023-10-09T09:24:22.325440+0000 mgr.y (mgr.25421) 1418 : cluster 0 pgmap v1624: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 163 op/s 2023-10-09T09:24:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[77892]: cluster 2023-10-09T09:24:22.925586+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:24:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[77892]: cluster 2023-10-09T09:24:22.934891+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[77892]: audit 2023-10-09T09:24:22.936018+0000 mon.a (mon.0) 1960 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[77892]: cluster 2023-10-09T09:24:22.936132+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[77892]: cluster 2023-10-09T09:24:22.939581+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[77892]: cluster 2023-10-09T09:24:22.952579+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[75179]: cluster 2023-10-09T09:24:22.325440+0000 mgr.y (mgr.25421) 1418 : cluster 0 pgmap v1624: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 163 op/s 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[75179]: cluster 2023-10-09T09:24:22.925586+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[75179]: cluster 2023-10-09T09:24:22.934891+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[75179]: audit 2023-10-09T09:24:22.936018+0000 mon.a (mon.0) 1960 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:24:23.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[75179]: cluster 2023-10-09T09:24:22.936132+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-09T09:24:23.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[75179]: cluster 2023-10-09T09:24:22.939581+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:23 smithi016 bash[75179]: cluster 2023-10-09T09:24:22.952579+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T09:24:23.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:24.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:24.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:24.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:24.216 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:24.250 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:24.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:24.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:24.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:24.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:24.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:24.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:24.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:24.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:24.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:24.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:24.835 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:24.865 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:24.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:24.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:24.969 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:25.000 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:25.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:25.104 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:25.137 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:25.240 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:25.269 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:25.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:25.373 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:25.404 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:25.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:25.441 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1622,"stamp":"2023-10-09T09:24:20.916977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":821327,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":3645,"num_read_kb":5361,"num_write":738,"num_write_kb":3753,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":2972810,"num_keys_recovered":70,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":743,"ondisk_log_size":743,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7829976,"kb_used_data":31984,"kb_used_omap":126,"kb_used_meta":7797761,"kb_avail":742131240,"statfs":{"total":767960285184,"available":759942389760,"internally_reserved":0,"allocated":32751616,"data_stored":24685656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":7984907899},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":104483,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":571,"num_read_kb":466,"num_write":61,"num_write_kb":39,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.469928"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":293,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473679+0000","last_change":"2023-10-09T09:20:19.894893+0000","last_active":"2023-10-09T09:24:18.473679+0000","last_peered":"2023-10-09T09:24:18.473679+0000","last_clean":"2023-10-09T09:24:18.473679+0000","last_became_active":"2023-10-09T09:20:19.893953+0000","last_became_peered":"2023-10-09T09:20:19.893953+0000","last_unstale":"2023-10-09T09:24:18.473679+0000","last_undegraded":"2023-10-09T09:24:18.473679+0000","last_fullsized":"2023-10-09T09:24:18.473679+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477390+0000","last_change":"2023-10-09T09:14:48.316760+0000","last_active":"2023-10-09T09:24:13.477390+0000","last_peered":"2023-10-09T09:24:13.477390+0000","last_clean":"2023-10-09T09:24:13.477390+0000","last_became_active":"2023-10-09T09:14:48.316432+0000","last_became_peered":"2023-10-09T09:14:48.316432+0000","last_unstale":"2023-10-09T09:24:13.477390+0000","last_undegraded":"2023-10-09T09:24:13.477390+0000","last_fullsized":"2023-10-09T09:24:13.477390+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:41.630374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":132,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477531+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:24:13.477531+0000","last_peered":"2023-10-09T09:24:13.477531+0000","last_clean":"2023-10-09T09:24:13.477531+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:24:13.477531+0000","last_undegraded":"2023-10-09T09:24:13.477531+0000","last_fullsized":"2023-10-09T09:24:13.477531+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":422,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456106+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:24:16.456106+0000","last_peered":"2023-10-09T09:24:16.456106+0000","last_clean":"2023-10-09T09:24:16.456106+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:24:16.456106+0000","last_undegraded":"2023-10-09T09:24:16.456106+0000","last_fullsized":"2023-10-09T09:24:16.456106+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":365,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.455982+0000","last_change":"2023-10-09T09:14:48.312713+0000","last_active":"2023-10-09T09:24:16.455982+0000","last_peered":"2023-10-09T09:24:16.455982+0000","last_clean":"2023-10-09T09:24:16.455982+0000","last_became_active":"2023-10-09T09:14:48.312429+0000","last_became_peered":"2023-10-09T09:14:48.312429+0000","last_unstale":"2023-10-09T09:24:16.455982+0000","last_undegraded":"2023-10-09T09:24:16.455982+0000","last_fullsized":"2023-10-09T09:24:16.455982+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:56.755105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":299,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.492168+0000","last_change":"2023-10-09T09:19:42.872407+0000","last_active":"2023-10-09T09:24:19.492168+0000","last_peered":"2023-10-09T09:24:19.492168+0000","last_clean":"2023-10-09T09:24:19.492168+0000","last_became_active":"2023-10-09T09:19:42.871637+0000","last_became_peered":"2023-10-09T09:19:42.871637+0000","last_unstale":"2023-10-09T09:24:19.492168+0000","last_undegraded":"2023-10-09T09:24:19.492168+0000","last_fullsized":"2023-10-09T09:24:19.492168+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":373,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456172+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:24:16.456172+0000","last_peered":"2023-10-09T09:24:16.456172+0000","last_clean":"2023-10-09T09:24:16.456172+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:24:16.456172+0000","last_undegraded":"2023-10-09T09:24:16.456172+0000","last_fullsized":"2023-10-09T09:24:16.456172+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":133,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477469+0000","last_change":"2023-10-09T09:14:48.306040+0000","last_active":"2023-10-09T09:24:13.477469+0000","last_peered":"2023-10-09T09:24:13.477469+0000","last_clean":"2023-10-09T09:24:13.477469+0000","last_became_active":"2023-10-09T09:14:48.304137+0000","last_became_peered":"2023-10-09T09:14:48.304137+0000","last_unstale":"2023-10-09T09:24:13.477469+0000","last_undegraded":"2023-10-09T09:24:13.477469+0000","last_fullsized":"2023-10-09T09:24:13.477469+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:47.446761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":191,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.478074+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:24:19.478074+0000","last_peered":"2023-10-09T09:24:19.478074+0000","last_clean":"2023-10-09T09:24:19.478074+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:24:19.478074+0000","last_undegraded":"2023-10-09T09:24:19.478074+0000","last_fullsized":"2023-10-09T09:24:19.478074+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":350,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.491810+0000","last_change":"2023-10-09T09:13:48.190068+0000","last_active":"2023-10-09T09:24:19.491810+0000","last_peered":"2023-10-09T09:24:19.491810+0000","last_clean":"2023-10-09T09:24:19.491810+0000","last_became_active":"2023-10-09T09:13:48.189740+0000","last_became_peered":"2023-10-09T09:13:48.189740+0000","last_unstale":"2023-10-09T09:24:19.491810+0000","last_undegraded":"2023-10-09T09:24:19.491810+0000","last_fullsized":"2023-10-09T09:24:19.491810+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":173,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.481108+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:24:19.481108+0000","last_peered":"2023-10-09T09:24:19.481108+0000","last_clean":"2023-10-09T09:24:19.481108+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:24:19.481108+0000","last_undegraded":"2023-10-09T09:24:19.481108+0000","last_fullsized":"2023-10-09T09:24:19.481108+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477730+0000","last_change":"2023-10-09T09:20:19.898526+0000","last_active":"2023-10-09T09:24:13.477730+0000","last_peered":"2023-10-09T09:24:13.477730+0000","last_clean":"2023-10-09T09:24:13.477730+0000","last_became_active":"2023-10-09T09:20:19.898197+0000","last_became_peered":"2023-10-09T09:20:19.898197+0000","last_unstale":"2023-10-09T09:24:13.477730+0000","last_undegraded":"2023-10-09T09:24:13.477730+0000","last_fullsized":"2023-10-09T09:24:13.477730+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:30.135339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":287,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473896+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:24:18.473896+0000","last_peered":"2023-10-09T09:24:18.473896+0000","last_clean":"2023-10-09T09:24:18.473896+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:24:18.473896+0000","last_undegraded":"2023-10-09T09:24:18.473896+0000","last_fullsized":"2023-10-09T09:24:18.473896+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":275,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473687+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:24:18.473687+0000","last_peered":"2023-10-09T09:24:18.473687+0000","last_clean":"2023-10-09T09:24:18.473687+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:24:18.473687+0000","last_undegraded":"2023-10-09T09:24:18.473687+0000","last_fullsized":"2023-10-09T09:24:18.473687+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":233,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.480965+0000","last_change":"2023-10-09T09:20:19.876172+0000","last_active":"2023-10-09T09:24:19.480965+0000","last_peered":"2023-10-09T09:24:19.480965+0000","last_clean":"2023-10-09T09:24:19.480965+0000","last_became_active":"2023-10-09T09:20:19.765721+0000","last_became_peered":"2023-10-09T09:20:19.765721+0000","last_unstale":"2023-10-09T09:24:19.480965+0000","last_undegraded":"2023-10-09T09:24:19.480965+0000","last_fullsized":"2023-10-09T09:24:19.480965+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":133,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.446318+0000","last_change":"2023-10-09T09:14:48.319108+0000","last_active":"2023-10-09T09:24:16.446318+0000","last_peered":"2023-10-09T09:24:16.446318+0000","last_clean":"2023-10-09T09:24:16.446318+0000","last_became_active":"2023-10-09T09:14:48.318443+0000","last_became_peered":"2023-10-09T09:14:48.318443+0000","last_unstale":"2023-10-09T09:24:16.446318+0000","last_undegraded":"2023-10-09T09:24:16.446318+0000","last_fullsized":"2023-10-09T09:24:16.446318+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:31.939020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":401,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.455829+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:24:16.455829+0000","last_peered":"2023-10-09T09:24:16.455829+0000","last_clean":"2023-10-09T09:24:16.455829+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:24:16.455829+0000","last_undegraded":"2023-10-09T09:24:16.455829+0000","last_fullsized":"2023-10-09T09:24:16.455829+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":186,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.481251+0000","last_change":"2023-10-09T09:15:23.415336+0000","last_active":"2023-10-09T09:24:19.481251+0000","last_peered":"2023-10-09T09:24:19.481251+0000","last_clean":"2023-10-09T09:24:19.481251+0000","last_became_active":"2023-10-09T09:15:23.414900+0000","last_became_peered":"2023-10-09T09:15:23.414900+0000","last_unstale":"2023-10-09T09:24:19.481251+0000","last_undegraded":"2023-10-09T09:24:19.481251+0000","last_fullsized":"2023-10-09T09:24:19.481251+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":229,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473811+0000","last_change":"2023-10-09T09:13:48.205269+0000","last_active":"2023-10-09T09:24:18.473811+0000","last_peered":"2023-10-09T09:24:18.473811+0000","last_clean":"2023-10-09T09:24:18.473811+0000","last_became_active":"2023-10-09T09:13:48.204894+0000","last_became_peered":"2023-10-09T09:13:48.204894+0000","last_unstale":"2023-10-09T09:24:18.473811+0000","last_undegraded":"2023-10-09T09:24:18.473811+0000","last_fullsized":"2023-10-09T09:24:18.473811+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":373,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456242+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:24:16.456242+0000","last_peered":"2023-10-09T09:24:16.456242+0000","last_clean":"2023-10-09T09:24:16.456242+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:24:16.456242+0000","last_undegraded":"2023-10-09T09:24:16.456242+0000","last_fullsized":"2023-10-09T09:24:16.456242+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":303,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.491824+0000","last_change":"2023-10-09T09:15:23.420333+0000","last_active":"2023-10-09T09:24:19.491824+0000","last_peered":"2023-10-09T09:24:19.491824+0000","last_clean":"2023-10-09T09:24:19.491824+0000","last_became_active":"2023-10-09T09:15:23.412581+0000","last_became_peered":"2023-10-09T09:15:23.412581+0000","last_unstale":"2023-10-09T09:24:19.491824+0000","last_undegraded":"2023-10-09T09:24:19.491824+0000","last_fullsized":"2023-10-09T09:24:19.491824+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.854489+0000","last_change":"2023-10-09T09:21:51.915241+0000","last_active":"2023-10-09T09:24:18.854489+0000","last_peered":"2023-10-09T09:24:18.854489+0000","last_clean":"2023-10-09T09:24:18.854489+0000","last_became_active":"2023-10-09T09:21:51.914856+0000","last_became_peered":"2023-10-09T09:21:51.914856+0000","last_unstale":"2023-10-09T09:24:18.854489+0000","last_undegraded":"2023-10-09T09:24:18.854489+0000","last_fullsized":"2023-10-09T09:24:18.854489+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:21.372893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":184,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.481256+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:24:19.481256+0000","last_peered":"2023-10-09T09:24:19.481256+0000","last_clean":"2023-10-09T09:24:19.481256+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:24:19.481256+0000","last_undegraded":"2023-10-09T09:24:19.481256+0000","last_fullsized":"2023-10-09T09:24:19.481256+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":146,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.480729+0000","last_change":"2023-10-09T09:15:23.414649+0000","last_active":"2023-10-09T09:24:19.480729+0000","last_peered":"2023-10-09T09:24:19.480729+0000","last_clean":"2023-10-09T09:24:19.480729+0000","last_became_active":"2023-10-09T09:15:23.414272+0000","last_became_peered":"2023-10-09T09:15:23.414272+0000","last_unstale":"2023-10-09T09:24:19.480729+0000","last_undegraded":"2023-10-09T09:24:19.480729+0000","last_fullsized":"2023-10-09T09:24:19.480729+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":184,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.481222+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:24:19.481222+0000","last_peered":"2023-10-09T09:24:19.481222+0000","last_clean":"2023-10-09T09:24:19.481222+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:24:19.481222+0000","last_undegraded":"2023-10-09T09:24:19.481222+0000","last_fullsized":"2023-10-09T09:24:19.481222+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":275,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473628+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:24:18.473628+0000","last_peered":"2023-10-09T09:24:18.473628+0000","last_clean":"2023-10-09T09:24:18.473628+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:24:18.473628+0000","last_undegraded":"2023-10-09T09:24:18.473628+0000","last_fullsized":"2023-10-09T09:24:18.473628+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":260,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.492129+0000","last_change":"2023-10-09T09:15:23.437420+0000","last_active":"2023-10-09T09:24:19.492129+0000","last_peered":"2023-10-09T09:24:19.492129+0000","last_clean":"2023-10-09T09:24:19.492129+0000","last_became_active":"2023-10-09T09:15:23.429358+0000","last_became_peered":"2023-10-09T09:15:23.429358+0000","last_unstale":"2023-10-09T09:24:19.492129+0000","last_undegraded":"2023-10-09T09:24:19.492129+0000","last_fullsized":"2023-10-09T09:24:19.492129+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":290,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.491940+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:24:19.491940+0000","last_peered":"2023-10-09T09:24:19.491940+0000","last_clean":"2023-10-09T09:24:19.491940+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:24:19.491940+0000","last_undegraded":"2023-10-09T09:24:19.491940+0000","last_fullsized":"2023-10-09T09:24:19.491940+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":174,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.477127+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:24:19.477127+0000","last_peered":"2023-10-09T09:24:19.477127+0000","last_clean":"2023-10-09T09:24:19.477127+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:24:19.477127+0000","last_undegraded":"2023-10-09T09:24:19.477127+0000","last_fullsized":"2023-10-09T09:24:19.477127+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":402,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456062+0000","last_change":"2023-10-09T09:18:51.869617+0000","last_active":"2023-10-09T09:24:16.456062+0000","last_peered":"2023-10-09T09:24:16.456062+0000","last_clean":"2023-10-09T09:24:16.456062+0000","last_became_active":"2023-10-09T09:18:51.869212+0000","last_became_peered":"2023-10-09T09:18:51.869212+0000","last_unstale":"2023-10-09T09:24:16.456062+0000","last_undegraded":"2023-10-09T09:24:16.456062+0000","last_fullsized":"2023-10-09T09:24:16.456062+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:36.753958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":195,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.455828+0000","last_change":"2023-10-09T09:09:06.659678+0000","last_active":"2023-10-09T09:24:16.455828+0000","last_peered":"2023-10-09T09:24:16.455828+0000","last_clean":"2023-10-09T09:24:16.455828+0000","last_became_active":"2023-10-09T09:09:06.659338+0000","last_became_peered":"2023-10-09T09:09:06.659338+0000","last_unstale":"2023-10-09T09:24:16.455828+0000","last_undegraded":"2023-10-09T09:24:16.455828+0000","last_fullsized":"2023-10-09T09:24:16.455828+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:19.082028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":248,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473767+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:24:18.473767+0000","last_peered":"2023-10-09T09:24:18.473767+0000","last_clean":"2023-10-09T09:24:18.473767+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:24:18.473767+0000","last_undegraded":"2023-10-09T09:24:18.473767+0000","last_fullsized":"2023-10-09T09:24:18.473767+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":254,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.491882+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:24:19.491882+0000","last_peered":"2023-10-09T09:24:19.491882+0000","last_clean":"2023-10-09T09:24:19.491882+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:24:19.491882+0000","last_undegraded":"2023-10-09T09:24:19.491882+0000","last_fullsized":"2023-10-09T09:24:19.491882+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":293,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.474151+0000","last_change":"2023-10-09T09:20:19.935938+0000","last_active":"2023-10-09T09:24:18.474151+0000","last_peered":"2023-10-09T09:24:18.474151+0000","last_clean":"2023-10-09T09:24:18.474151+0000","last_became_active":"2023-10-09T09:20:19.935423+0000","last_became_peered":"2023-10-09T09:20:19.935423+0000","last_unstale":"2023-10-09T09:24:18.474151+0000","last_undegraded":"2023-10-09T09:24:18.474151+0000","last_fullsized":"2023-10-09T09:24:18.474151+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":320,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473594+0000","last_change":"2023-10-09T09:20:19.755858+0000","last_active":"2023-10-09T09:24:18.473594+0000","last_peered":"2023-10-09T09:24:18.473594+0000","last_clean":"2023-10-09T09:24:18.473594+0000","last_became_active":"2023-10-09T09:20:19.755442+0000","last_became_peered":"2023-10-09T09:20:19.755442+0000","last_unstale":"2023-10-09T09:24:18.473594+0000","last_undegraded":"2023-10-09T09:24:18.473594+0000","last_fullsized":"2023-10-09T09:24:18.473594+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":266,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.474019+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:24:18.474019+0000","last_peered":"2023-10-09T09:24:18.474019+0000","last_clean":"2023-10-09T09:24:18.474019+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:24:18.474019+0000","last_undegraded":"2023-10-09T09:24:18.474019+0000","last_fullsized":"2023-10-09T09:24:18.474019+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":133,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477805+0000","last_change":"2023-10-09T09:14:48.306639+0000","last_active":"2023-10-09T09:24:13.477805+0000","last_peered":"2023-10-09T09:24:13.477805+0000","last_clean":"2023-10-09T09:24:13.477805+0000","last_became_active":"2023-10-09T09:14:48.305729+0000","last_became_peered":"2023-10-09T09:14:48.305729+0000","last_unstale":"2023-10-09T09:24:13.477805+0000","last_undegraded":"2023-10-09T09:24:13.477805+0000","last_fullsized":"2023-10-09T09:24:13.477805+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:08.892989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":133,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477651+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:24:13.477651+0000","last_peered":"2023-10-09T09:24:13.477651+0000","last_clean":"2023-10-09T09:24:13.477651+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:24:13.477651+0000","last_undegraded":"2023-10-09T09:24:13.477651+0000","last_fullsized":"2023-10-09T09:24:13.477651+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"383'36","reported_seq":226,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.519820+0000","last_change":"2023-10-09T09:24:13.478583+0000","last_active":"2023-10-09T09:24:16.519820+0000","last_peered":"2023-10-09T09:24:16.519820+0000","last_clean":"2023-10-09T09:24:16.519820+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:24:16.519820+0000","last_undegraded":"2023-10-09T09:24:16.519820+0000","last_fullsized":"2023-10-09T09:24:16.519820+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":36,"log_dups_size":0,"ondisk_log_size":36,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072168899999999997,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":98,"num_read_kb":867,"num_write":45,"num_write_kb":405,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.5","version":"0'0","reported_seq":210,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.478739+0000","last_change":"2023-10-09T09:20:19.757594+0000","last_active":"2023-10-09T09:24:19.478739+0000","last_peered":"2023-10-09T09:24:19.478739+0000","last_clean":"2023-10-09T09:24:19.478739+0000","last_became_active":"2023-10-09T09:20:19.757216+0000","last_became_peered":"2023-10-09T09:20:19.757216+0000","last_unstale":"2023-10-09T09:24:19.478739+0000","last_undegraded":"2023-10-09T09:24:19.478739+0000","last_fullsized":"2023-10-09T09:24:19.478739+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477633+0000","last_change":"2023-10-09T09:15:23.413972+0000","last_active":"2023-10-09T09:24:13.477633+0000","last_peered":"2023-10-09T09:24:13.477633+0000","last_clean":"2023-10-09T09:24:13.477633+0000","last_became_active":"2023-10-09T09:15:23.413531+0000","last_became_peered":"2023-10-09T09:15:23.413531+0000","last_unstale":"2023-10-09T09:24:13.477633+0000","last_undegraded":"2023-10-09T09:24:13.477633+0000","last_fullsized":"2023-10-09T09:24:13.477633+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:54.516595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"386'70","reported_seq":476,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.925213+0000","last_change":"2023-10-09T09:24:19.925414+0000","last_active":"2023-10-09T09:24:19.925213+0000","last_peered":"2023-10-09T09:24:19.925213+0000","last_clean":"2023-10-09T09:24:19.925213+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:24:19.925213+0000","last_undegraded":"2023-10-09T09:24:19.925213+0000","last_fullsized":"2023-10-09T09:24:19.925213+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":390,"num_read_kb":518,"num_write":70,"num_write_kb":131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"8","length":"1"}]},{"pgid":"3.3","version":"0'0","reported_seq":248,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.474363+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:24:18.474363+0000","last_peered":"2023-10-09T09:24:18.474363+0000","last_clean":"2023-10-09T09:24:18.474363+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:24:18.474363+0000","last_undegraded":"2023-10-09T09:24:18.474363+0000","last_fullsized":"2023-10-09T09:24:18.474363+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":160,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477676+0000","last_change":"2023-10-09T09:20:19.932428+0000","last_active":"2023-10-09T09:24:13.477676+0000","last_peered":"2023-10-09T09:24:13.477676+0000","last_clean":"2023-10-09T09:24:13.477676+0000","last_became_active":"2023-10-09T09:20:19.931684+0000","last_became_peered":"2023-10-09T09:20:19.931684+0000","last_unstale":"2023-10-09T09:24:13.477676+0000","last_undegraded":"2023-10-09T09:24:13.477676+0000","last_fullsized":"2023-10-09T09:24:13.477676+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:40.774870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"382'100","reported_seq":467,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T09:24:17.568812+0000","last_change":"2023-10-09T09:24:13.434429+0000","last_active":"2023-10-09T09:24:17.568812+0000","last_peered":"2023-10-09T09:24:17.568812+0000","last_clean":"2023-10-09T09:24:17.568812+0000","last_became_active":"2023-10-09T09:20:19.757450+0000","last_became_peered":"2023-10-09T09:20:19.757450+0000","last_unstale":"2023-10-09T09:24:17.568812+0000","last_undegraded":"2023-10-09T09:24:17.568812+0000","last_fullsized":"2023-10-09T09:24:17.568812+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:57.053230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096056699999999995,"stat_sum":{"num_bytes":16,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":656,"num_read_kb":515,"num_write":88,"num_write_kb":38,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.0","version":"0'0","reported_seq":160,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477274+0000","last_change":"2023-10-09T09:20:19.902503+0000","last_active":"2023-10-09T09:24:13.477274+0000","last_peered":"2023-10-09T09:24:13.477274+0000","last_clean":"2023-10-09T09:24:13.477274+0000","last_became_active":"2023-10-09T09:20:19.902131+0000","last_became_peered":"2023-10-09T09:20:19.902131+0000","last_unstale":"2023-10-09T09:24:13.477274+0000","last_undegraded":"2023-10-09T09:24:13.477274+0000","last_fullsized":"2023-10-09T09:24:13.477274+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:43.897708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T09:24:17.449063+0000","last_change":"2023-10-09T09:21:51.904699+0000","last_active":"2023-10-09T09:24:17.449063+0000","last_peered":"2023-10-09T09:24:17.449063+0000","last_clean":"2023-10-09T09:24:17.449063+0000","last_became_active":"2023-10-09T09:21:51.904381+0000","last_became_peered":"2023-10-09T09:21:51.904381+0000","last_unstale":"2023-10-09T09:24:17.449063+0000","last_undegraded":"2023-10-09T09:24:17.449063+0000","last_fullsized":"2023-10-09T09:24:17.449063+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:46.915626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"387'85","reported_seq":380,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.928669+0000","last_change":"2023-10-09T09:24:19.925202+0000","last_active":"2023-10-09T09:24:19.928669+0000","last_peered":"2023-10-09T09:24:19.928669+0000","last_clean":"2023-10-09T09:24:19.928669+0000","last_became_active":"2023-10-09T09:20:19.759254+0000","last_became_peered":"2023-10-09T09:20:19.759254+0000","last_unstale":"2023-10-09T09:24:19.928669+0000","last_undegraded":"2023-10-09T09:24:19.928669+0000","last_fullsized":"2023-10-09T09:24:19.928669+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":85,"log_dups_size":0,"ondisk_log_size":85,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":171,"num_read_kb":149,"num_write":90,"num_write_kb":61,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"8","length":"1"}]},{"pgid":"1.0","version":"74'139","reported_seq":625,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.480759+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:24:19.480759+0000","last_peered":"2023-10-09T09:24:19.480759+0000","last_clean":"2023-10-09T09:24:19.480759+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:24:19.480759+0000","last_undegraded":"2023-10-09T09:24:19.480759+0000","last_fullsized":"2023-10-09T09:24:19.480759+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":213,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.477972+0000","last_change":"2023-10-09T09:09:06.658884+0000","last_active":"2023-10-09T09:24:19.477972+0000","last_peered":"2023-10-09T09:24:19.477972+0000","last_clean":"2023-10-09T09:24:19.477972+0000","last_became_active":"2023-10-09T09:09:06.658543+0000","last_became_peered":"2023-10-09T09:09:06.658543+0000","last_unstale":"2023-10-09T09:24:19.477972+0000","last_undegraded":"2023-10-09T09:24:19.477972+0000","last_fullsized":"2023-10-09T09:24:19.477972+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":245,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.492343+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:24:19.492343+0000","last_peered":"2023-10-09T09:24:19.492343+0000","last_clean":"2023-10-09T09:24:19.492343+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:24:19.492343+0000","last_undegraded":"2023-10-09T09:24:19.492343+0000","last_fullsized":"2023-10-09T09:24:19.492343+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"382'62","reported_seq":326,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:15.865078+0000","last_change":"2023-10-09T09:24:13.478222+0000","last_active":"2023-10-09T09:24:15.865078+0000","last_peered":"2023-10-09T09:24:15.865078+0000","last_clean":"2023-10-09T09:24:15.865078+0000","last_became_active":"2023-10-09T09:14:48.301379+0000","last_became_peered":"2023-10-09T09:14:48.301379+0000","last_unstale":"2023-10-09T09:24:15.865078+0000","last_undegraded":"2023-10-09T09:24:15.865078+0000","last_fullsized":"2023-10-09T09:24:15.865078+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:11.963433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045939400000000001,"stat_sum":{"num_bytes":112640,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":400,"num_read_kb":334,"num_write":53,"num_write_kb":43,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.1","version":"0'0","reported_seq":421,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456020+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:24:16.456020+0000","last_peered":"2023-10-09T09:24:16.456020+0000","last_clean":"2023-10-09T09:24:16.456020+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:24:16.456020+0000","last_undegraded":"2023-10-09T09:24:16.456020+0000","last_fullsized":"2023-10-09T09:24:16.456020+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":444,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.455886+0000","last_change":"2023-10-09T09:20:19.890699+0000","last_active":"2023-10-09T09:24:16.455886+0000","last_peered":"2023-10-09T09:24:16.455886+0000","last_clean":"2023-10-09T09:24:16.455886+0000","last_became_active":"2023-10-09T09:20:19.889949+0000","last_became_peered":"2023-10-09T09:20:19.889949+0000","last_unstale":"2023-10-09T09:24:16.455886+0000","last_undegraded":"2023-10-09T09:24:16.455886+0000","last_fullsized":"2023-10-09T09:24:16.455886+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:59.920953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"386'90","reported_seq":638,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.934692+0000","last_change":"2023-10-09T09:24:19.935038+0000","last_active":"2023-10-09T09:24:19.934692+0000","last_peered":"2023-10-09T09:24:19.934692+0000","last_clean":"2023-10-09T09:24:19.934692+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:24:19.934692+0000","last_undegraded":"2023-10-09T09:24:19.934692+0000","last_fullsized":"2023-10-09T09:24:19.934692+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":90,"log_dups_size":0,"ondisk_log_size":90,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112640,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":937,"num_read_kb":1373,"num_write":60,"num_write_kb":324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"8","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":160,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T09:24:13.477483+0000","last_change":"2023-10-09T09:20:19.912177+0000","last_active":"2023-10-09T09:24:13.477483+0000","last_peered":"2023-10-09T09:24:13.477483+0000","last_clean":"2023-10-09T09:24:13.477483+0000","last_became_active":"2023-10-09T09:20:19.908966+0000","last_became_peered":"2023-10-09T09:20:19.908966+0000","last_unstale":"2023-10-09T09:24:13.477483+0000","last_undegraded":"2023-10-09T09:24:13.477483+0000","last_fullsized":"2023-10-09T09:24:13.477483+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:54.778943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":397,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456137+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:24:16.456137+0000","last_peered":"2023-10-09T09:24:16.456137+0000","last_clean":"2023-10-09T09:24:16.456137+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:24:16.456137+0000","last_undegraded":"2023-10-09T09:24:16.456137+0000","last_fullsized":"2023-10-09T09:24:16.456137+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":153,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456311+0000","last_change":"2023-10-09T09:14:19.749169+0000","last_active":"2023-10-09T09:24:16.456311+0000","last_peered":"2023-10-09T09:24:16.456311+0000","last_clean":"2023-10-09T09:24:16.456311+0000","last_became_active":"2023-10-09T09:14:19.740888+0000","last_became_peered":"2023-10-09T09:14:19.740888+0000","last_unstale":"2023-10-09T09:24:16.456311+0000","last_undegraded":"2023-10-09T09:24:16.456311+0000","last_fullsized":"2023-10-09T09:24:16.456311+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:01.361061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"383'71","reported_seq":442,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.521043+0000","last_change":"2023-10-09T09:24:13.478676+0000","last_active":"2023-10-09T09:24:16.521043+0000","last_peered":"2023-10-09T09:24:16.521043+0000","last_clean":"2023-10-09T09:24:16.521043+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:24:16.521043+0000","last_undegraded":"2023-10-09T09:24:16.521043+0000","last_fullsized":"2023-10-09T09:24:16.521043+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080345600000000003,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":531,"num_read_kb":604,"num_write":47,"num_write_kb":106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":167,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.480471+0000","last_change":"2023-10-09T09:09:44.393987+0000","last_active":"2023-10-09T09:24:19.480471+0000","last_peered":"2023-10-09T09:24:19.480471+0000","last_clean":"2023-10-09T09:24:19.480471+0000","last_became_active":"2023-10-09T09:09:44.393499+0000","last_became_peered":"2023-10-09T09:09:44.393499+0000","last_unstale":"2023-10-09T09:24:19.480471+0000","last_undegraded":"2023-10-09T09:24:19.480471+0000","last_fullsized":"2023-10-09T09:24:19.480471+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"382'35","reported_seq":417,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.921510+0000","last_change":"2023-10-09T09:24:19.921711+0000","last_active":"2023-10-09T09:24:19.921510+0000","last_peered":"2023-10-09T09:24:19.921510+0000","last_clean":"2023-10-09T09:24:19.921510+0000","last_became_active":"2023-10-09T09:20:19.757552+0000","last_became_peered":"2023-10-09T09:20:19.757552+0000","last_unstale":"2023-10-09T09:24:19.921510+0000","last_undegraded":"2023-10-09T09:24:19.921510+0000","last_fullsized":"2023-10-09T09:24:19.921510+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":35,"log_dups_size":0,"ondisk_log_size":35,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":168,"num_read_kb":501,"num_write":25,"num_write_kb":200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"8","length":"1"}]},{"pgid":"3.6","version":"60'1","reported_seq":439,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456010+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:24:16.456010+0000","last_peered":"2023-10-09T09:24:16.456010+0000","last_clean":"2023-10-09T09:24:16.456010+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:24:16.456010+0000","last_undegraded":"2023-10-09T09:24:16.456010+0000","last_fullsized":"2023-10-09T09:24:16.456010+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":266,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473760+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:24:18.473760+0000","last_peered":"2023-10-09T09:24:18.473760+0000","last_clean":"2023-10-09T09:24:18.473760+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:24:18.473760+0000","last_undegraded":"2023-10-09T09:24:18.473760+0000","last_fullsized":"2023-10-09T09:24:18.473760+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":407,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456007+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:24:16.456007+0000","last_peered":"2023-10-09T09:24:16.456007+0000","last_clean":"2023-10-09T09:24:16.456007+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:24:16.456007+0000","last_undegraded":"2023-10-09T09:24:16.456007+0000","last_fullsized":"2023-10-09T09:24:16.456007+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T09:24:17.448792+0000","last_change":"2023-10-09T09:20:19.916151+0000","last_active":"2023-10-09T09:24:17.448792+0000","last_peered":"2023-10-09T09:24:17.448792+0000","last_clean":"2023-10-09T09:24:17.448792+0000","last_became_active":"2023-10-09T09:20:19.915607+0000","last_became_peered":"2023-10-09T09:20:19.915607+0000","last_unstale":"2023-10-09T09:24:17.448792+0000","last_undegraded":"2023-10-09T09:24:17.448792+0000","last_fullsized":"2023-10-09T09:24:17.448792+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:57:51.697949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":191,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.476959+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:24:19.476959+0000","last_peered":"2023-10-09T09:24:19.476959+0000","last_clean":"2023-10-09T09:24:19.476959+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:24:19.476959+0000","last_undegraded":"2023-10-09T09:24:19.476959+0000","last_fullsized":"2023-10-09T09:24:19.476959+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":266,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473923+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:24:18.473923+0000","last_peered":"2023-10-09T09:24:18.473923+0000","last_clean":"2023-10-09T09:24:18.473923+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:24:18.473923+0000","last_undegraded":"2023-10-09T09:24:18.473923+0000","last_fullsized":"2023-10-09T09:24:18.473923+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":178,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.477380+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:24:19.477380+0000","last_peered":"2023-10-09T09:24:19.477380+0000","last_clean":"2023-10-09T09:24:19.477380+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:24:19.477380+0000","last_undegraded":"2023-10-09T09:24:19.477380+0000","last_fullsized":"2023-10-09T09:24:19.477380+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":248,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473931+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:24:18.473931+0000","last_peered":"2023-10-09T09:24:18.473931+0000","last_clean":"2023-10-09T09:24:18.473931+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:24:18.473931+0000","last_undegraded":"2023-10-09T09:24:18.473931+0000","last_fullsized":"2023-10-09T09:24:18.473931+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":368,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-10-09T09:24:19.492195+0000","last_change":"2023-10-09T09:20:19.760410+0000","last_active":"2023-10-09T09:24:19.492195+0000","last_peered":"2023-10-09T09:24:19.492195+0000","last_clean":"2023-10-09T09:24:19.492195+0000","last_became_active":"2023-10-09T09:20:19.759916+0000","last_became_peered":"2023-10-09T09:20:19.759916+0000","last_unstale":"2023-10-09T09:24:19.492195+0000","last_undegraded":"2023-10-09T09:24:19.492195+0000","last_fullsized":"2023-10-09T09:24:19.492195+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":89,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T09:24:17.448669+0000","last_change":"2023-10-09T09:20:19.759403+0000","last_active":"2023-10-09T09:24:17.448669+0000","last_peered":"2023-10-09T09:24:17.448669+0000","last_clean":"2023-10-09T09:24:17.448669+0000","last_became_active":"2023-10-09T09:20:19.759030+0000","last_became_peered":"2023-10-09T09:20:19.759030+0000","last_unstale":"2023-10-09T09:24:17.448669+0000","last_undegraded":"2023-10-09T09:24:17.448669+0000","last_fullsized":"2023-10-09T09:24:17.448669+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:10.903233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":527,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T09:24:16.456216+0000","last_change":"2023-10-09T09:20:19.788084+0000","last_active":"2023-10-09T09:24:16.456216+0000","last_peered":"2023-10-09T09:24:16.456216+0000","last_clean":"2023-10-09T09:24:16.456216+0000","last_became_active":"2023-10-09T09:20:19.762112+0000","last_became_peered":"2023-10-09T09:20:19.762112+0000","last_unstale":"2023-10-09T09:24:16.456216+0000","last_undegraded":"2023-10-09T09:24:16.456216+0000","last_fullsized":"2023-10-09T09:24:16.456216+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:47.406781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":275,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T09:24:18.473462+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:24:18.473462+0000","last_peered":"2023-10-09T09:24:18.473462+0000","last_clean":"2023-10-09T09:24:18.473462+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:24:18.473462+0000","last_undegraded":"2023-10-09T09:24:18.473462+0000","last_fullsized":"2023-10-09T09:24:18.473462+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":97,"num_bytes_recovered":20970,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":225455,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":3351,"num_read_kb":4861,"num_write":478,"num_write_kb":1308,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":61997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":549,"ondisk_log_size":549,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430846,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907288,"kb_used_data":3980,"kb_used_omap":0,"kb_used_meta":903296,"kb_avail":92837864,"statfs":{"total":95995035648,"available":95065972736,"internally_reserved":0,"allocated":4075520,"data_stored":3434882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924975104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:38 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.2629999999999999,"15min":2.1739999999999999},"min":{"1min":0.53400000000000003,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":28.184999999999999,"5min":51.859999999999999,"15min":51.859999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":2.468,"5min":2.3410000000000002,"15min":2.1680000000000001},"min":{"1min":0.57499999999999996,"5min":0.28899999999999998,"15min":0.28799999999999998},"max":{"1min":28.282,"5min":51.957999999999998,"15min":51.957999999999998},"last":0.64400000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":3.4119999999999999,"5min":2.0550000000000002,"15min":1.5980000000000001},"min":{"1min":0.56799999999999995,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":52.539999999999999,"5min":52.539999999999999,"15min":52.539999999999999},"last":0.98699999999999999},{"interface":"front","average":{"1min":3.387,"5min":2.0990000000000002,"15min":1.544},"min":{"1min":0.5,"5min":0.35499999999999998,"15min":0.316},"max":{"1min":52.593000000000004,"5min":52.593000000000004,"15min":52.593000000000004},"last":0.73399999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:23:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":2.0960000000000001,"15min":1.8700000000000001},"min":{"1min":0.60199999999999998,"5min":0.32800000000000001,"15min":0.31},"max":{"1min":1.113,"5min":51.914000000000001,"15min":51.914000000000001},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.7210000000000001,"15min":1.8},"min":{"1min":0.60799999999999998,"5min":0.33200000000000002,"15min":0.30599999999999999},"max":{"1min":1.4159999999999999,"5min":51.902999999999999,"15min":51.902999999999999},"last":1.1439999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.637,"15min":1.883},"min":{"1min":0.51800000000000002,"5min":0.374,"15min":0.316},"max":{"1min":2.8879999999999999,"5min":18.178999999999998,"15min":62.060000000000002},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.484,"15min":1.9179999999999999},"min":{"1min":0.42999999999999999,"5min":0.31,"15min":0.31},"max":{"1min":1.4870000000000001,"5min":17.931999999999999,"15min":63.268999999999998},"last":1.214}]},{"osd":4,"last update":"Mon Oct 9 09:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.179,"15min":1.548},"min":{"1min":0.82399999999999995,"5min":0.32000000000000001,"15min":0.28699999999999998},"max":{"1min":1.331,"5min":7.1799999999999997,"15min":13.895},"last":1.034},{"interface":"front","average":{"1min":1.111,"5min":1.079,"15min":1.506},"min":{"1min":0.625,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.3819999999999999,"5min":6.6669999999999998,"15min":16.433},"last":1.2829999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.98199999999999998,"15min":1.395},"min":{"1min":0.76100000000000001,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":1.395,"5min":7.2789999999999999,"15min":15.616},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.1839999999999999,"15min":1.7130000000000001},"min":{"1min":0.81499999999999995,"5min":0.38,"15min":0.34499999999999997},"max":{"1min":1.379,"5min":6.8029999999999999,"15min":18.501000000000001},"last":1.2310000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.45,"15min":1.675},"min":{"1min":0.73399999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.4570000000000001,"5min":12.898,"15min":21.145},"last":1.1080000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.2989999999999999,"15min":1.7130000000000001},"min":{"1min":0.84099999999999997,"5min":0.38900000000000001,"15min":0.34999999999999998},"max":{"1min":1.5349999999999999,"5min":9.4030000000000005,"15min":19.068999999999999},"last":1.3320000000000001}]}]},{"osd":6,"up_from":42,"seq":180388627079,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511168,"kb_used_data":3960,"kb_used_omap":0,"kb_used_meta":1507200,"kb_avail":92233984,"statfs":{"total":95995035648,"available":94447599616,"internally_reserved":0,"allocated":4055040,"data_stored":3414311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1543372800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.9059999999999999,"15min":3},"min":{"1min":0.57299999999999995,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.323,"5min":44.720999999999997,"15min":120.812},"last":1.091},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.5900000000000001,"15min":2.8199999999999998},"min":{"1min":0.76300000000000001,"5min":0.25600000000000001,"15min":0.24099999999999999},"max":{"1min":1.385,"5min":44.738999999999997,"15min":120.857},"last":1.341}]},{"osd":1,"last update":"Mon Oct 9 09:23:27 2023","interfaces":[{"interface":"back","average":{"1min":2.8730000000000002,"5min":4.0140000000000002,"15min":2.4119999999999999},"min":{"1min":0.57699999999999996,"5min":0.30199999999999999,"15min":0.28100000000000003},"max":{"1min":45.07,"5min":142.31100000000001,"15min":142.31100000000001},"last":0.97599999999999998},{"interface":"front","average":{"1min":2.9780000000000002,"5min":4.1150000000000002,"15min":2.2829999999999999},"min":{"1min":0.626,"5min":0.29799999999999999,"15min":0.25700000000000001},"max":{"1min":45.097999999999999,"5min":132.48099999999999,"15min":132.48099999999999},"last":0.93000000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":2.3940000000000001,"15min":1.9950000000000001},"min":{"1min":0.59199999999999997,"5min":0.33600000000000002,"15min":0.28999999999999998},"max":{"1min":1.4370000000000001,"5min":82.170000000000002,"15min":82.170000000000002},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.95999999999999996,"5min":2.1400000000000001,"15min":1.8640000000000001},"min":{"1min":0.73699999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.339,"5min":82.352000000000004,"15min":82.352000000000004},"last":1.054}]},{"osd":3,"last update":"Mon Oct 9 09:23:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":2.0209999999999999,"15min":1.8440000000000001},"min":{"1min":0.42499999999999999,"5min":0.34000000000000002,"15min":0.28999999999999998},"max":{"1min":1.2929999999999999,"5min":14.525,"15min":16.977},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.085,"5min":1.391,"15min":1.5620000000000001},"min":{"1min":0.49299999999999999,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":6.4820000000000002,"5min":6.4820000000000002,"15min":20.573},"last":0.84599999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.222,"15min":1.6739999999999999},"min":{"1min":0.61499999999999999,"5min":0.33000000000000002,"15min":0.27600000000000002},"max":{"1min":1.5429999999999999,"5min":11.433999999999999,"15min":29.585999999999999},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.02,"15min":1.615},"min":{"1min":0.64500000000000002,"5min":0.316,"15min":0.316},"max":{"1min":1.397,"5min":6.0220000000000002,"15min":20.318000000000001},"last":1.016}]},{"osd":5,"last update":"Mon Oct 9 09:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.177,"15min":1.6930000000000001},"min":{"1min":0.66300000000000003,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":1.528,"5min":11.567,"15min":18.103000000000002},"last":1.131},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0880000000000001,"15min":1.593},"min":{"1min":0.872,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":1.306,"5min":6.1509999999999998,"15min":13.609},"last":0.95799999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.288,"15min":1.768},"min":{"1min":0.55100000000000005,"5min":0.378,"15min":0.33700000000000002},"max":{"1min":1.9139999999999999,"5min":11.597,"15min":29.530000000000001},"last":1.2809999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.244,"15min":1.5620000000000001},"min":{"1min":0.56999999999999995,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.577,"5min":7.9489999999999998,"15min":13.523999999999999},"last":1.2669999999999999}]}]},{"osd":1,"up_from":83,"seq":356482286026,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":651920,"kb_used_data":4152,"kb_used_omap":9,"kb_used_meta":647734,"kb_avail":93093232,"statfs":{"total":95995035648,"available":95327469568,"internally_reserved":0,"allocated":4251648,"data_stored":2896660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":663280617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:36 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":4.2489999999999997,"15min":2.4500000000000002},"min":{"1min":0.371,"5min":0.217,"15min":0.189},"max":{"1min":1.0249999999999999,"5min":133.76499999999999,"15min":133.76499999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.69299999999999995,"5min":3.1659999999999999,"15min":2.27},"min":{"1min":0.41099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.0269999999999999,"5min":127.395,"15min":127.395},"last":0.55600000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":3.6509999999999998,"15min":1.863},"min":{"1min":0.377,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.272,"5min":236.29599999999999,"15min":236.29599999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":3.7599999999999998,"15min":1.7210000000000001},"min":{"1min":0.39400000000000002,"5min":0.29299999999999998,"15min":0.252},"max":{"1min":1.0640000000000001,"5min":236.21000000000001,"15min":236.21000000000001},"last":0.64800000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:23:36 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":3.2719999999999998,"15min":2.1070000000000002},"min":{"1min":0.42699999999999999,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":1.3400000000000001,"5min":152.637,"15min":152.637},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.78900000000000003,"5min":4.923,"15min":2.6800000000000002},"min":{"1min":0.51700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.113,"5min":152.374,"15min":152.374},"last":0.97099999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:23:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.4570000000000001,"15min":1.4790000000000001},"min":{"1min":0.66800000000000004,"5min":0.38,"15min":0.376},"max":{"1min":1.623,"5min":14.56,"15min":14.56},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.998,"5min":3.0630000000000002,"15min":1.8799999999999999},"min":{"1min":0.63800000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.431,"5min":133.702,"15min":133.702},"last":0.86799999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.248,"15min":1.3320000000000001},"min":{"1min":0.68300000000000005,"5min":0.441,"15min":0.33800000000000002},"max":{"1min":1.339,"5min":6.7510000000000003,"15min":9.4290000000000003},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.081,"5min":1.46,"15min":1.302},"min":{"1min":0.60599999999999998,"5min":0.39100000000000001,"15min":0.35199999999999998},"max":{"1min":1.571,"5min":14.464,"15min":14.464},"last":1.3280000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":3.089,"15min":2.0550000000000002},"min":{"1min":0.56499999999999995,"5min":0.40100000000000002,"15min":0.33200000000000002},"max":{"1min":1.448,"5min":133.739,"15min":133.739},"last":1.087},{"interface":"front","average":{"1min":1.087,"5min":1.325,"15min":1.492},"min":{"1min":0.76300000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.4610000000000001,"5min":9.4619999999999997,"15min":9.4619999999999997},"last":0.81499999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.9630000000000001,"15min":1.534},"min":{"1min":0.67100000000000004,"5min":0.41499999999999998,"15min":0.38500000000000001},"max":{"1min":1.617,"5min":19.861999999999998,"15min":19.861999999999998},"last":1.417},{"interface":"front","average":{"1min":1.0860000000000001,"5min":3.1179999999999999,"15min":1.99},"min":{"1min":0.747,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.4950000000000001,"5min":133.66499999999999,"15min":133.66499999999999},"last":1.0349999999999999}]}]},{"osd":0,"up_from":78,"seq":335007449552,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":993088,"kb_used_data":4776,"kb_used_omap":28,"kb_used_meta":988259,"kb_avail":92752064,"statfs":{"total":95995035648,"available":94978113536,"internally_reserved":0,"allocated":4890624,"data_stored":3492063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1011978064},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":2.395,"15min":1.458},"min":{"1min":0.34300000000000003,"5min":0.19700000000000001,"15min":0.193},"max":{"1min":1.4390000000000001,"5min":83.671999999999997,"15min":83.671999999999997},"last":0.432},{"interface":"front","average":{"1min":0.72799999999999998,"5min":2.298,"15min":1.423},"min":{"1min":0.311,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.3580000000000001,"5min":83.953000000000003,"15min":83.953000000000003},"last":0.56599999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.5149999999999999,"15min":1.6559999999999999},"min":{"1min":0.36899999999999999,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.93300000000000005,"5min":9.5020000000000007,"15min":74.200999999999993},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.4159999999999999,"15min":1.7110000000000001},"min":{"1min":0.52400000000000002,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":1.081,"5min":9.4239999999999995,"15min":73.983999999999995},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.754,"15min":1.7210000000000001},"min":{"1min":0.32300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":7.9939999999999998,"5min":9.2569999999999997,"15min":20.364000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.373,"15min":1.9390000000000001},"min":{"1min":0.36899999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.4589999999999996,"5min":9.9269999999999996,"15min":137.94999999999999},"last":0.63}]},{"osd":4,"last update":"Mon Oct 9 09:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0920000000000001,"15min":1.589},"min":{"1min":0.68899999999999995,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":1.1439999999999999,"5min":8.8040000000000003,"15min":14.163},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.88500000000000001,"5min":1.4139999999999999,"15min":1.8620000000000001},"min":{"1min":0.54700000000000004,"5min":0.316,"15min":0.314},"max":{"1min":1.405,"5min":9.5359999999999996,"15min":11.346},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.0780000000000001,"15min":1.52},"min":{"1min":0.64900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":1.3520000000000001,"5min":5.5919999999999996,"15min":9.9770000000000003},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.028,"5min":1.145,"15min":1.4710000000000001},"min":{"1min":0.66500000000000004,"5min":0.36499999999999999,"15min":0.35299999999999998},"max":{"1min":1.329,"5min":8.8279999999999994,"15min":16.364000000000001},"last":0.82699999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:24:14 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.508,"15min":2.0390000000000001},"min":{"1min":0.75600000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.232,"5min":10.709,"15min":98.864000000000004},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.125,"5min":1.581,"15min":2.1309999999999998},"min":{"1min":0.79800000000000004,"5min":0.39400000000000002,"15min":0.38300000000000001},"max":{"1min":1.4550000000000001,"5min":16.911000000000001,"15min":98.878},"last":0.875}]},{"osd":7,"last update":"Mon Oct 9 09:24:14 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.349,"15min":1.5429999999999999},"min":{"1min":0.81299999999999994,"5min":0.47599999999999998,"15min":0.41699999999999998},"max":{"1min":1.4390000000000001,"5min":8.8640000000000008,"15min":9.6780000000000008},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.6739999999999999,"15min":1.9099999999999999},"min":{"1min":0.623,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.4330000000000001,"5min":9.5850000000000009,"15min":10.734},"last":0.755}]}]},{"osd":2,"up_from":88,"seq":377957122500,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":970852,"kb_used_data":4188,"kb_used_omap":81,"kb_used_meta":966574,"kb_avail":92774300,"statfs":{"total":95995035648,"available":95000883200,"internally_reserved":0,"allocated":4288512,"data_stored":2896467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":989771994},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":2.5489999999999999,"15min":1.9650000000000001},"min":{"1min":0.32400000000000001,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":4.46,"5min":83.421999999999997,"15min":85.091999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.075,"5min":2.2149999999999999,"15min":1.8540000000000001},"min":{"1min":0.33100000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.4909999999999997,"5min":83.373999999999995,"15min":84.373999999999995},"last":0.875}]},{"osd":1,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.887,"15min":1.8120000000000001},"min":{"1min":0.35499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":4.29,"5min":16.241,"15min":122.367},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.782,"15min":1.7370000000000001},"min":{"1min":0.42199999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0049999999999999,"5min":16.948,"15min":122.777},"last":1.042}]},{"osd":3,"last update":"Mon Oct 9 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":4.569,"15min":2.6600000000000001},"min":{"1min":0.38900000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.4139999999999999,"5min":152.29300000000001,"15min":152.29300000000001},"last":0.622},{"interface":"front","average":{"1min":2.1360000000000001,"5min":5.5949999999999998,"15min":2.9830000000000001},"min":{"1min":0.38100000000000001,"5min":0.311,"15min":0.27100000000000002},"max":{"1min":26.050999999999998,"5min":153.70400000000001,"15min":153.70400000000001},"last":0.74299999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.341,"15min":1.5009999999999999},"min":{"1min":0.60899999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.6319999999999999,"5min":7.1719999999999997,"15min":14.343999999999999},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.341,"5min":1.4119999999999999,"15min":1.579},"min":{"1min":0.64400000000000002,"5min":0.35799999999999998,"15min":0.29699999999999999},"max":{"1min":4.3300000000000001,"5min":6.8600000000000003,"15min":12.351000000000001},"last":1.286}]},{"osd":5,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.244,"15min":1.5549999999999999},"min":{"1min":0.442,"5min":0.35999999999999999,"15min":0.35799999999999998},"max":{"1min":1.976,"5min":10.135,"15min":72.311000000000007},"last":1.3240000000000001},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.4830000000000001,"15min":1.833},"min":{"1min":0.65200000000000002,"5min":0.39700000000000002,"15min":0.36899999999999999},"max":{"1min":1.571,"5min":7.2080000000000002,"15min":73.042000000000002},"last":1.1020000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.76,"15min":1.724},"min":{"1min":0.72599999999999998,"5min":0.35899999999999999,"15min":0.34300000000000003},"max":{"1min":4.3949999999999996,"5min":8.5730000000000004,"15min":14.714},"last":1.3879999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.663,"15min":1.7070000000000001},"min":{"1min":0.48899999999999999,"5min":0.44900000000000001,"15min":0.40999999999999998},"max":{"1min":2.089,"5min":10.167,"15min":14.571999999999999},"last":1.421}]},{"osd":7,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":1.8220000000000001,"15min":1.603},"min":{"1min":0.59999999999999998,"5min":0.41599999999999998,"15min":0.40200000000000002},"max":{"1min":4.359,"5min":29.513999999999999,"15min":29.513999999999999},"last":1.476},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.704,"15min":1.54},"min":{"1min":0.60799999999999998,"5min":0.42999999999999999,"15min":0.40200000000000002},"max":{"1min":2.2370000000000001,"5min":30.216999999999999,"15min":30.216999999999999},"last":1.516}]}]},{"osd":3,"up_from":93,"seq":399431958975,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":973024,"kb_used_data":4184,"kb_used_omap":7,"kb_used_meta":968824,"kb_avail":92772128,"statfs":{"total":95995035648,"available":94998659072,"internally_reserved":0,"allocated":4284416,"data_stored":2911912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":992076392},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.681,"15min":2.0030000000000001},"min":{"1min":0.36499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":12.922000000000001,"5min":12.922000000000001,"15min":130.45500000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.804,"15min":2.2090000000000001},"min":{"1min":0.32800000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":11.9,"5min":11.9,"15min":129.74799999999999},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.9219999999999999,"15min":1.393},"min":{"1min":0.42099999999999999,"5min":0.309,"15min":0.23599999999999999},"max":{"1min":8.7490000000000006,"5min":9.7029999999999994,"15min":32.520000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.6639999999999999,"15min":1.21},"min":{"1min":0.42299999999999999,"5min":0.23100000000000001,"15min":0.22900000000000001},"max":{"1min":9.8399999999999999,"5min":9.8399999999999999,"15min":32.585999999999999},"last":0.63300000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.869,"15min":1.9079999999999999},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.4710000000000001,"5min":37.997,"15min":129.67699999999999},"last":0.436},{"interface":"front","average":{"1min":1.2130000000000001,"5min":2.0630000000000002,"15min":1.9770000000000001},"min":{"1min":0.44900000000000001,"5min":0.254,"15min":0.254},"max":{"1min":9.7420000000000009,"5min":37.499000000000002,"15min":130.40000000000001},"last":0.51000000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.4199999999999999,"15min":1.3899999999999999},"min":{"1min":0.57099999999999995,"5min":0.316,"15min":0.316},"max":{"1min":9.8119999999999994,"5min":10.445,"15min":10.445},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.4350000000000001,"15min":1.671},"min":{"1min":0.39800000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.927,"5min":8.4510000000000005,"15min":12.617000000000001},"last":0.84299999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.4890000000000001,"15min":1.6319999999999999},"min":{"1min":0.46899999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.9649999999999999,"5min":8.5540000000000003,"15min":16.132999999999999},"last":0.90600000000000003},{"interface":"front","average":{"1min":1.3,"5min":1.387,"15min":1.599},"min":{"1min":0.52600000000000002,"5min":0.39800000000000002,"15min":0.36899999999999999},"max":{"1min":7.5449999999999999,"5min":7.9880000000000004,"15min":14.361000000000001},"last":0.72099999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.845,"15min":1.863},"min":{"1min":0.55000000000000004,"5min":0.38500000000000001,"15min":0.37},"max":{"1min":3,"5min":14.124000000000001,"15min":14.124000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.833,"15min":1.76},"min":{"1min":0.89400000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.5069999999999997,"5min":13.778,"15min":13.778},"last":0.91900000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.6499999999999999,"15min":1.623},"min":{"1min":0.77900000000000003,"5min":0.42699999999999999,"15min":0.41499999999999998},"max":{"1min":9.673,"5min":9.673,"15min":9.673},"last":0.749},{"interface":"front","average":{"1min":1.524,"5min":1.5529999999999999,"15min":1.4910000000000001},"min":{"1min":0.73399999999999999,"5min":0.377,"15min":0.377},"max":{"1min":9.7850000000000001,"5min":9.7850000000000001,"15min":9.7850000000000001},"last":0.97399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019542,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909756,"kb_used_data":3380,"kb_used_omap":0,"kb_used_meta":906368,"kb_avail":92835396,"statfs":{"total":95995035648,"available":95063445504,"internally_reserved":0,"allocated":3461120,"data_stored":2819404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928120832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.3759999999999999,"15min":2.052},"min":{"1min":0.51400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.26,"5min":23.32,"15min":116.59999999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.82799999999999996,"5min":1.1839999999999999,"15min":1.9099999999999999},"min":{"1min":0.46400000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.2070000000000001,"5min":23.581,"15min":116.843},"last":0.754}]},{"osd":1,"last update":"Mon Oct 9 09:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.9279999999999999,"15min":1.4690000000000001},"min":{"1min":0.45100000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.1100000000000001,"5min":37.481999999999999,"15min":37.481999999999999},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.1020000000000001,"5min":2.032,"15min":1.595},"min":{"1min":0.65800000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.5569999999999999,"5min":37.332999999999998,"15min":37.332999999999998},"last":0.82799999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.3080000000000001,"15min":1.9379999999999999},"min":{"1min":0.77100000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":1.329,"5min":8.1400000000000006,"15min":116.7},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.069,"15min":1.869},"min":{"1min":0.55700000000000005,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.1459999999999999,"5min":5.1340000000000003,"15min":116.63800000000001},"last":1.272}]},{"osd":3,"last update":"Mon Oct 9 09:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":2.1920000000000002,"15min":3.5920000000000001},"min":{"1min":0.48699999999999999,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":1.1180000000000001,"5min":55.299999999999997,"15min":165.995},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.9970000000000001,"15min":3.677},"min":{"1min":0.442,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.264,"5min":55.622,"15min":165.53200000000001},"last":1.179}]},{"osd":5,"last update":"Mon Oct 9 09:23:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.105,"15min":1.6319999999999999},"min":{"1min":0.63400000000000001,"5min":0.314,"15min":0.30099999999999999},"max":{"1min":1.45,"5min":8.1110000000000007,"15min":13.907},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.034,"5min":0.95599999999999996,"15min":1.3919999999999999},"min":{"1min":0.54600000000000004,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.367,"5min":5.1840000000000002,"15min":11.042999999999999},"last":1.429}]},{"osd":6,"last update":"Mon Oct 9 09:24:18 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.258,"15min":1.859},"min":{"1min":0.70399999999999996,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.3300000000000001,"5min":12.074,"15min":74.700999999999993},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.163,"5min":1.26,"15min":1.97},"min":{"1min":0.84899999999999998,"5min":0.34000000000000002,"15min":0.33000000000000002},"max":{"1min":1.4930000000000001,"5min":8.0640000000000001,"15min":73.572999999999993},"last":1.3149999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.0680000000000001,"15min":1.6659999999999999},"min":{"1min":0.754,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.603,"5min":5.4749999999999996,"15min":15.484999999999999},"last":1.5700000000000001},{"interface":"front","average":{"1min":1.585,"5min":1.3100000000000001,"15min":1.6459999999999999},"min":{"1min":0.90700000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":11.106999999999999,"5min":12.003,"15min":13.916},"last":1.339}]}]},{"osd":5,"up_from":36,"seq":154618823313,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":912880,"kb_used_data":3364,"kb_used_omap":0,"kb_used_meta":909504,"kb_avail":92832272,"statfs":{"total":95995035648,"available":95060246528,"internally_reserved":0,"allocated":3444736,"data_stored":2819957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931332096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.4370000000000001,"15min":2.8839999999999999},"min":{"1min":0.50700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":36.121000000000002,"5min":36.121000000000002,"15min":139.81700000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.887,"5min":1.776,"15min":2.9449999999999998},"min":{"1min":0.64300000000000002,"5min":0.251,"15min":0.251},"max":{"1min":36.067999999999998,"5min":36.067999999999998,"15min":141.66900000000001},"last":0.98099999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:23:37 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":2.177,"15min":1.798},"min":{"1min":0.499,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.2170000000000001,"5min":36.609999999999999,"15min":36.609999999999999},"last":1.016},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.3119999999999998,"15min":1.72},"min":{"1min":0.65600000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.1719999999999999,"5min":36.856999999999999,"15min":36.856999999999999},"last":0.95199999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.391,"15min":2.1150000000000002},"min":{"1min":0.59999999999999998,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":3.556,"5min":38.191000000000003,"15min":128.77199999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.5720000000000001,"15min":2.0499999999999998},"min":{"1min":0.68300000000000005,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":1.399,"5min":36.281999999999996,"15min":128.72999999999999},"last":0.85799999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.4339999999999999,"15min":1.76},"min":{"1min":0.56399999999999995,"5min":0.34499999999999997,"15min":0.32000000000000001},"max":{"1min":2.9670000000000001,"5min":8.8219999999999992,"15min":57.963000000000001},"last":1.075},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.3240000000000001,"15min":1.8049999999999999},"min":{"1min":0.66000000000000003,"5min":0.33400000000000002,"15min":0.31},"max":{"1min":1.1659999999999999,"5min":10.630000000000001,"15min":57.603000000000002},"last":0.80400000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:23:37 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":0.80100000000000005,"15min":1.7430000000000001},"min":{"1min":0.49199999999999999,"5min":0.32300000000000001,"15min":0.26600000000000001},"max":{"1min":3.5059999999999998,"5min":8.0540000000000003,"15min":13.904},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.97299999999999998,"15min":1.635},"min":{"1min":0.69799999999999995,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":1.8320000000000001,"5min":4.5599999999999996,"15min":22.228000000000002},"last":1.1830000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.454,"15min":1.8149999999999999},"min":{"1min":0.64900000000000002,"5min":0.35499999999999998,"15min":0.32000000000000001},"max":{"1min":1.536,"5min":53.840000000000003,"15min":53.840000000000003},"last":1.0549999999999999},{"interface":"front","average":{"1min":1.163,"5min":1.8,"15min":1.897},"min":{"1min":0.68000000000000005,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.7789999999999999,"5min":50.404000000000003,"15min":50.404000000000003},"last":1.133}]},{"osd":7,"last update":"Mon Oct 9 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.1399999999999999,"15min":1.5649999999999999},"min":{"1min":0.61499999999999999,"5min":0.39200000000000002,"15min":0.34799999999999998},"max":{"1min":3.2450000000000001,"5min":4.649,"15min":12.159000000000001},"last":1.232},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.137,"15min":1.631},"min":{"1min":0.70099999999999996,"5min":0.38600000000000001,"15min":0.33400000000000002},"max":{"1min":1.746,"5min":11.331,"15min":11.331},"last":1.115}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":20531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":10329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":51,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":16,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":10326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":89,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":20515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:24:25.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-09T09:24:25.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:25.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:25.555 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:24:25.641 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_240707/sparse1 2023-10-09T09:24:25.646 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-10-09T09:24:25.646 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-10-09T09:24:25.646 INFO:tasks.workunit.client.0.smithi016.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00441859 s, 237 MB/s 2023-10-09T09:24:25.646 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_240707/sparse2 2023-10-09T09:24:25.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:25.652 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-10-09T09:24:25.652 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-10-09T09:24:25.652 INFO:tasks.workunit.client.0.smithi016.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00529587 s, 198 MB/s 2023-10-09T09:24:25.652 INFO:tasks.workunit.client.0.smithi016.stderr:+ truncate /tmp/rbd_import_export_240707/sparse2 -s 2M 2023-10-09T09:24:25.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:25.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:25.683 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse1 2023-10-09T09:24:25.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[75179]: cluster 2023-10-09T09:24:24.326174+0000 mgr.y (mgr.25421) 1419 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 120 KiB/s rd, 12 KiB/s wr, 159 op/s 2023-10-09T09:24:25.752 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[75179]: cluster 2023-10-09T09:24:24.913523+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:25.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[75179]: cluster 2023-10-09T09:24:24.913824+0000 mon.a (mon.0) 1962 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-09T09:24:25.753 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[75179]: cluster 2023-10-09T09:24:25.011808+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:25.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[77892]: cluster 2023-10-09T09:24:24.326174+0000 mgr.y (mgr.25421) 1419 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 120 KiB/s rd, 12 KiB/s wr, 159 op/s 2023-10-09T09:24:25.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[77892]: cluster 2023-10-09T09:24:24.913523+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:25.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[77892]: cluster 2023-10-09T09:24:24.913824+0000 mon.a (mon.0) 1962 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-09T09:24:25.753 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[77892]: cluster 2023-10-09T09:24:25.011808+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:25.754 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 0% complete...failed. 2023-10-09T09:24:25.754 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T09:24:25.754 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-10-09T09:24:25.754 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_240707/sparse1 2023-10-09T09:24:25.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:25 smithi094 bash[59677]: cluster 2023-10-09T09:24:24.326174+0000 mgr.y (mgr.25421) 1419 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 802 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 120 KiB/s rd, 12 KiB/s wr, 159 op/s 2023-10-09T09:24:25.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:25 smithi094 bash[59677]: cluster 2023-10-09T09:24:24.913523+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:25.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:25 smithi094 bash[59677]: cluster 2023-10-09T09:24:24.913824+0000 mon.a (mon.0) 1962 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-09T09:24:25.755 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:25 smithi094 bash[59677]: cluster 2023-10-09T09:24:25.011808+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:25.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:25.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:25.844 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T09:24:25.845 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:25.845 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:24:25.850 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-10-09T09:24:25.850 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep sparse1 2023-10-09T09:24:25.850 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T09:24:25.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:25.912 INFO:tasks.workunit.client.0.smithi016.stdout:sparse1 2 MiB 2 2023-10-09T09:24:25.913 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse1 2023-10-09T09:24:25.913 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-10-09T09:24:25.913 INFO:tasks.workunit.client.0.smithi016.stderr:+ + rbd info sparse1 2023-10-09T09:24:25.913 INFO:tasks.workunit.client.0.smithi016.stderr:grep block_name_prefix 2023-10-09T09:24:25.913 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:25.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:25.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:25.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:25.973 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.b4e34f36ff2c 2023-10-09T09:24:25.974 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.b4e34f36ff2c 2023-10-09T09:24:25.974 INFO:tasks.workunit.client.0.smithi016.stderr:+ + sed -e s/rbd_data.b4e34f36ff2c\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:24:25.975 INFO:tasks.workunit.client.0.smithi016.stderr:get_image_data_pool sparse1 2023-10-09T09:24:25.975 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-10-09T09:24:25.975 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-10-09T09:24:25.976 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse1 2023-10-09T09:24:25.976 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-10-09T09:24:25.976 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-10-09T09:24:25.999 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:24:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:24:25] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.33.4" 2023-10-09T09:24:26.031 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:26.032 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:26.032 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:26.032 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:26.032 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:26.032 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-10-09T09:24:26.064 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:26.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:26.083 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=1 2023-10-09T09:24:26.083 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 1 2023-10-09T09:24:26.083 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T09:24:26.083 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse1 /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:26.144 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:26.150 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_240707/sparse1 /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:26.150 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_240707/sparse1 /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:26.154 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse1 --format %b 2023-10-09T09:24:26.156 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=2048 2023-10-09T09:24:26.156 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse1.out --format %b 2023-10-09T09:24:26.157 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=2048 2023-10-09T09:24:26.157 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:26.157 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:26.157 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:24:26.157 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:26.159 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse1 2023-10-09T09:24:26.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:26.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:26.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:26.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:26.325 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T09:24:26.330 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse2 2023-10-09T09:24:26.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:26.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:26.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:26.388 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 0% complete...failed. 2023-10-09T09:24:26.388 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T09:24:26.393 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-10-09T09:24:26.393 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_240707/sparse2 2023-10-09T09:24:26.436 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:26.468 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:26.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:26.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:26.487 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T09:24:26.488 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:26.488 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:24:26.492 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-10-09T09:24:26.492 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep sparse2 2023-10-09T09:24:26.492 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T09:24:26.550 INFO:tasks.workunit.client.0.smithi016.stdout:sparse2 2 MiB 2 2023-10-09T09:24:26.551 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse2 2023-10-09T09:24:26.551 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse2 2023-10-09T09:24:26.552 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2 2023-10-09T09:24:26.552 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-10-09T09:24:26.552 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:26.572 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:26.604 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:26.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:26.609 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.b516b8abd030 2023-10-09T09:24:26.610 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.b516b8abd030 2023-10-09T09:24:26.610 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse2 2023-10-09T09:24:26.610 INFO:tasks.workunit.client.0.smithi016.stderr:+ + image=sparse2 2023-10-09T09:24:26.610 INFO:tasks.workunit.client.0.smithi016.stderr:sed -e s/rbd_data.b516b8abd030\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:24:26.611 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-10-09T09:24:26.611 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2 2023-10-09T09:24:26.611 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-10-09T09:24:26.611 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-10-09T09:24:26.666 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:26.667 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:26.667 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:26.667 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:26.667 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-10-09T09:24:26.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:26.717 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=0 2023-10-09T09:24:26.717 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 0 2023-10-09T09:24:26.717 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T09:24:26.717 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse2 /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:26.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:26 smithi094 bash[59677]: audit 2023-10-09T09:24:25.493254+0000 mon.a (mon.0) 1963 : audit 0 from='client.? 172.21.15.16:0/3519945198' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:26.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:26 smithi094 bash[59677]: cluster 2023-10-09T09:24:25.532890+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:26.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:26 smithi094 bash[59677]: audit 2023-10-09T09:24:25.620215+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.16:0/3519945198' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:26.719 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:26 smithi094 bash[59677]: cluster 2023-10-09T09:24:25.632814+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:26.736 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:26.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:26.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[75179]: audit 2023-10-09T09:24:25.493254+0000 mon.a (mon.0) 1963 : audit 0 from='client.? 172.21.15.16:0/3519945198' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[75179]: cluster 2023-10-09T09:24:25.532890+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[75179]: audit 2023-10-09T09:24:25.620215+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.16:0/3519945198' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[75179]: cluster 2023-10-09T09:24:25.632814+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[77892]: audit 2023-10-09T09:24:25.493254+0000 mon.a (mon.0) 1963 : audit 0 from='client.? 172.21.15.16:0/3519945198' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[77892]: cluster 2023-10-09T09:24:25.532890+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[77892]: audit 2023-10-09T09:24:25.620215+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.16:0/3519945198' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:26.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:26 smithi016 bash[77892]: cluster 2023-10-09T09:24:25.632814+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T09:24:26.774 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:26.781 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_240707/sparse2 /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:26.781 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_240707/sparse2 /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:26.785 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse2 --format %b 2023-10-09T09:24:26.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=2048 2023-10-09T09:24:26.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse2.out --format %b 2023-10-09T09:24:26.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=2048 2023-10-09T09:24:26.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:26.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:26.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:24:26.788 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:26.789 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse2 2023-10-09T09:24:26.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:26.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:26.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:26.938 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T09:24:26.944 INFO:tasks.workunit.client.0.smithi016.stderr:+ truncate /tmp/rbd_import_export_240707/sparse1 -s 10M 2023-10-09T09:24:26.945 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T09:24:26.970 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:27.002 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:27.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:27.106 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:27.110 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:27.112 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:27.112 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:24:27.118 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-10-09T09:24:27.118 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep sparse1 2023-10-09T09:24:27.118 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep -Ei (10 MiB|10240k) 2023-10-09T09:24:27.133 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:27.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:27.176 INFO:tasks.workunit.client.0.smithi016.stdout:sparse1 10 MiB 2 2023-10-09T09:24:27.177 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse1 2023-10-09T09:24:27.178 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-10-09T09:24:27.178 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse1 2023-10-09T09:24:27.178 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-10-09T09:24:27.178 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:27.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:27.237 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.8ebfe6b12f65 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr:+ + get_image_data_poolgrep sparse1 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr: rbd_data.8ebfe6b12f65 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.8ebfe6b12f65\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse1 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-10-09T09:24:27.240 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-10-09T09:24:27.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:27.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:27.296 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:27.296 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:27.296 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:27.296 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:27.296 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-10-09T09:24:27.347 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=1 2023-10-09T09:24:27.347 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 1 2023-10-09T09:24:27.347 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T09:24:27.347 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse1 /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:27.373 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:27.407 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:27.410 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:27.416 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_240707/sparse1 /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:27.416 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_240707/sparse1 /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:27.428 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse1 --format %b 2023-10-09T09:24:27.429 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=2048 2023-10-09T09:24:27.429 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse1.out --format %b 2023-10-09T09:24:27.430 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=2048 2023-10-09T09:24:27.430 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:27.430 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:27.431 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:24:27.431 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/sparse1.out 2023-10-09T09:24:27.432 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse1 2023-10-09T09:24:27.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:27 smithi016 bash[75179]: cluster 2023-10-09T09:24:26.327071+0000 mgr.y (mgr.25421) 1420 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 79 KiB/s rd, 7.0 KiB/s wr, 107 op/s 2023-10-09T09:24:27.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:27 smithi016 bash[77892]: cluster 2023-10-09T09:24:26.327071+0000 mgr.y (mgr.25421) 1420 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 79 KiB/s rd, 7.0 KiB/s wr, 107 op/s 2023-10-09T09:24:27.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:27.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:27.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:24:27.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:24:27.596 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T09:24:27.602 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_240707/sparse2 oflag=append conv=notrunc 2023-10-09T09:24:27.612 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-10-09T09:24:27.612 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-10-09T09:24:27.612 INFO:tasks.workunit.client.0.smithi016.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00885154 s, 237 MB/s 2023-10-09T09:24:27.612 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T09:24:27.651 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:27.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:27 smithi094 bash[59677]: cluster 2023-10-09T09:24:26.327071+0000 mgr.y (mgr.25421) 1420 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 79 KiB/s rd, 7.0 KiB/s wr, 107 op/s 2023-10-09T09:24:27.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:27.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:27.729 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:27.730 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:27.730 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:24:27.734 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-10-09T09:24:27.735 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep sparse2 2023-10-09T09:24:27.735 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-09T09:24:27.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:27.785 INFO:tasks.workunit.client.0.smithi016.stdout:sparse2 4 MiB 2 2023-10-09T09:24:27.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse2 2023-10-09T09:24:27.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse2 2023-10-09T09:24:27.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2 2023-10-09T09:24:27.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-10-09T09:24:27.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:27.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:24:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:24:27.839 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.b5703af618f0 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.b5703af618f0 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse2 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:+ + image=sparse2 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:sed -e s/rbd_data.b5703af618f0\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-10-09T09:24:27.841 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-10-09T09:24:27.892 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:27.892 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:27.892 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:27.892 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:27.892 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-10-09T09:24:27.918 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:27.941 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=0 2023-10-09T09:24:27.941 INFO:tasks.workunit.client.0.smithi016.stderr:2 2023-10-09T09:24:27.941 INFO:tasks.workunit.client.0.smithi016.stderr:3 2023-10-09T09:24:27.941 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 0 2 3 2023-10-09T09:24:27.942 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-09T09:24:27.942 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse2 /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:27.949 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:27.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:28.000 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T09:24:28.006 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_240707/sparse2 /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:28.006 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_240707/sparse2 /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:28.011 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse2 --format %b 2023-10-09T09:24:28.012 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=6144 2023-10-09T09:24:28.012 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse2.out --format %b 2023-10-09T09:24:28.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=6144 2023-10-09T09:24:28.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:28.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-10-09T09:24:28.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:24:28.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/sparse2.out 2023-10-09T09:24:28.015 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse2 2023-10-09T09:24:28.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:28.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:28.171 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T09:24:28.177 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-09T09:24:28.177 INFO:tasks.workunit.client.0.smithi016.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T09:24:28.178 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_240707/sparse1 sparse 2023-10-09T09:24:28.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:28.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:28.277 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T09:24:28.278 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:28.279 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:24:28.283 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse 2023-10-09T09:24:28.284 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-10-09T09:24:28.284 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:28.284 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-10-09T09:24:28.285 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:28.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.339 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.8f1992f16aa4 2023-10-09T09:24:28.340 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.8f1992f16aa4 2023-10-09T09:24:28.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-10-09T09:24:28.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-10-09T09:24:28.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.8f1992f16aa4\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:24:28.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-10-09T09:24:28.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:28.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ + awk -F: { print $NF } 2023-10-09T09:24:28.342 INFO:tasks.workunit.client.0.smithi016.stderr:grep data_pool: 2023-10-09T09:24:28.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:28.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:28.397 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:28.397 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:28.397 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:28.397 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:28.398 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-10-09T09:24:28.445 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=1 2023-10-09T09:24:28.446 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 1 2023-10-09T09:24:28.446 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T09:24:28.446 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse 2023-10-09T09:24:28.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:28.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:28.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:28.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.607 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T09:24:28.612 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo zeros import through stdin to sparse image 2023-10-09T09:24:28.612 INFO:tasks.workunit.client.0.smithi016.stdout:zeros import through stdin to sparse image 2023-10-09T09:24:28.612 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T09:24:28.613 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format - sparse 2023-10-09T09:24:28.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:28.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:28.693 INFO:tasks.workunit.client.0.smithi016.stderr:4+0 records in 2023-10-09T09:24:28.693 INFO:tasks.workunit.client.0.smithi016.stderr:4+0 records out 2023-10-09T09:24:28.693 INFO:tasks.workunit.client.0.smithi016.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0803199 s, 52.2 MB/s 2023-10-09T09:24:28.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:28.717 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-10-09T09:24:28.718 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:28.723 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse 2023-10-09T09:24:28.723 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-10-09T09:24:28.723 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:28.724 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-10-09T09:24:28.724 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:28.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:28.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:28.782 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.8f3ac3e75071 2023-10-09T09:24:28.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.8f3ac3e75071 2023-10-09T09:24:28.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-10-09T09:24:28.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-10-09T09:24:28.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.8f3ac3e75071\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:24:28.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-10-09T09:24:28.784 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:28.785 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-10-09T09:24:28.785 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-10-09T09:24:28.842 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:28.842 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:28.842 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:28.842 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:28.843 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-10-09T09:24:28.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:28.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:28.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:28.891 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects= 2023-10-09T09:24:28.891 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 2023-10-09T09:24:28.891 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o = ] 2023-10-09T09:24:28.891 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse 2023-10-09T09:24:28.984 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:29.012 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:29.049 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:29.055 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo zeros export to sparse file 2023-10-09T09:24:29.055 INFO:tasks.workunit.client.0.smithi016.stdout:zeros export to sparse file 2023-10-09T09:24:29.055 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T09:24:29.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:29.116 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:24:29.121 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:29.121 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-10-09T09:24:29.121 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:29.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:24:29.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:24:29.175 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.8f58d7d4762 2023-10-09T09:24:29.175 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T09:24:29.176 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-10-09T09:24:29.176 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-10-09T09:24:29.176 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:29.177 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-10-09T09:24:29.177 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-10-09T09:24:29.233 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:29.233 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:29.233 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:29.233 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:29.233 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados -p rbd put rbd_data.8f58d7d4762.000000000000 - 2023-10-09T09:24:29.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:29.277 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-10-09T09:24:29.277 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-10-09T09:24:29.277 INFO:tasks.workunit.client.0.smithi016.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.100169 s, 41.9 MB/s 2023-10-09T09:24:29.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:29.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:29.304 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse 2023-10-09T09:24:29.305 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-10-09T09:24:29.305 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:29.305 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-10-09T09:24:29.305 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-10-09T09:24:29.357 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.8f58d7d4762 2023-10-09T09:24:29.359 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.8f58d7d4762 2023-10-09T09:24:29.359 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-10-09T09:24:29.359 INFO:tasks.workunit.client.0.smithi016.stderr:+ + image=sparse 2023-10-09T09:24:29.359 INFO:tasks.workunit.client.0.smithi016.stderr:sed -e s/rbd_data.8f58d7d4762\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:24:29.359 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-10-09T09:24:29.359 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-10-09T09:24:29.360 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-10-09T09:24:29.360 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-10-09T09:24:29.385 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:29.407 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-10-09T09:24:29.407 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-10-09T09:24:29.407 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-10-09T09:24:29.407 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-10-09T09:24:29.407 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-10-09T09:24:29.414 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:24:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:24:29.457 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=0 2023-10-09T09:24:29.457 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 0 2023-10-09T09:24:29.457 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T09:24:29.457 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/sparse 2023-10-09T09:24:29.458 INFO:tasks.workunit.client.0.smithi016.stderr:rm: cannot remove '/tmp/rbd_import_export_240707/sparse': No such file or directory 2023-10-09T09:24:29.458 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-10-09T09:24:29.458 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse /tmp/rbd_import_export_240707/sparse 2023-10-09T09:24:29.513 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-10-09T09:24:29.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:29.520 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_240707/sparse --format=%b 2023-10-09T09:24:29.521 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 = 0 ] 2023-10-09T09:24:29.521 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse 2023-10-09T09:24:29.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:24:29.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:24:29.652 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:29.663 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:29 smithi094 bash[59677]: cluster 2023-10-09T09:24:28.327691+0000 mgr.y (mgr.25421) 1421 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 283 KiB/s rd, 169 KiB/s wr, 166 op/s 2023-10-09T09:24:29.669 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-10-09T09:24:29.675 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_240707/sparse /tmp/rbd_import_export_240707/sparse1 /tmp/rbd_import_export_240707/sparse2 /tmp/rbd_import_export_240707/sparse3 2023-10-09T09:24:29.679 INFO:tasks.workunit.client.0.smithi016.stderr:rm: cannot remove '/tmp/rbd_import_export_240707/sparse3': No such file or directory 2023-10-09T09:24:29.679 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-10-09T09:24:29.679 INFO:tasks.workunit.client.0.smithi016.stdout:OK 2023-10-09T09:24:29.680 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo OK 2023-10-09T09:24:29.680 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm -rf /tmp/rbd_import_export_240707 2023-10-09T09:24:29.680 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:24:29.681 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T09:24:29.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T09:24:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:24:29.695 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T09:24:29.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:24:29.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:29 smithi016 bash[75179]: cluster 2023-10-09T09:24:28.327691+0000 mgr.y (mgr.25421) 1421 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 283 KiB/s rd, 169 KiB/s wr, 166 op/s 2023-10-09T09:24:29.748 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:29 smithi016 bash[77892]: cluster 2023-10-09T09:24:28.327691+0000 mgr.y (mgr.25421) 1421 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 283 KiB/s rd, 169 KiB/s wr, 166 op/s 2023-10-09T09:24:29.784 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:29.815 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:24:29.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:24:29.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:29.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:29.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:29.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:29.981 DEBUG:teuthology.parallel:result is None 2023-10-09T09:24:29.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:24:30.039 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:24:30.039 DEBUG:teuthology.orchestra.run.smithi016:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:24:30.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:30.104 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:24:30.104 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:24:30.104 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T09:24:30.104 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T09:24:30.105 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T09:24:30.105 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T09:24:30.105 DEBUG:teuthology.orchestra.run.smithi016:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:24:30.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:24:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:30.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:24:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:30.116 INFO:teuthology.orchestra.run.smithi016.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T09:24:30.116 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:24:30.169 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T09:24:30.169 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T09:24:30.214 DEBUG:teuthology.orchestra.run.smithi094:> sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:30.219 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T09:24:30.233 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:30.246 INFO:teuthology.orchestra.run.smithi094.stderr:Error response from daemon: Cannot kill container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:24:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:24:30.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi094 with status 1: 'sudo docker kill -s 1 ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:24:30.332 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:24:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:30 smithi094 bash[59677]: cluster 2023-10-09T09:24:29.917562+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-09T09:24:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:30 smithi094 bash[59677]: cluster 2023-10-09T09:24:29.928078+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:30 smithi094 bash[59677]: cluster 2023-10-09T09:24:29.929321+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:30 smithi094 bash[59677]: cluster 2023-10-09T09:24:30.328319+0000 mgr.y (mgr.25421) 1422 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 354 KiB/s rd, 282 KiB/s wr, 106 op/s 2023-10-09T09:24:31.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[77892]: cluster 2023-10-09T09:24:29.917562+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-09T09:24:31.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[77892]: cluster 2023-10-09T09:24:29.928078+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:31.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[77892]: cluster 2023-10-09T09:24:29.929321+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:31.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[77892]: cluster 2023-10-09T09:24:30.328319+0000 mgr.y (mgr.25421) 1422 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 354 KiB/s rd, 282 KiB/s wr, 106 op/s 2023-10-09T09:24:31.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[75179]: cluster 2023-10-09T09:24:29.917562+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-09T09:24:31.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[75179]: cluster 2023-10-09T09:24:29.928078+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:31.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[75179]: cluster 2023-10-09T09:24:29.929321+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:31.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:30 smithi016 bash[75179]: cluster 2023-10-09T09:24:30.328319+0000 mgr.y (mgr.25421) 1422 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 354 KiB/s rd, 282 KiB/s wr, 106 op/s 2023-10-09T09:24:32.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:31 smithi094 bash[59677]: cluster 2023-10-09T09:24:31.055944+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:31 smithi094 bash[59677]: cluster 2023-10-09T09:24:31.056995+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:31 smithi094 bash[59677]: cluster 2023-10-09T09:24:31.212396+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:31 smithi016 bash[77892]: cluster 2023-10-09T09:24:31.055944+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:31 smithi016 bash[77892]: cluster 2023-10-09T09:24:31.056995+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:31 smithi016 bash[77892]: cluster 2023-10-09T09:24:31.212396+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:31 smithi016 bash[75179]: cluster 2023-10-09T09:24:31.055944+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:31 smithi016 bash[75179]: cluster 2023-10-09T09:24:31.056995+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:31 smithi016 bash[75179]: cluster 2023-10-09T09:24:31.212396+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T09:24:32.268 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:32.268 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:33.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:32 smithi094 bash[59677]: cluster 2023-10-09T09:24:32.328977+0000 mgr.y (mgr.25421) 1423 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 699 KiB/s rd, 1.1 MiB/s wr, 248 op/s 2023-10-09T09:24:33.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:32 smithi016 bash[75179]: cluster 2023-10-09T09:24:32.328977+0000 mgr.y (mgr.25421) 1423 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 699 KiB/s rd, 1.1 MiB/s wr, 248 op/s 2023-10-09T09:24:33.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:32 smithi016 bash[77892]: cluster 2023-10-09T09:24:32.328977+0000 mgr.y (mgr.25421) 1423 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 699 KiB/s rd, 1.1 MiB/s wr, 248 op/s 2023-10-09T09:24:34.392 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2023-10-09T09:24:34.392 INFO:teuthology.orchestra.run.smithi094.stdout: "ops": [], 2023-10-09T09:24:34.392 INFO:teuthology.orchestra.run.smithi094.stdout: "num_ops": 0 2023-10-09T09:24:34.392 INFO:teuthology.orchestra.run.smithi094.stdout:} 2023-10-09T09:24:34.435 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:34 smithi094 bash[59677]: audit 2023-10-09T09:24:34.266413+0000 mon.a (mon.0) 1966 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:34.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:34 smithi016 bash[75179]: audit 2023-10-09T09:24:34.266413+0000 mon.a (mon.0) 1966 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:34.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:34 smithi016 bash[77892]: audit 2023-10-09T09:24:34.266413+0000 mon.a (mon.0) 1966 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:35.276 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:24:35.427 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:35 smithi016 bash[77892]: cluster 2023-10-09T09:24:34.329664+0000 mgr.y (mgr.25421) 1424 : cluster 0 pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 866 KiB/s rd, 1.2 MiB/s wr, 285 op/s 2023-10-09T09:24:35.428 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:35 smithi016 bash[75179]: cluster 2023-10-09T09:24:34.329664+0000 mgr.y (mgr.25421) 1424 : cluster 0 pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 866 KiB/s rd, 1.2 MiB/s wr, 285 op/s 2023-10-09T09:24:35.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:35 smithi094 bash[59677]: cluster 2023-10-09T09:24:34.329664+0000 mgr.y (mgr.25421) 1424 : cluster 0 pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 866 KiB/s rd, 1.2 MiB/s wr, 285 op/s 2023-10-09T09:24:35.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T09:24:36.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:24:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:24:35] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.33.4" 2023-10-09T09:24:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:36 smithi094 bash[59677]: audit 2023-10-09T09:24:35.836825+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.16:0/2644148065' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:24:36.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:36 smithi094 bash[59677]: audit 2023-10-09T09:24:35.837459+0000 mon.a (mon.0) 1967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:24:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:36 smithi016 bash[75179]: audit 2023-10-09T09:24:35.836825+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.16:0/2644148065' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:24:36.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:36 smithi016 bash[75179]: audit 2023-10-09T09:24:35.837459+0000 mon.a (mon.0) 1967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:24:36.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:36 smithi016 bash[77892]: audit 2023-10-09T09:24:35.836825+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.16:0/2644148065' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:24:36.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:36 smithi016 bash[77892]: audit 2023-10-09T09:24:35.837459+0000 mon.a (mon.0) 1967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T09:24:37.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:37 smithi094 bash[59677]: cluster 2023-10-09T09:24:36.330254+0000 mgr.y (mgr.25421) 1425 : cluster 0 pgmap v1634: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 796 KiB/s rd, 1.1 MiB/s wr, 242 op/s 2023-10-09T09:24:37.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:37 smithi016 bash[75179]: cluster 2023-10-09T09:24:36.330254+0000 mgr.y (mgr.25421) 1425 : cluster 0 pgmap v1634: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 796 KiB/s rd, 1.1 MiB/s wr, 242 op/s 2023-10-09T09:24:37.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:37 smithi016 bash[77892]: cluster 2023-10-09T09:24:36.330254+0000 mgr.y (mgr.25421) 1425 : cluster 0 pgmap v1634: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 796 KiB/s rd, 1.1 MiB/s wr, 242 op/s 2023-10-09T09:24:38.121 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T09:24:39.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:39 smithi094 bash[59677]: cluster 2023-10-09T09:24:38.330795+0000 mgr.y (mgr.25421) 1426 : cluster 0 pgmap v1635: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 657 KiB/s rd, 1.0 MiB/s wr, 194 op/s 2023-10-09T09:24:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:39 smithi016 bash[75179]: cluster 2023-10-09T09:24:38.330795+0000 mgr.y (mgr.25421) 1426 : cluster 0 pgmap v1635: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 657 KiB/s rd, 1.0 MiB/s wr, 194 op/s 2023-10-09T09:24:39.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:39 smithi016 bash[77892]: cluster 2023-10-09T09:24:38.330795+0000 mgr.y (mgr.25421) 1426 : cluster 0 pgmap v1635: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 657 KiB/s rd, 1.0 MiB/s wr, 194 op/s 2023-10-09T09:24:40.056 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:24:40.304 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-10-09T09:24:40.327 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-10-09T09:24:40.356 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-10-09T09:24:40.392 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-10-09T09:24:40.419 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-10-09T09:24:40.444 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-10-09T09:24:40.470 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-10-09T09:24:40.494 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-10-09T09:24:40.497 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.497 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.498 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.498 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.498 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.498 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.498 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.498 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:24:40.935 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:24:41.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:40 smithi016 bash[75179]: audit 2023-10-09T09:24:40.276476+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.16:0/4082157444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:24:41.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:40 smithi016 bash[75179]: cluster 2023-10-09T09:24:40.331357+0000 mgr.y (mgr.25421) 1427 : cluster 0 pgmap v1636: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 532 KiB/s rd, 893 KiB/s wr, 182 op/s 2023-10-09T09:24:41.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:40 smithi016 bash[75179]: audit 2023-10-09T09:24:40.638147+0000 mon.a (mon.0) 1968 : audit 1 from='client.? 172.21.15.16:0/746984325' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:24:41.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:40 smithi094 bash[59677]: audit 2023-10-09T09:24:40.276476+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.16:0/4082157444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:24:41.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:40 smithi094 bash[59677]: cluster 2023-10-09T09:24:40.331357+0000 mgr.y (mgr.25421) 1427 : cluster 0 pgmap v1636: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 532 KiB/s rd, 893 KiB/s wr, 182 op/s 2023-10-09T09:24:41.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:40 smithi094 bash[59677]: audit 2023-10-09T09:24:40.638147+0000 mon.a (mon.0) 1968 : audit 1 from='client.? 172.21.15.16:0/746984325' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:24:41.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: audit 2023-10-09T09:24:40.276476+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.16:0/4082157444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:24:41.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: cluster 2023-10-09T09:24:40.331357+0000 mgr.y (mgr.25421) 1427 : cluster 0 pgmap v1636: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 532 KiB/s rd, 893 KiB/s wr, 182 op/s 2023-10-09T09:24:41.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: audit 2023-10-09T09:24:40.638147+0000 mon.a (mon.0) 1968 : audit 1 from='client.? 172.21.15.16:0/746984325' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:24:42.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:41 smithi094 bash[59677]: cluster 2023-10-09T09:24:40.917698+0000 mon.a (mon.0) 1969 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:24:42.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:41 smithi094 bash[59677]: cluster 2023-10-09T09:24:40.917781+0000 mon.a (mon.0) 1970 : cluster 1 Cluster is now healthy 2023-10-09T09:24:42.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:41 smithi094 bash[59677]: cluster 2023-10-09T09:24:40.926397+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T09:24:42.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:41 smithi094 bash[59677]: audit 2023-10-09T09:24:40.928975+0000 mon.a (mon.0) 1971 : audit 1 from='client.? 172.21.15.16:0/746984325' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:24:42.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:41 smithi094 bash[59677]: cluster 2023-10-09T09:24:40.929059+0000 mon.a (mon.0) 1972 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-09T09:24:42.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[75179]: cluster 2023-10-09T09:24:40.917698+0000 mon.a (mon.0) 1969 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:24:42.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[75179]: cluster 2023-10-09T09:24:40.917781+0000 mon.a (mon.0) 1970 : cluster 1 Cluster is now healthy 2023-10-09T09:24:42.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[75179]: cluster 2023-10-09T09:24:40.926397+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T09:24:42.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[75179]: audit 2023-10-09T09:24:40.928975+0000 mon.a (mon.0) 1971 : audit 1 from='client.? 172.21.15.16:0/746984325' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:24:42.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[75179]: cluster 2023-10-09T09:24:40.929059+0000 mon.a (mon.0) 1972 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-09T09:24:42.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: cluster 2023-10-09T09:24:40.917698+0000 mon.a (mon.0) 1969 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:24:42.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: cluster 2023-10-09T09:24:40.917781+0000 mon.a (mon.0) 1970 : cluster 1 Cluster is now healthy 2023-10-09T09:24:42.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: cluster 2023-10-09T09:24:40.926397+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T09:24:42.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: audit 2023-10-09T09:24:40.928975+0000 mon.a (mon.0) 1971 : audit 1 from='client.? 172.21.15.16:0/746984325' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:24:42.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:41 smithi016 bash[77892]: cluster 2023-10-09T09:24:40.929059+0000 mon.a (mon.0) 1972 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-09T09:24:43.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T09:24:43.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:42 smithi094 bash[59677]: cluster 2023-10-09T09:24:42.230457+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T09:24:43.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:42 smithi094 bash[59677]: cluster 2023-10-09T09:24:42.332106+0000 mgr.y (mgr.25421) 1428 : cluster 0 pgmap v1638: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 256 KiB/s rd, 208 KiB/s wr, 70 op/s 2023-10-09T09:24:43.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:42 smithi016 bash[75179]: cluster 2023-10-09T09:24:42.230457+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T09:24:43.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:42 smithi016 bash[75179]: cluster 2023-10-09T09:24:42.332106+0000 mgr.y (mgr.25421) 1428 : cluster 0 pgmap v1638: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 256 KiB/s rd, 208 KiB/s wr, 70 op/s 2023-10-09T09:24:43.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:42 smithi016 bash[77892]: cluster 2023-10-09T09:24:42.230457+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T09:24:43.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:42 smithi016 bash[77892]: cluster 2023-10-09T09:24:42.332106+0000 mgr.y (mgr.25421) 1428 : cluster 0 pgmap v1638: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 256 KiB/s rd, 208 KiB/s wr, 70 op/s 2023-10-09T09:24:43.694 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:24:43.695 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:24:44.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:24:44.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:24:45.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:45 smithi016 bash[75179]: cluster 2023-10-09T09:24:44.333011+0000 mgr.y (mgr.25421) 1429 : cluster 0 pgmap v1639: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:45.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:45 smithi016 bash[77892]: cluster 2023-10-09T09:24:44.333011+0000 mgr.y (mgr.25421) 1429 : cluster 0 pgmap v1639: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:45.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:45 smithi094 bash[59677]: cluster 2023-10-09T09:24:44.333011+0000 mgr.y (mgr.25421) 1429 : cluster 0 pgmap v1639: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:46.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:24:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:24:45] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.33.4" 2023-10-09T09:24:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:47 smithi094 bash[59677]: cluster 2023-10-09T09:24:46.333954+0000 mgr.y (mgr.25421) 1430 : cluster 0 pgmap v1640: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:47 smithi016 bash[75179]: cluster 2023-10-09T09:24:46.333954+0000 mgr.y (mgr.25421) 1430 : cluster 0 pgmap v1640: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:47.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:47 smithi016 bash[77892]: cluster 2023-10-09T09:24:46.333954+0000 mgr.y (mgr.25421) 1430 : cluster 0 pgmap v1640: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:47.911 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:48.930 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:49.357 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:24:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:49 smithi094 bash[59677]: cluster 2023-10-09T09:24:48.334807+0000 mgr.y (mgr.25421) 1431 : cluster 0 pgmap v1641: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:49 smithi094 bash[59677]: audit 2023-10-09T09:24:49.271601+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:49 smithi094 bash[59677]: audit 2023-10-09T09:24:49.272860+0000 mon.a (mon.0) 1974 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:49 smithi016 bash[75179]: cluster 2023-10-09T09:24:48.334807+0000 mgr.y (mgr.25421) 1431 : cluster 0 pgmap v1641: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:49 smithi016 bash[75179]: audit 2023-10-09T09:24:49.271601+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:49 smithi016 bash[75179]: audit 2023-10-09T09:24:49.272860+0000 mon.a (mon.0) 1974 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:49 smithi016 bash[77892]: cluster 2023-10-09T09:24:48.334807+0000 mgr.y (mgr.25421) 1431 : cluster 0 pgmap v1641: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:49.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:49 smithi016 bash[77892]: audit 2023-10-09T09:24:49.271601+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:49 smithi016 bash[77892]: audit 2023-10-09T09:24:49.272860+0000 mon.a (mon.0) 1974 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:50 smithi094 bash[59677]: cluster 2023-10-09T09:24:50.335628+0000 mgr.y (mgr.25421) 1432 : cluster 0 pgmap v1642: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:50 smithi094 bash[59677]: audit 2023-10-09T09:24:50.500224+0000 mon.a (mon.0) 1975 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:24:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:50 smithi016 bash[77892]: cluster 2023-10-09T09:24:50.335628+0000 mgr.y (mgr.25421) 1432 : cluster 0 pgmap v1642: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:50 smithi016 bash[77892]: audit 2023-10-09T09:24:50.500224+0000 mon.a (mon.0) 1975 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:24:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:50 smithi016 bash[75179]: cluster 2023-10-09T09:24:50.335628+0000 mgr.y (mgr.25421) 1432 : cluster 0 pgmap v1642: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:50 smithi016 bash[75179]: audit 2023-10-09T09:24:50.500224+0000 mon.a (mon.0) 1975 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:24:52.945 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:24:52.946 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:24:53.394 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:53 smithi016 bash[77892]: cluster 2023-10-09T09:24:52.336394+0000 mgr.y (mgr.25421) 1433 : cluster 0 pgmap v1643: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:53.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:53 smithi094 bash[59677]: cluster 2023-10-09T09:24:52.336394+0000 mgr.y (mgr.25421) 1433 : cluster 0 pgmap v1643: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:53.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:53 smithi016 bash[75179]: cluster 2023-10-09T09:24:52.336394+0000 mgr.y (mgr.25421) 1433 : cluster 0 pgmap v1643: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:53.880 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:24:54.621 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a: {} 2023-10-09T09:24:54.640 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b: {} 2023-10-09T09:24:54.653 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c: {} 2023-10-09T09:24:54.655 INFO:teuthology.orchestra.run.smithi016.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T09:24:54.655 INFO:teuthology.orchestra.run.smithi016.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T09:24:54.656 INFO:teuthology.orchestra.run.smithi016.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T09:24:55.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[75179]: cluster 2023-10-09T09:24:54.337171+0000 mgr.y (mgr.25421) 1434 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:55.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[75179]: audit 2023-10-09T09:24:54.604547+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.16:0/630490582' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:24:55.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[75179]: audit 2023-10-09T09:24:55.106677+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.16:0/2996203081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:55.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[75179]: audit 2023-10-09T09:24:55.107338+0000 mon.a (mon.0) 1976 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:55.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[77892]: cluster 2023-10-09T09:24:54.337171+0000 mgr.y (mgr.25421) 1434 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:55.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[77892]: audit 2023-10-09T09:24:54.604547+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.16:0/630490582' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:24:55.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[77892]: audit 2023-10-09T09:24:55.106677+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.16:0/2996203081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:55.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[77892]: audit 2023-10-09T09:24:55.107338+0000 mon.a (mon.0) 1976 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:55.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:55 smithi094 bash[59677]: cluster 2023-10-09T09:24:54.337171+0000 mgr.y (mgr.25421) 1434 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:55.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:55 smithi094 bash[59677]: audit 2023-10-09T09:24:54.604547+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.16:0/630490582' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:24:55.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:55 smithi094 bash[59677]: audit 2023-10-09T09:24:55.106677+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.16:0/2996203081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:55 smithi094 bash[59677]: audit 2023-10-09T09:24:55.107338+0000 mon.a (mon.0) 1976 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:24:55.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:24:55.933 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-10-09T09:24:55.993 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:24:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:24:55] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.33.4" 2023-10-09T09:24:56.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:24:57.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:56 smithi094 bash[59677]: cluster 2023-10-09T09:24:55.923776+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:57.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:56 smithi094 bash[59677]: audit 2023-10-09T09:24:55.925149+0000 mon.a (mon.0) 1977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:24:57.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:56 smithi094 bash[59677]: cluster 2023-10-09T09:24:55.925229+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-09T09:24:57.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:56 smithi094 bash[59677]: cluster 2023-10-09T09:24:56.337828+0000 mgr.y (mgr.25421) 1435 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:57.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:56 smithi094 bash[59677]: audit 2023-10-09T09:24:56.792837+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:57.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:56 smithi094 bash[59677]: audit 2023-10-09T09:24:56.800206+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:57.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[77892]: cluster 2023-10-09T09:24:55.923776+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:57.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[77892]: audit 2023-10-09T09:24:55.925149+0000 mon.a (mon.0) 1977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:24:57.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[77892]: cluster 2023-10-09T09:24:55.925229+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-09T09:24:57.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[77892]: cluster 2023-10-09T09:24:56.337828+0000 mgr.y (mgr.25421) 1435 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:57.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[77892]: audit 2023-10-09T09:24:56.792837+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:57.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[77892]: audit 2023-10-09T09:24:56.800206+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:57.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[75179]: cluster 2023-10-09T09:24:55.923776+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:57.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[75179]: audit 2023-10-09T09:24:55.925149+0000 mon.a (mon.0) 1977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:24:57.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[75179]: cluster 2023-10-09T09:24:55.925229+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-09T09:24:57.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[75179]: cluster 2023-10-09T09:24:56.337828+0000 mgr.y (mgr.25421) 1435 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:57.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[75179]: audit 2023-10-09T09:24:56.792837+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:57.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:56 smithi016 bash[75179]: audit 2023-10-09T09:24:56.800206+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:58.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:57 smithi094 bash[59677]: cluster 2023-10-09T09:24:56.511842+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:58.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:57 smithi094 bash[59677]: cluster 2023-10-09T09:24:57.069058+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:58.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:57 smithi016 bash[75179]: cluster 2023-10-09T09:24:56.511842+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:58.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:57 smithi016 bash[75179]: cluster 2023-10-09T09:24:57.069058+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:58.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:57 smithi016 bash[77892]: cluster 2023-10-09T09:24:56.511842+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:58.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:57 smithi016 bash[77892]: cluster 2023-10-09T09:24:57.069058+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T09:24:58.425 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:24:59.967 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:59 smithi094 bash[59677]: cluster 2023-10-09T09:24:58.338509+0000 mgr.y (mgr.25421) 1436 : cluster 0 pgmap v1647: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:24:59.968 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:59 smithi094 bash[59677]: audit 2023-10-09T09:24:58.707740+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:59.968 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:59 smithi094 bash[59677]: audit 2023-10-09T09:24:58.716950+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:24:59.968 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:59 smithi094 bash[59677]: audit 2023-10-09T09:24:59.189264+0000 mon.a (mon.0) 1983 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:24:59.968 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:59 smithi094 bash[59677]: audit 2023-10-09T09:24:59.190895+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:24:59.968 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:24:59 smithi094 bash[59677]: audit 2023-10-09T09:24:59.202134+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:25:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[77892]: cluster 2023-10-09T09:24:58.338509+0000 mgr.y (mgr.25421) 1436 : cluster 0 pgmap v1647: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[77892]: audit 2023-10-09T09:24:58.707740+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:25:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[77892]: audit 2023-10-09T09:24:58.716950+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:25:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[77892]: audit 2023-10-09T09:24:59.189264+0000 mon.a (mon.0) 1983 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:25:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[77892]: audit 2023-10-09T09:24:59.190895+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:25:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[77892]: audit 2023-10-09T09:24:59.202134+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:25:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[75179]: cluster 2023-10-09T09:24:58.338509+0000 mgr.y (mgr.25421) 1436 : cluster 0 pgmap v1647: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[75179]: audit 2023-10-09T09:24:58.707740+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:25:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[75179]: audit 2023-10-09T09:24:58.716950+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:25:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[75179]: audit 2023-10-09T09:24:59.189264+0000 mon.a (mon.0) 1983 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:25:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[75179]: audit 2023-10-09T09:24:59.190895+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:25:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:24:59 smithi016 bash[75179]: audit 2023-10-09T09:24:59.202134+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:00 smithi094 bash[59677]: cluster 2023-10-09T09:25:00.339399+0000 mgr.y (mgr.25421) 1437 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:01.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:00 smithi016 bash[75179]: cluster 2023-10-09T09:25:00.339399+0000 mgr.y (mgr.25421) 1437 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:01.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:00 smithi016 bash[77892]: cluster 2023-10-09T09:25:00.339399+0000 mgr.y (mgr.25421) 1437 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:01.540 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:02.234 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:02.234 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:03.199 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:25:03.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:03 smithi094 bash[59677]: cluster 2023-10-09T09:25:02.340000+0000 mgr.y (mgr.25421) 1438 : cluster 0 pgmap v1649: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:03.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:03 smithi016 bash[75179]: cluster 2023-10-09T09:25:02.340000+0000 mgr.y (mgr.25421) 1438 : cluster 0 pgmap v1649: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:03.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:03 smithi016 bash[77892]: cluster 2023-10-09T09:25:02.340000+0000 mgr.y (mgr.25421) 1438 : cluster 0 pgmap v1649: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:04.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:04 smithi016 bash[75179]: audit 2023-10-09T09:25:04.268011+0000 mon.a (mon.0) 1986 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:04.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:04 smithi016 bash[77892]: audit 2023-10-09T09:25:04.268011+0000 mon.a (mon.0) 1986 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:04.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:04 smithi094 bash[59677]: audit 2023-10-09T09:25:04.268011+0000 mon.a (mon.0) 1986 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:05.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:05 smithi016 bash[77892]: cluster 2023-10-09T09:25:04.340814+0000 mgr.y (mgr.25421) 1439 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:05.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:05 smithi016 bash[75179]: cluster 2023-10-09T09:25:04.340814+0000 mgr.y (mgr.25421) 1439 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:05.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:05 smithi094 bash[59677]: cluster 2023-10-09T09:25:04.340814+0000 mgr.y (mgr.25421) 1439 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:25:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:25:05] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-10-09T09:25:07.423 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-10-09T09:25:07.426 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:25:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:07 smithi094 bash[59677]: cluster 2023-10-09T09:25:06.341507+0000 mgr.y (mgr.25421) 1440 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:07 smithi094 bash[59677]: audit 2023-10-09T09:25:07.141350+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.16:0/655838797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:07 smithi094 bash[59677]: audit 2023-10-09T09:25:07.142093+0000 mon.a (mon.0) 1987 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:07 smithi094 bash[59677]: audit 2023-10-09T09:25:07.166841+0000 mon.a (mon.0) 1988 : audit 1 from='client.? 172.21.15.16:0/2789287703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[75179]: cluster 2023-10-09T09:25:06.341507+0000 mgr.y (mgr.25421) 1440 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[75179]: audit 2023-10-09T09:25:07.141350+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.16:0/655838797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[75179]: audit 2023-10-09T09:25:07.142093+0000 mon.a (mon.0) 1987 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[75179]: audit 2023-10-09T09:25:07.166841+0000 mon.a (mon.0) 1988 : audit 1 from='client.? 172.21.15.16:0/2789287703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[77892]: cluster 2023-10-09T09:25:06.341507+0000 mgr.y (mgr.25421) 1440 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[77892]: audit 2023-10-09T09:25:07.141350+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.16:0/655838797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[77892]: audit 2023-10-09T09:25:07.142093+0000 mon.a (mon.0) 1987 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:07.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:07 smithi016 bash[77892]: audit 2023-10-09T09:25:07.166841+0000 mon.a (mon.0) 1988 : audit 1 from='client.? 172.21.15.16:0/2789287703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:07.966 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:25:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:08 smithi094 bash[59677]: cluster 2023-10-09T09:25:07.412745+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:08 smithi094 bash[59677]: audit 2023-10-09T09:25:07.416317+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:25:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:08 smithi094 bash[59677]: audit 2023-10-09T09:25:07.416446+0000 mon.a (mon.0) 1990 : audit 1 from='client.? 172.21.15.16:0/2789287703' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:08 smithi094 bash[59677]: cluster 2023-10-09T09:25:07.416539+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-09T09:25:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:08 smithi094 bash[59677]: cluster 2023-10-09T09:25:07.731631+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:08.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[75179]: cluster 2023-10-09T09:25:07.412745+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:08.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[75179]: audit 2023-10-09T09:25:07.416317+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:25:08.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[75179]: audit 2023-10-09T09:25:07.416446+0000 mon.a (mon.0) 1990 : audit 1 from='client.? 172.21.15.16:0/2789287703' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:08.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[75179]: cluster 2023-10-09T09:25:07.416539+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-09T09:25:08.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[75179]: cluster 2023-10-09T09:25:07.731631+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[77892]: cluster 2023-10-09T09:25:07.412745+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[77892]: audit 2023-10-09T09:25:07.416317+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:25:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[77892]: audit 2023-10-09T09:25:07.416446+0000 mon.a (mon.0) 1990 : audit 1 from='client.? 172.21.15.16:0/2789287703' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[77892]: cluster 2023-10-09T09:25:07.416539+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-09T09:25:08.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:08 smithi016 bash[77892]: cluster 2023-10-09T09:25:07.731631+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:08.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T09:25:09.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:09 smithi094 bash[59677]: cluster 2023-10-09T09:25:07.721112+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:09 smithi094 bash[59677]: cluster 2023-10-09T09:25:08.042704+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:09 smithi094 bash[59677]: cluster 2023-10-09T09:25:08.342175+0000 mgr.y (mgr.25421) 1441 : cluster 0 pgmap v1653: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:09 smithi016 bash[75179]: cluster 2023-10-09T09:25:07.721112+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:09 smithi016 bash[75179]: cluster 2023-10-09T09:25:08.042704+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:09 smithi016 bash[75179]: cluster 2023-10-09T09:25:08.342175+0000 mgr.y (mgr.25421) 1441 : cluster 0 pgmap v1653: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:09 smithi016 bash[77892]: cluster 2023-10-09T09:25:07.721112+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:09 smithi016 bash[77892]: cluster 2023-10-09T09:25:08.042704+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T09:25:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:09 smithi016 bash[77892]: cluster 2023-10-09T09:25:08.342175+0000 mgr.y (mgr.25421) 1441 : cluster 0 pgmap v1653: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:11.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:10 smithi094 bash[59677]: cluster 2023-10-09T09:25:10.342928+0000 mgr.y (mgr.25421) 1442 : cluster 0 pgmap v1654: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:11.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:10 smithi016 bash[75179]: cluster 2023-10-09T09:25:10.342928+0000 mgr.y (mgr.25421) 1442 : cluster 0 pgmap v1654: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:11.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:10 smithi016 bash[77892]: cluster 2023-10-09T09:25:10.342928+0000 mgr.y (mgr.25421) 1442 : cluster 0 pgmap v1654: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:11.326 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:11.326 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:12.247 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:25:13.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:13 smithi094 bash[59677]: cluster 2023-10-09T09:25:12.343756+0000 mgr.y (mgr.25421) 1443 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:13 smithi016 bash[75179]: cluster 2023-10-09T09:25:12.343756+0000 mgr.y (mgr.25421) 1443 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:13 smithi016 bash[77892]: cluster 2023-10-09T09:25:12.343756+0000 mgr.y (mgr.25421) 1443 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:13.754 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:15 smithi016 bash[75179]: cluster 2023-10-09T09:25:14.344596+0000 mgr.y (mgr.25421) 1444 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:15.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:15 smithi016 bash[77892]: cluster 2023-10-09T09:25:14.344596+0000 mgr.y (mgr.25421) 1444 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:15.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:15 smithi094 bash[59677]: cluster 2023-10-09T09:25:14.344596+0000 mgr.y (mgr.25421) 1444 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:25:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:25:15] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-10-09T09:25:17.029 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:25:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:17 smithi094 bash[59677]: cluster 2023-10-09T09:25:16.345294+0000 mgr.y (mgr.25421) 1445 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:17 smithi016 bash[75179]: cluster 2023-10-09T09:25:16.345294+0000 mgr.y (mgr.25421) 1445 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:17 smithi016 bash[77892]: cluster 2023-10-09T09:25:16.345294+0000 mgr.y (mgr.25421) 1445 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:18.429 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T09:25:18.656 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:18 smithi094 bash[59677]: audit 2023-10-09T09:25:17.789026+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.16:0/1224542679' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:18.657 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:18 smithi094 bash[59677]: audit 2023-10-09T09:25:17.789686+0000 mon.a (mon.0) 1992 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:18.707 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:18 smithi016 bash[75179]: audit 2023-10-09T09:25:17.789026+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.16:0/1224542679' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:18.707 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:18 smithi016 bash[75179]: audit 2023-10-09T09:25:17.789686+0000 mon.a (mon.0) 1992 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:18.707 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:18 smithi016 bash[77892]: audit 2023-10-09T09:25:17.789026+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.16:0/1224542679' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:18.707 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:18 smithi016 bash[77892]: audit 2023-10-09T09:25:17.789686+0000 mon.a (mon.0) 1992 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:19.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T09:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:19 smithi094 bash[59677]: cluster 2023-10-09T09:25:18.346105+0000 mgr.y (mgr.25421) 1446 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:19 smithi094 bash[59677]: cluster 2023-10-09T09:25:18.418213+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:19 smithi094 bash[59677]: cluster 2023-10-09T09:25:18.418527+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:19 smithi094 bash[59677]: audit 2023-10-09T09:25:18.420252+0000 mon.a (mon.0) 1993 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:19 smithi094 bash[59677]: cluster 2023-10-09T09:25:18.420346+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-09T09:25:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:19 smithi094 bash[59677]: audit 2023-10-09T09:25:19.268366+0000 mon.a (mon.0) 1995 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[75179]: cluster 2023-10-09T09:25:18.346105+0000 mgr.y (mgr.25421) 1446 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[75179]: cluster 2023-10-09T09:25:18.418213+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[75179]: cluster 2023-10-09T09:25:18.418527+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[75179]: audit 2023-10-09T09:25:18.420252+0000 mon.a (mon.0) 1993 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[75179]: cluster 2023-10-09T09:25:18.420346+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-09T09:25:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[75179]: audit 2023-10-09T09:25:19.268366+0000 mon.a (mon.0) 1995 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[77892]: cluster 2023-10-09T09:25:18.346105+0000 mgr.y (mgr.25421) 1446 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[77892]: cluster 2023-10-09T09:25:18.418213+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[77892]: cluster 2023-10-09T09:25:18.418527+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[77892]: audit 2023-10-09T09:25:18.420252+0000 mon.a (mon.0) 1993 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[77892]: cluster 2023-10-09T09:25:18.420346+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-09T09:25:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:19 smithi016 bash[77892]: audit 2023-10-09T09:25:19.268366+0000 mon.a (mon.0) 1995 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:20.601 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:20.601 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:21.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:20 smithi094 bash[59677]: cluster 2023-10-09T09:25:19.690114+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:21.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:20 smithi094 bash[59677]: cluster 2023-10-09T09:25:20.346886+0000 mgr.y (mgr.25421) 1447 : cluster 0 pgmap v1660: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:20 smithi016 bash[75179]: cluster 2023-10-09T09:25:19.690114+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:20 smithi016 bash[75179]: cluster 2023-10-09T09:25:20.346886+0000 mgr.y (mgr.25421) 1447 : cluster 0 pgmap v1660: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:20 smithi016 bash[77892]: cluster 2023-10-09T09:25:19.690114+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T09:25:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:20 smithi016 bash[77892]: cluster 2023-10-09T09:25:20.346886+0000 mgr.y (mgr.25421) 1447 : cluster 0 pgmap v1660: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:21.457 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:25:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:23 smithi094 bash[59677]: cluster 2023-10-09T09:25:22.347900+0000 mgr.y (mgr.25421) 1448 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:23 smithi016 bash[75179]: cluster 2023-10-09T09:25:22.347900+0000 mgr.y (mgr.25421) 1448 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:23 smithi016 bash[77892]: cluster 2023-10-09T09:25:22.347900+0000 mgr.y (mgr.25421) 1448 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:24.243 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:25.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:25 smithi016 bash[77892]: cluster 2023-10-09T09:25:24.348982+0000 mgr.y (mgr.25421) 1449 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:25.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:25 smithi016 bash[75179]: cluster 2023-10-09T09:25:24.348982+0000 mgr.y (mgr.25421) 1449 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:25.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:25 smithi094 bash[59677]: cluster 2023-10-09T09:25:24.348982+0000 mgr.y (mgr.25421) 1449 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:26.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:25:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:25:25] "GET /metrics HTTP/1.1" 200 34099 "" "Prometheus/2.33.4" 2023-10-09T09:25:26.232 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:25:27.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:27 smithi016 bash[75179]: cluster 2023-10-09T09:25:26.350016+0000 mgr.y (mgr.25421) 1450 : cluster 0 pgmap v1663: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:27.686 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:27 smithi016 bash[77892]: cluster 2023-10-09T09:25:26.350016+0000 mgr.y (mgr.25421) 1450 : cluster 0 pgmap v1663: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:27.721 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:27 smithi094 bash[59677]: cluster 2023-10-09T09:25:26.350016+0000 mgr.y (mgr.25421) 1450 : cluster 0 pgmap v1663: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:28.431 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T09:25:28.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:28 smithi094 bash[59677]: audit 2023-10-09T09:25:28.081947+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.16:0/1108056704' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:28.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:28 smithi094 bash[59677]: audit 2023-10-09T09:25:28.082593+0000 mon.a (mon.0) 1996 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:28 smithi016 bash[75179]: audit 2023-10-09T09:25:28.081947+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.16:0/1108056704' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:28 smithi016 bash[75179]: audit 2023-10-09T09:25:28.082593+0000 mon.a (mon.0) 1996 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:28.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:28 smithi016 bash[77892]: audit 2023-10-09T09:25:28.081947+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.16:0/1108056704' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:28.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:28 smithi016 bash[77892]: audit 2023-10-09T09:25:28.082593+0000 mon.a (mon.0) 1996 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:29.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T09:25:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:29 smithi094 bash[59677]: cluster 2023-10-09T09:25:28.350946+0000 mgr.y (mgr.25421) 1451 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:29 smithi094 bash[59677]: cluster 2023-10-09T09:25:28.423588+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:29 smithi094 bash[59677]: audit 2023-10-09T09:25:28.425095+0000 mon.a (mon.0) 1997 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:29 smithi094 bash[59677]: cluster 2023-10-09T09:25:28.425246+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-09T09:25:29.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:29 smithi094 bash[59677]: cluster 2023-10-09T09:25:29.009484+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[75179]: cluster 2023-10-09T09:25:28.350946+0000 mgr.y (mgr.25421) 1451 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[75179]: cluster 2023-10-09T09:25:28.423588+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[75179]: audit 2023-10-09T09:25:28.425095+0000 mon.a (mon.0) 1997 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[75179]: cluster 2023-10-09T09:25:28.425246+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-09T09:25:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[75179]: cluster 2023-10-09T09:25:29.009484+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[77892]: cluster 2023-10-09T09:25:28.350946+0000 mgr.y (mgr.25421) 1451 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:29.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[77892]: cluster 2023-10-09T09:25:28.423588+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:29.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[77892]: audit 2023-10-09T09:25:28.425095+0000 mon.a (mon.0) 1997 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:29.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[77892]: cluster 2023-10-09T09:25:28.425246+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-09T09:25:29.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:29 smithi016 bash[77892]: cluster 2023-10-09T09:25:29.009484+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:29.751 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:29.751 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:30.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:25:30.680 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:30 smithi094 bash[59677]: cluster 2023-10-09T09:25:28.795761+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:30.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:30 smithi016 bash[75179]: cluster 2023-10-09T09:25:28.795761+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:30.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:30 smithi016 bash[77892]: cluster 2023-10-09T09:25:28.795761+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T09:25:31.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:31 smithi094 bash[59677]: cluster 2023-10-09T09:25:30.351761+0000 mgr.y (mgr.25421) 1452 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:31.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:31 smithi016 bash[75179]: cluster 2023-10-09T09:25:30.351761+0000 mgr.y (mgr.25421) 1452 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:31.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:31 smithi016 bash[77892]: cluster 2023-10-09T09:25:30.351761+0000 mgr.y (mgr.25421) 1452 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:33.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:33 smithi094 bash[59677]: cluster 2023-10-09T09:25:32.352583+0000 mgr.y (mgr.25421) 1453 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:33 smithi016 bash[75179]: cluster 2023-10-09T09:25:32.352583+0000 mgr.y (mgr.25421) 1453 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:33.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:33 smithi016 bash[77892]: cluster 2023-10-09T09:25:32.352583+0000 mgr.y (mgr.25421) 1453 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:34.174 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:34.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:34 smithi016 bash[75179]: audit 2023-10-09T09:25:34.268789+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:34.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:34 smithi016 bash[77892]: audit 2023-10-09T09:25:34.268789+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:34.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:34 smithi094 bash[59677]: audit 2023-10-09T09:25:34.268789+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:35.447 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:25:35.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:35 smithi016 bash[75179]: cluster 2023-10-09T09:25:34.353446+0000 mgr.y (mgr.25421) 1454 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:35.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:35 smithi016 bash[77892]: cluster 2023-10-09T09:25:34.353446+0000 mgr.y (mgr.25421) 1454 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:35.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:35 smithi094 bash[59677]: cluster 2023-10-09T09:25:34.353446+0000 mgr.y (mgr.25421) 1454 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:36.010 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:25:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:25:35] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.33.4" 2023-10-09T09:25:37.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:37 smithi094 bash[59677]: cluster 2023-10-09T09:25:36.354384+0000 mgr.y (mgr.25421) 1455 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:37.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:37 smithi016 bash[75179]: cluster 2023-10-09T09:25:36.354384+0000 mgr.y (mgr.25421) 1455 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:37.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:37 smithi016 bash[77892]: cluster 2023-10-09T09:25:36.354384+0000 mgr.y (mgr.25421) 1455 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:38.465 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T09:25:38.736 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:38 smithi094 bash[59677]: audit 2023-10-09T09:25:38.213577+0000 mon.a (mon.0) 2000 : audit 1 from='client.? 172.21.15.16:0/4057659145' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:38.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:38 smithi016 bash[75179]: audit 2023-10-09T09:25:38.213577+0000 mon.a (mon.0) 2000 : audit 1 from='client.? 172.21.15.16:0/4057659145' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:38.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:38 smithi016 bash[77892]: audit 2023-10-09T09:25:38.213577+0000 mon.a (mon.0) 2000 : audit 1 from='client.? 172.21.15.16:0/4057659145' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:38.915 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:38.916 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:39.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:25:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:39 smithi094 bash[59677]: cluster 2023-10-09T09:25:38.355230+0000 mgr.y (mgr.25421) 1456 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:39 smithi094 bash[59677]: cluster 2023-10-09T09:25:38.456205+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T09:25:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:39 smithi094 bash[59677]: cluster 2023-10-09T09:25:38.456628+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T09:25:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:39 smithi094 bash[59677]: audit 2023-10-09T09:25:38.458233+0000 mon.a (mon.0) 2001 : audit 1 from='client.? 172.21.15.16:0/4057659145' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:39 smithi094 bash[59677]: cluster 2023-10-09T09:25:38.458338+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-09T09:25:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[75179]: cluster 2023-10-09T09:25:38.355230+0000 mgr.y (mgr.25421) 1456 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[75179]: cluster 2023-10-09T09:25:38.456205+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T09:25:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[75179]: cluster 2023-10-09T09:25:38.456628+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T09:25:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[75179]: audit 2023-10-09T09:25:38.458233+0000 mon.a (mon.0) 2001 : audit 1 from='client.? 172.21.15.16:0/4057659145' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[75179]: cluster 2023-10-09T09:25:38.458338+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-09T09:25:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[77892]: cluster 2023-10-09T09:25:38.355230+0000 mgr.y (mgr.25421) 1456 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[77892]: cluster 2023-10-09T09:25:38.456205+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T09:25:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[77892]: cluster 2023-10-09T09:25:38.456628+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T09:25:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[77892]: audit 2023-10-09T09:25:38.458233+0000 mon.a (mon.0) 2001 : audit 1 from='client.? 172.21.15.16:0/4057659145' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:39 smithi016 bash[77892]: cluster 2023-10-09T09:25:38.458338+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-09T09:25:39.804 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:25:41.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:40 smithi094 bash[59677]: cluster 2023-10-09T09:25:40.356191+0000 mgr.y (mgr.25421) 1457 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:41.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:40 smithi016 bash[75179]: cluster 2023-10-09T09:25:40.356191+0000 mgr.y (mgr.25421) 1457 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:41.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:40 smithi016 bash[77892]: cluster 2023-10-09T09:25:40.356191+0000 mgr.y (mgr.25421) 1457 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:43.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:43 smithi094 bash[59677]: cluster 2023-10-09T09:25:42.357192+0000 mgr.y (mgr.25421) 1458 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:43.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:43 smithi016 bash[75179]: cluster 2023-10-09T09:25:42.357192+0000 mgr.y (mgr.25421) 1458 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:43.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:43 smithi016 bash[77892]: cluster 2023-10-09T09:25:42.357192+0000 mgr.y (mgr.25421) 1458 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:44.297 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:44.630 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:25:45.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:45 smithi016 bash[75179]: cluster 2023-10-09T09:25:44.358082+0000 mgr.y (mgr.25421) 1459 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:45.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:45 smithi016 bash[77892]: cluster 2023-10-09T09:25:44.358082+0000 mgr.y (mgr.25421) 1459 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:45.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:45 smithi094 bash[59677]: cluster 2023-10-09T09:25:44.358082+0000 mgr.y (mgr.25421) 1459 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:46.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:25:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:25:45] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.33.4" 2023-10-09T09:25:47.692 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:47 smithi016 bash[75179]: cluster 2023-10-09T09:25:46.359104+0000 mgr.y (mgr.25421) 1460 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:47.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:47 smithi016 bash[77892]: cluster 2023-10-09T09:25:46.359104+0000 mgr.y (mgr.25421) 1460 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:47.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:47 smithi094 bash[59677]: cluster 2023-10-09T09:25:46.359104+0000 mgr.y (mgr.25421) 1460 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:48.161 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:48.161 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:48.443 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:25:48.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:48 smithi094 bash[59677]: audit 2023-10-09T09:25:48.102533+0000 mon.b (mon.1) 34 : audit 1 from='client.? 172.21.15.16:0/2819057414' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:48.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:48 smithi094 bash[59677]: audit 2023-10-09T09:25:48.102636+0000 mon.a (mon.0) 2003 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:48.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:48 smithi016 bash[75179]: audit 2023-10-09T09:25:48.102533+0000 mon.b (mon.1) 34 : audit 1 from='client.? 172.21.15.16:0/2819057414' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:48.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:48 smithi016 bash[75179]: audit 2023-10-09T09:25:48.102636+0000 mon.a (mon.0) 2003 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:48.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:48 smithi016 bash[77892]: audit 2023-10-09T09:25:48.102533+0000 mon.b (mon.1) 34 : audit 1 from='client.? 172.21.15.16:0/2819057414' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:48.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:48 smithi016 bash[77892]: audit 2023-10-09T09:25:48.102636+0000 mon.a (mon.0) 2003 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:49.162 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:25:49.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T09:25:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:49 smithi094 bash[59677]: cluster 2023-10-09T09:25:48.360061+0000 mgr.y (mgr.25421) 1461 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:49 smithi094 bash[59677]: cluster 2023-10-09T09:25:48.432851+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T09:25:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:49 smithi094 bash[59677]: audit 2023-10-09T09:25:48.436447+0000 mon.a (mon.0) 2004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:25:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:49 smithi094 bash[59677]: cluster 2023-10-09T09:25:48.436541+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-09T09:25:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:49 smithi094 bash[59677]: audit 2023-10-09T09:25:49.269241+0000 mon.a (mon.0) 2006 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[75179]: cluster 2023-10-09T09:25:48.360061+0000 mgr.y (mgr.25421) 1461 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[75179]: cluster 2023-10-09T09:25:48.432851+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[75179]: audit 2023-10-09T09:25:48.436447+0000 mon.a (mon.0) 2004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[75179]: cluster 2023-10-09T09:25:48.436541+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[75179]: audit 2023-10-09T09:25:49.269241+0000 mon.a (mon.0) 2006 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[77892]: cluster 2023-10-09T09:25:48.360061+0000 mgr.y (mgr.25421) 1461 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[77892]: cluster 2023-10-09T09:25:48.432851+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[77892]: audit 2023-10-09T09:25:48.436447+0000 mon.a (mon.0) 2004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[77892]: cluster 2023-10-09T09:25:48.436541+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-09T09:25:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:49 smithi016 bash[77892]: audit 2023-10-09T09:25:49.269241+0000 mon.a (mon.0) 2006 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:51.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:50 smithi094 bash[59677]: cluster 2023-10-09T09:25:49.471471+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T09:25:51.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:50 smithi094 bash[59677]: cluster 2023-10-09T09:25:50.360954+0000 mgr.y (mgr.25421) 1462 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:50 smithi016 bash[75179]: cluster 2023-10-09T09:25:49.471471+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T09:25:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:50 smithi016 bash[75179]: cluster 2023-10-09T09:25:50.360954+0000 mgr.y (mgr.25421) 1462 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:50 smithi016 bash[77892]: cluster 2023-10-09T09:25:49.471471+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T09:25:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:50 smithi016 bash[77892]: cluster 2023-10-09T09:25:50.360954+0000 mgr.y (mgr.25421) 1462 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:53.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:53 smithi094 bash[59677]: cluster 2023-10-09T09:25:52.361724+0000 mgr.y (mgr.25421) 1463 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:53.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:53 smithi016 bash[75179]: cluster 2023-10-09T09:25:52.361724+0000 mgr.y (mgr.25421) 1463 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:53.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:53 smithi016 bash[77892]: cluster 2023-10-09T09:25:52.361724+0000 mgr.y (mgr.25421) 1463 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:53.944 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:25:54.306 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:55.690 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:55 smithi016 bash[75179]: cluster 2023-10-09T09:25:54.362646+0000 mgr.y (mgr.25421) 1464 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:55.690 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:55 smithi016 bash[77892]: cluster 2023-10-09T09:25:54.362646+0000 mgr.y (mgr.25421) 1464 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:55.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:55 smithi094 bash[59677]: cluster 2023-10-09T09:25:54.362646+0000 mgr.y (mgr.25421) 1464 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:56.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:25:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:25:55] "GET /metrics HTTP/1.1" 200 34098 "" "Prometheus/2.33.4" 2023-10-09T09:25:57.401 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:57.401 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:57.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:57 smithi016 bash[77892]: cluster 2023-10-09T09:25:56.363634+0000 mgr.y (mgr.25421) 1465 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:57.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:57 smithi016 bash[75179]: cluster 2023-10-09T09:25:56.363634+0000 mgr.y (mgr.25421) 1465 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:57.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:57 smithi094 bash[59677]: cluster 2023-10-09T09:25:56.363634+0000 mgr.y (mgr.25421) 1465 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:58.333 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:25:58.446 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T09:25:58.693 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:58 smithi016 bash[75179]: audit 2023-10-09T09:25:58.117562+0000 mon.a (mon.0) 2007 : audit 1 from='client.? 172.21.15.16:0/2694267805' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:58.693 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:58 smithi016 bash[77892]: audit 2023-10-09T09:25:58.117562+0000 mon.a (mon.0) 2007 : audit 1 from='client.? 172.21.15.16:0/2694267805' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:58.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:58 smithi094 bash[59677]: audit 2023-10-09T09:25:58.117562+0000 mon.a (mon.0) 2007 : audit 1 from='client.? 172.21.15.16:0/2694267805' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:25:59.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T09:25:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:59 smithi094 bash[59677]: cluster 2023-10-09T09:25:58.364561+0000 mgr.y (mgr.25421) 1466 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:59 smithi094 bash[59677]: cluster 2023-10-09T09:25:58.436997+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T09:25:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:59 smithi094 bash[59677]: audit 2023-10-09T09:25:58.438819+0000 mon.a (mon.0) 2008 : audit 1 from='client.? 172.21.15.16:0/2694267805' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:25:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:59 smithi094 bash[59677]: cluster 2023-10-09T09:25:58.438904+0000 mon.a (mon.0) 2009 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-09T09:25:59.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:59 smithi094 bash[59677]: cluster 2023-10-09T09:25:58.749001+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T09:25:59.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:25:59 smithi094 bash[59677]: audit 2023-10-09T09:25:59.286863+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:25:59.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[75179]: cluster 2023-10-09T09:25:58.364561+0000 mgr.y (mgr.25421) 1466 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:59.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[75179]: cluster 2023-10-09T09:25:58.436997+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T09:25:59.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[75179]: audit 2023-10-09T09:25:58.438819+0000 mon.a (mon.0) 2008 : audit 1 from='client.? 172.21.15.16:0/2694267805' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[75179]: cluster 2023-10-09T09:25:58.438904+0000 mon.a (mon.0) 2009 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[75179]: cluster 2023-10-09T09:25:58.749001+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[75179]: audit 2023-10-09T09:25:59.286863+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[77892]: cluster 2023-10-09T09:25:58.364561+0000 mgr.y (mgr.25421) 1466 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[77892]: cluster 2023-10-09T09:25:58.436997+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[77892]: audit 2023-10-09T09:25:58.438819+0000 mon.a (mon.0) 2008 : audit 1 from='client.? 172.21.15.16:0/2694267805' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[77892]: cluster 2023-10-09T09:25:58.438904+0000 mon.a (mon.0) 2009 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[77892]: cluster 2023-10-09T09:25:58.749001+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T09:25:59.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:25:59 smithi016 bash[77892]: audit 2023-10-09T09:25:59.286863+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:26:01.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:00 smithi094 bash[59677]: cluster 2023-10-09T09:26:00.365430+0000 mgr.y (mgr.25421) 1467 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:01.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:00 smithi016 bash[75179]: cluster 2023-10-09T09:26:00.365430+0000 mgr.y (mgr.25421) 1467 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:01.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:00 smithi016 bash[77892]: cluster 2023-10-09T09:26:00.365430+0000 mgr.y (mgr.25421) 1467 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:03.111 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:26:03.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:03 smithi094 bash[59677]: cluster 2023-10-09T09:26:02.366194+0000 mgr.y (mgr.25421) 1468 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:03 smithi016 bash[75179]: cluster 2023-10-09T09:26:02.366194+0000 mgr.y (mgr.25421) 1468 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:03.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:03 smithi016 bash[77892]: cluster 2023-10-09T09:26:02.366194+0000 mgr.y (mgr.25421) 1468 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:04.182 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:04.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:04 smithi016 bash[77892]: audit 2023-10-09T09:26:04.269825+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:04.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:04 smithi016 bash[75179]: audit 2023-10-09T09:26:04.269825+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:04.732 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:04 smithi094 bash[59677]: audit 2023-10-09T09:26:04.269825+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:05.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[77892]: cluster 2023-10-09T09:26:04.366743+0000 mgr.y (mgr.25421) 1469 : cluster 0 pgmap v1686: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:05.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[77892]: audit 2023-10-09T09:26:05.386850+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[77892]: audit 2023-10-09T09:26:05.392353+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[77892]: audit 2023-10-09T09:26:05.396362+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[77892]: audit 2023-10-09T09:26:05.402978+0000 mon.a (mon.0) 2015 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[75179]: cluster 2023-10-09T09:26:04.366743+0000 mgr.y (mgr.25421) 1469 : cluster 0 pgmap v1686: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:05.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[75179]: audit 2023-10-09T09:26:05.386850+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[75179]: audit 2023-10-09T09:26:05.392353+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[75179]: audit 2023-10-09T09:26:05.396362+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[75179]: audit 2023-10-09T09:26:05.402978+0000 mon.a (mon.0) 2015 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:05 smithi094 bash[59677]: cluster 2023-10-09T09:26:04.366743+0000 mgr.y (mgr.25421) 1469 : cluster 0 pgmap v1686: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:05.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:05 smithi094 bash[59677]: audit 2023-10-09T09:26:05.386850+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:05 smithi094 bash[59677]: audit 2023-10-09T09:26:05.392353+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:05 smithi094 bash[59677]: audit 2023-10-09T09:26:05.396362+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:05.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:05 smithi094 bash[59677]: audit 2023-10-09T09:26:05.402978+0000 mon.a (mon.0) 2015 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:26:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:26:05] "GET /metrics HTTP/1.1" 200 34098 "" "Prometheus/2.33.4" 2023-10-09T09:26:06.735 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:26:06.735 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:26:06.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:06 smithi094 bash[59677]: audit 2023-10-09T09:26:05.933200+0000 mon.a (mon.0) 2016 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:26:06.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:06 smithi094 bash[59677]: audit 2023-10-09T09:26:05.934516+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:26:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:06 smithi094 bash[59677]: audit 2023-10-09T09:26:05.942473+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:06 smithi016 bash[77892]: audit 2023-10-09T09:26:05.933200+0000 mon.a (mon.0) 2016 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:26:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:06 smithi016 bash[77892]: audit 2023-10-09T09:26:05.934516+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:26:06.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:06 smithi016 bash[77892]: audit 2023-10-09T09:26:05.942473+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:06 smithi016 bash[75179]: audit 2023-10-09T09:26:05.933200+0000 mon.a (mon.0) 2016 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:26:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:06 smithi016 bash[75179]: audit 2023-10-09T09:26:05.934516+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:26:06.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:06 smithi016 bash[75179]: audit 2023-10-09T09:26:05.942473+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:26:07.628 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:26:07.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:07 smithi094 bash[59677]: cluster 2023-10-09T09:26:06.367680+0000 mgr.y (mgr.25421) 1470 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:07 smithi016 bash[75179]: cluster 2023-10-09T09:26:06.367680+0000 mgr.y (mgr.25421) 1470 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:07 smithi016 bash[77892]: cluster 2023-10-09T09:26:06.367680+0000 mgr.y (mgr.25421) 1470 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:08.451 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T09:26:08.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:08 smithi016 bash[77892]: audit 2023-10-09T09:26:08.432643+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.16:0/4293600546' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:08.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:08 smithi016 bash[77892]: audit 2023-10-09T09:26:08.433308+0000 mon.a (mon.0) 2019 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:08.692 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:08 smithi016 bash[75179]: audit 2023-10-09T09:26:08.432643+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.16:0/4293600546' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:08.692 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:08 smithi016 bash[75179]: audit 2023-10-09T09:26:08.433308+0000 mon.a (mon.0) 2019 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:08.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:08 smithi094 bash[59677]: audit 2023-10-09T09:26:08.432643+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.16:0/4293600546' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:08.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:08 smithi094 bash[59677]: audit 2023-10-09T09:26:08.433308+0000 mon.a (mon.0) 2019 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:09.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T09:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:09 smithi094 bash[59677]: cluster 2023-10-09T09:26:08.368287+0000 mgr.y (mgr.25421) 1471 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:09 smithi094 bash[59677]: cluster 2023-10-09T09:26:08.443861+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:09 smithi094 bash[59677]: cluster 2023-10-09T09:26:08.444429+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:09 smithi094 bash[59677]: audit 2023-10-09T09:26:08.444774+0000 mon.a (mon.0) 2020 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:09 smithi094 bash[59677]: cluster 2023-10-09T09:26:08.444866+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-09T09:26:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[75179]: cluster 2023-10-09T09:26:08.368287+0000 mgr.y (mgr.25421) 1471 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[75179]: cluster 2023-10-09T09:26:08.443861+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[75179]: cluster 2023-10-09T09:26:08.444429+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[75179]: audit 2023-10-09T09:26:08.444774+0000 mon.a (mon.0) 2020 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[75179]: cluster 2023-10-09T09:26:08.444866+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-09T09:26:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[77892]: cluster 2023-10-09T09:26:08.368287+0000 mgr.y (mgr.25421) 1471 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[77892]: cluster 2023-10-09T09:26:08.443861+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[77892]: cluster 2023-10-09T09:26:08.444429+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[77892]: audit 2023-10-09T09:26:08.444774+0000 mon.a (mon.0) 2020 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:09.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:09 smithi016 bash[77892]: cluster 2023-10-09T09:26:08.444866+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-09T09:26:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:10 smithi094 bash[59677]: cluster 2023-10-09T09:26:09.205265+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:10 smithi016 bash[75179]: cluster 2023-10-09T09:26:09.205265+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:10 smithi016 bash[77892]: cluster 2023-10-09T09:26:09.205265+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T09:26:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:11 smithi094 bash[59677]: cluster 2023-10-09T09:26:10.369113+0000 mgr.y (mgr.25421) 1472 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:11 smithi016 bash[75179]: cluster 2023-10-09T09:26:10.369113+0000 mgr.y (mgr.25421) 1472 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:11.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:11 smithi016 bash[77892]: cluster 2023-10-09T09:26:10.369113+0000 mgr.y (mgr.25421) 1472 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:12.403 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:26:13.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:13 smithi094 bash[59677]: cluster 2023-10-09T09:26:12.369856+0000 mgr.y (mgr.25421) 1473 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:13 smithi016 bash[75179]: cluster 2023-10-09T09:26:12.369856+0000 mgr.y (mgr.25421) 1473 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:13 smithi016 bash[77892]: cluster 2023-10-09T09:26:12.369856+0000 mgr.y (mgr.25421) 1473 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:14.191 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:15.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:15 smithi016 bash[77892]: cluster 2023-10-09T09:26:14.370561+0000 mgr.y (mgr.25421) 1474 : cluster 0 pgmap v1692: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:15 smithi016 bash[75179]: cluster 2023-10-09T09:26:14.370561+0000 mgr.y (mgr.25421) 1474 : cluster 0 pgmap v1692: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:15.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:15 smithi094 bash[59677]: cluster 2023-10-09T09:26:14.370561+0000 mgr.y (mgr.25421) 1474 : cluster 0 pgmap v1692: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:15.825 INFO:teuthology.orchestra.run.smithi094.stdout:ERROR: (22) Invalid argument 2023-10-09T09:26:15.825 INFO:teuthology.orchestra.run.smithi094.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:26:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:26:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:26:15] "GET /metrics HTTP/1.1" 200 34098 "" "Prometheus/2.33.4" 2023-10-09T09:26:17.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:17 smithi094 bash[59677]: cluster 2023-10-09T09:26:16.371495+0000 mgr.y (mgr.25421) 1475 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:17 smithi016 bash[75179]: cluster 2023-10-09T09:26:16.371495+0000 mgr.y (mgr.25421) 1475 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:17 smithi016 bash[77892]: cluster 2023-10-09T09:26:16.371495+0000 mgr.y (mgr.25421) 1475 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:18.479 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T09:26:18.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:18 smithi094 bash[59677]: audit 2023-10-09T09:26:18.106599+0000 mon.a (mon.0) 2022 : audit 1 from='client.? 172.21.15.16:0/1618956549' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:18.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:18 smithi016 bash[77892]: audit 2023-10-09T09:26:18.106599+0000 mon.a (mon.0) 2022 : audit 1 from='client.? 172.21.15.16:0/1618956549' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:18.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:18 smithi016 bash[75179]: audit 2023-10-09T09:26:18.106599+0000 mon.a (mon.0) 2022 : audit 1 from='client.? 172.21.15.16:0/1618956549' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:19.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:19 smithi094 bash[59677]: cluster 2023-10-09T09:26:18.372453+0000 mgr.y (mgr.25421) 1476 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:19 smithi094 bash[59677]: cluster 2023-10-09T09:26:18.469971+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T09:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:19 smithi094 bash[59677]: cluster 2023-10-09T09:26:18.470094+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T09:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:19 smithi094 bash[59677]: audit 2023-10-09T09:26:18.470647+0000 mon.a (mon.0) 2023 : audit 1 from='client.? 172.21.15.16:0/1618956549' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:19 smithi094 bash[59677]: cluster 2023-10-09T09:26:18.470722+0000 mon.a (mon.0) 2024 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-09T09:26:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:19 smithi094 bash[59677]: audit 2023-10-09T09:26:19.270128+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[75179]: cluster 2023-10-09T09:26:18.372453+0000 mgr.y (mgr.25421) 1476 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[75179]: cluster 2023-10-09T09:26:18.469971+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T09:26:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[75179]: cluster 2023-10-09T09:26:18.470094+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T09:26:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[75179]: audit 2023-10-09T09:26:18.470647+0000 mon.a (mon.0) 2023 : audit 1 from='client.? 172.21.15.16:0/1618956549' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[75179]: cluster 2023-10-09T09:26:18.470722+0000 mon.a (mon.0) 2024 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-09T09:26:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[75179]: audit 2023-10-09T09:26:19.270128+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[77892]: cluster 2023-10-09T09:26:18.372453+0000 mgr.y (mgr.25421) 1476 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[77892]: cluster 2023-10-09T09:26:18.469971+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T09:26:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[77892]: cluster 2023-10-09T09:26:18.470094+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T09:26:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[77892]: audit 2023-10-09T09:26:18.470647+0000 mon.a (mon.0) 2023 : audit 1 from='client.? 172.21.15.16:0/1618956549' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[77892]: cluster 2023-10-09T09:26:18.470722+0000 mon.a (mon.0) 2024 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-09T09:26:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:19 smithi016 bash[77892]: audit 2023-10-09T09:26:19.270128+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:20 smithi094 bash[59677]: cluster 2023-10-09T09:26:19.479305+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:20 smithi094 bash[59677]: cluster 2023-10-09T09:26:19.480711+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-09T09:26:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:20 smithi094 bash[59677]: cluster 2023-10-09T09:26:19.484117+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:20 smithi094 bash[59677]: cluster 2023-10-09T09:26:19.484225+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:20 smithi094 bash[59677]: cluster 2023-10-09T09:26:19.487108+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[75179]: cluster 2023-10-09T09:26:19.479305+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[75179]: cluster 2023-10-09T09:26:19.480711+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-09T09:26:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[75179]: cluster 2023-10-09T09:26:19.484117+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[75179]: cluster 2023-10-09T09:26:19.484225+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[75179]: cluster 2023-10-09T09:26:19.487108+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[77892]: cluster 2023-10-09T09:26:19.479305+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[77892]: cluster 2023-10-09T09:26:19.480711+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-09T09:26:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[77892]: cluster 2023-10-09T09:26:19.484117+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[77892]: cluster 2023-10-09T09:26:19.484225+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:20 smithi016 bash[77892]: cluster 2023-10-09T09:26:19.487108+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T09:26:21.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:21 smithi094 bash[59677]: cluster 2023-10-09T09:26:20.373290+0000 mgr.y (mgr.25421) 1477 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:21.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:21 smithi016 bash[75179]: cluster 2023-10-09T09:26:20.373290+0000 mgr.y (mgr.25421) 1477 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:21 smithi016 bash[77892]: cluster 2023-10-09T09:26:20.373290+0000 mgr.y (mgr.25421) 1477 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:23 smithi094 bash[59677]: cluster 2023-10-09T09:26:22.374181+0000 mgr.y (mgr.25421) 1478 : cluster 0 pgmap v1698: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:23 smithi016 bash[75179]: cluster 2023-10-09T09:26:22.374181+0000 mgr.y (mgr.25421) 1478 : cluster 0 pgmap v1698: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:23 smithi016 bash[77892]: cluster 2023-10-09T09:26:22.374181+0000 mgr.y (mgr.25421) 1478 : cluster 0 pgmap v1698: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:24.367 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:25.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:25 smithi016 bash[75179]: cluster 2023-10-09T09:26:24.374978+0000 mgr.y (mgr.25421) 1479 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:25.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:25 smithi016 bash[77892]: cluster 2023-10-09T09:26:24.374978+0000 mgr.y (mgr.25421) 1479 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:25.751 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:26:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:26:25] "GET /metrics HTTP/1.1" 200 34099 "" "Prometheus/2.33.4" 2023-10-09T09:26:25.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:25 smithi094 bash[59677]: cluster 2023-10-09T09:26:24.374978+0000 mgr.y (mgr.25421) 1479 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:27.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:27 smithi016 bash[77892]: cluster 2023-10-09T09:26:26.376112+0000 mgr.y (mgr.25421) 1480 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:27 smithi016 bash[75179]: cluster 2023-10-09T09:26:26.376112+0000 mgr.y (mgr.25421) 1480 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:27.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:27 smithi094 bash[59677]: cluster 2023-10-09T09:26:26.376112+0000 mgr.y (mgr.25421) 1480 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:28.512 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:26:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:28 smithi016 bash[75179]: audit 2023-10-09T09:26:28.301177+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.16:0/1901453527' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:28.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:28 smithi016 bash[75179]: audit 2023-10-09T09:26:28.301978+0000 mon.a (mon.0) 2027 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:28.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:28 smithi016 bash[77892]: audit 2023-10-09T09:26:28.301177+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.16:0/1901453527' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:28.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:28 smithi016 bash[77892]: audit 2023-10-09T09:26:28.301978+0000 mon.a (mon.0) 2027 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:28.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:28 smithi094 bash[59677]: audit 2023-10-09T09:26:28.301177+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.16:0/1901453527' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:28.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:28 smithi094 bash[59677]: audit 2023-10-09T09:26:28.301978+0000 mon.a (mon.0) 2027 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:29.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T09:26:29.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:29 smithi094 bash[59677]: cluster 2023-10-09T09:26:28.376960+0000 mgr.y (mgr.25421) 1481 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:30.068 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:29 smithi094 bash[59677]: audit 2023-10-09T09:26:28.505497+0000 mon.a (mon.0) 2028 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:30.068 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:29 smithi094 bash[59677]: cluster 2023-10-09T09:26:28.505640+0000 mon.a (mon.0) 2029 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-09T09:26:30.068 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:29 smithi094 bash[59677]: cluster 2023-10-09T09:26:29.040128+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.068 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:29 smithi094 bash[59677]: cluster 2023-10-09T09:26:29.040198+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.068 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:29 smithi094 bash[59677]: cluster 2023-10-09T09:26:29.040372+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.068 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:29 smithi094 bash[59677]: cluster 2023-10-09T09:26:29.040433+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[75179]: cluster 2023-10-09T09:26:28.376960+0000 mgr.y (mgr.25421) 1481 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:30.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[75179]: audit 2023-10-09T09:26:28.505497+0000 mon.a (mon.0) 2028 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:30.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[75179]: cluster 2023-10-09T09:26:28.505640+0000 mon.a (mon.0) 2029 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-09T09:26:30.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[75179]: cluster 2023-10-09T09:26:29.040128+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[75179]: cluster 2023-10-09T09:26:29.040198+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[75179]: cluster 2023-10-09T09:26:29.040372+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[75179]: cluster 2023-10-09T09:26:29.040433+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.070 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[77892]: cluster 2023-10-09T09:26:28.376960+0000 mgr.y (mgr.25421) 1481 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:30.074 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[77892]: audit 2023-10-09T09:26:28.505497+0000 mon.a (mon.0) 2028 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:30.074 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[77892]: cluster 2023-10-09T09:26:28.505640+0000 mon.a (mon.0) 2029 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-09T09:26:30.074 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[77892]: cluster 2023-10-09T09:26:29.040128+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.074 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[77892]: cluster 2023-10-09T09:26:29.040198+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.075 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[77892]: cluster 2023-10-09T09:26:29.040372+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:30.075 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:29 smithi016 bash[77892]: cluster 2023-10-09T09:26:29.040433+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T09:26:31.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:30 smithi094 bash[59677]: cluster 2023-10-09T09:26:30.377831+0000 mgr.y (mgr.25421) 1482 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:31.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:30 smithi016 bash[75179]: cluster 2023-10-09T09:26:30.377831+0000 mgr.y (mgr.25421) 1482 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:31.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:30 smithi016 bash[77892]: cluster 2023-10-09T09:26:30.377831+0000 mgr.y (mgr.25421) 1482 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:33.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:33 smithi094 bash[59677]: cluster 2023-10-09T09:26:32.378621+0000 mgr.y (mgr.25421) 1483 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:33.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:33 smithi016 bash[75179]: cluster 2023-10-09T09:26:32.378621+0000 mgr.y (mgr.25421) 1483 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:33.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:33 smithi016 bash[77892]: cluster 2023-10-09T09:26:32.378621+0000 mgr.y (mgr.25421) 1483 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:34.296 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:34.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:34 smithi094 bash[59677]: audit 2023-10-09T09:26:34.270584+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:34.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:34 smithi016 bash[75179]: audit 2023-10-09T09:26:34.270584+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:34.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:34 smithi016 bash[77892]: audit 2023-10-09T09:26:34.270584+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:35.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:35 smithi016 bash[75179]: cluster 2023-10-09T09:26:34.379466+0000 mgr.y (mgr.25421) 1484 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:35.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:35 smithi016 bash[77892]: cluster 2023-10-09T09:26:34.379466+0000 mgr.y (mgr.25421) 1484 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:35.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:35 smithi094 bash[59677]: cluster 2023-10-09T09:26:34.379466+0000 mgr.y (mgr.25421) 1484 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:36.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:26:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:26:35] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.33.4" 2023-10-09T09:26:37.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:37 smithi094 bash[59677]: cluster 2023-10-09T09:26:36.380482+0000 mgr.y (mgr.25421) 1485 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:37.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:37 smithi016 bash[75179]: cluster 2023-10-09T09:26:36.380482+0000 mgr.y (mgr.25421) 1485 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:37.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:37 smithi016 bash[77892]: cluster 2023-10-09T09:26:36.380482+0000 mgr.y (mgr.25421) 1485 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:38.468 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T09:26:38.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:38 smithi016 bash[75179]: audit 2023-10-09T09:26:38.349455+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.16:0/2041361753' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:38.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:38 smithi016 bash[75179]: audit 2023-10-09T09:26:38.350158+0000 mon.a (mon.0) 2031 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:38.722 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:38 smithi016 bash[77892]: audit 2023-10-09T09:26:38.349455+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.16:0/2041361753' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:38.722 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:38 smithi016 bash[77892]: audit 2023-10-09T09:26:38.350158+0000 mon.a (mon.0) 2031 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:38.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:38 smithi094 bash[59677]: audit 2023-10-09T09:26:38.349455+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.16:0/2041361753' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:38.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:38 smithi094 bash[59677]: audit 2023-10-09T09:26:38.350158+0000 mon.a (mon.0) 2031 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:39.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T09:26:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:39 smithi094 bash[59677]: cluster 2023-10-09T09:26:38.381283+0000 mgr.y (mgr.25421) 1486 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:39 smithi094 bash[59677]: audit 2023-10-09T09:26:38.459199+0000 mon.a (mon.0) 2032 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:39.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:39 smithi094 bash[59677]: cluster 2023-10-09T09:26:38.459332+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-09T09:26:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:39 smithi016 bash[75179]: cluster 2023-10-09T09:26:38.381283+0000 mgr.y (mgr.25421) 1486 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:39 smithi016 bash[75179]: audit 2023-10-09T09:26:38.459199+0000 mon.a (mon.0) 2032 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:39.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:39 smithi016 bash[75179]: cluster 2023-10-09T09:26:38.459332+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-09T09:26:39.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:39 smithi016 bash[77892]: cluster 2023-10-09T09:26:38.381283+0000 mgr.y (mgr.25421) 1486 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:39 smithi016 bash[77892]: audit 2023-10-09T09:26:38.459199+0000 mon.a (mon.0) 2032 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:39.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:39 smithi016 bash[77892]: cluster 2023-10-09T09:26:38.459332+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-09T09:26:40.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:40 smithi094 bash[59677]: cluster 2023-10-09T09:26:38.457192+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T09:26:40.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:40 smithi094 bash[59677]: cluster 2023-10-09T09:26:39.620111+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T09:26:40.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:40 smithi016 bash[75179]: cluster 2023-10-09T09:26:38.457192+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T09:26:40.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:40 smithi016 bash[75179]: cluster 2023-10-09T09:26:39.620111+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T09:26:40.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:40 smithi016 bash[77892]: cluster 2023-10-09T09:26:38.457192+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T09:26:40.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:40 smithi016 bash[77892]: cluster 2023-10-09T09:26:39.620111+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T09:26:41.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:41 smithi094 bash[59677]: cluster 2023-10-09T09:26:40.382126+0000 mgr.y (mgr.25421) 1487 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:41.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:41 smithi016 bash[75179]: cluster 2023-10-09T09:26:40.382126+0000 mgr.y (mgr.25421) 1487 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:41.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:41 smithi016 bash[77892]: cluster 2023-10-09T09:26:40.382126+0000 mgr.y (mgr.25421) 1487 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:43.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:43 smithi094 bash[59677]: cluster 2023-10-09T09:26:42.382997+0000 mgr.y (mgr.25421) 1488 : cluster 0 pgmap v1710: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:43.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:43 smithi016 bash[75179]: cluster 2023-10-09T09:26:42.382997+0000 mgr.y (mgr.25421) 1488 : cluster 0 pgmap v1710: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:43.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:43 smithi016 bash[77892]: cluster 2023-10-09T09:26:42.382997+0000 mgr.y (mgr.25421) 1488 : cluster 0 pgmap v1710: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:44.222 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:45.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:45 smithi016 bash[75179]: cluster 2023-10-09T09:26:44.383835+0000 mgr.y (mgr.25421) 1489 : cluster 0 pgmap v1711: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:45.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:45 smithi016 bash[77892]: cluster 2023-10-09T09:26:44.383835+0000 mgr.y (mgr.25421) 1489 : cluster 0 pgmap v1711: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:45.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:45 smithi094 bash[59677]: cluster 2023-10-09T09:26:44.383835+0000 mgr.y (mgr.25421) 1489 : cluster 0 pgmap v1711: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:46.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:26:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:26:45] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.33.4" 2023-10-09T09:26:47.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:47 smithi094 bash[59677]: cluster 2023-10-09T09:26:46.384827+0000 mgr.y (mgr.25421) 1490 : cluster 0 pgmap v1712: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:47.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:47 smithi016 bash[75179]: cluster 2023-10-09T09:26:46.384827+0000 mgr.y (mgr.25421) 1490 : cluster 0 pgmap v1712: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:47.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:47 smithi016 bash[77892]: cluster 2023-10-09T09:26:46.384827+0000 mgr.y (mgr.25421) 1490 : cluster 0 pgmap v1712: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:48.493 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T09:26:48.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:48 smithi094 bash[59677]: audit 2023-10-09T09:26:48.019415+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.16:0/1666241812' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:48.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:48 smithi094 bash[59677]: audit 2023-10-09T09:26:48.019925+0000 mon.a (mon.0) 2034 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:48.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:48 smithi016 bash[75179]: audit 2023-10-09T09:26:48.019415+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.16:0/1666241812' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:48.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:48 smithi016 bash[75179]: audit 2023-10-09T09:26:48.019925+0000 mon.a (mon.0) 2034 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:48 smithi016 bash[77892]: audit 2023-10-09T09:26:48.019415+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.16:0/1666241812' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:48.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:48 smithi016 bash[77892]: audit 2023-10-09T09:26:48.019925+0000 mon.a (mon.0) 2034 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:49.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T09:26:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:49 smithi094 bash[59677]: cluster 2023-10-09T09:26:48.385616+0000 mgr.y (mgr.25421) 1491 : cluster 0 pgmap v1713: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:49 smithi094 bash[59677]: cluster 2023-10-09T09:26:48.482717+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:49 smithi094 bash[59677]: audit 2023-10-09T09:26:48.485046+0000 mon.a (mon.0) 2035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:26:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:49 smithi094 bash[59677]: cluster 2023-10-09T09:26:48.485131+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-09T09:26:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:49 smithi094 bash[59677]: cluster 2023-10-09T09:26:48.487537+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:49.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:49 smithi094 bash[59677]: audit 2023-10-09T09:26:49.270896+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[75179]: cluster 2023-10-09T09:26:48.385616+0000 mgr.y (mgr.25421) 1491 : cluster 0 pgmap v1713: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[75179]: cluster 2023-10-09T09:26:48.482717+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[75179]: audit 2023-10-09T09:26:48.485046+0000 mon.a (mon.0) 2035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:26:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[75179]: cluster 2023-10-09T09:26:48.485131+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-09T09:26:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[75179]: cluster 2023-10-09T09:26:48.487537+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:49.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[75179]: audit 2023-10-09T09:26:49.270896+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[77892]: cluster 2023-10-09T09:26:48.385616+0000 mgr.y (mgr.25421) 1491 : cluster 0 pgmap v1713: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:26:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[77892]: cluster 2023-10-09T09:26:48.482717+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[77892]: audit 2023-10-09T09:26:48.485046+0000 mon.a (mon.0) 2035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:26:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[77892]: cluster 2023-10-09T09:26:48.485131+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-09T09:26:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[77892]: cluster 2023-10-09T09:26:48.487537+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:49.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:49 smithi016 bash[77892]: audit 2023-10-09T09:26:49.270896+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:50.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[75179]: cluster 2023-10-09T09:26:48.487584+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:50.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[75179]: cluster 2023-10-09T09:26:49.492148+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[75179]: cluster 2023-10-09T09:26:49.492275+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[75179]: cluster 2023-10-09T09:26:49.492892+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-10-09T09:26:50.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[75179]: cluster 2023-10-09T09:26:49.496358+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[77892]: cluster 2023-10-09T09:26:48.487584+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:50.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[77892]: cluster 2023-10-09T09:26:49.492148+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[77892]: cluster 2023-10-09T09:26:49.492275+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[77892]: cluster 2023-10-09T09:26:49.492892+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-10-09T09:26:50.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:50 smithi016 bash[77892]: cluster 2023-10-09T09:26:49.496358+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:50 smithi094 bash[59677]: cluster 2023-10-09T09:26:48.487584+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T09:26:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:50 smithi094 bash[59677]: cluster 2023-10-09T09:26:49.492148+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:50 smithi094 bash[59677]: cluster 2023-10-09T09:26:49.492275+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:50 smithi094 bash[59677]: cluster 2023-10-09T09:26:49.492892+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-10-09T09:26:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:50 smithi094 bash[59677]: cluster 2023-10-09T09:26:49.496358+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T09:26:51.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:51 smithi016 bash[75179]: cluster 2023-10-09T09:26:50.386570+0000 mgr.y (mgr.25421) 1492 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:51.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:51 smithi016 bash[77892]: cluster 2023-10-09T09:26:50.386570+0000 mgr.y (mgr.25421) 1492 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:51.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:51 smithi094 bash[59677]: cluster 2023-10-09T09:26:50.386570+0000 mgr.y (mgr.25421) 1492 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:53.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:53 smithi094 bash[59677]: cluster 2023-10-09T09:26:52.387432+0000 mgr.y (mgr.25421) 1493 : cluster 0 pgmap v1717: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:54.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:53 smithi016 bash[75179]: cluster 2023-10-09T09:26:52.387432+0000 mgr.y (mgr.25421) 1493 : cluster 0 pgmap v1717: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:54.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:53 smithi016 bash[77892]: cluster 2023-10-09T09:26:52.387432+0000 mgr.y (mgr.25421) 1493 : cluster 0 pgmap v1717: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:54.310 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:55.776 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:26:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:26:55] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.33.4" 2023-10-09T09:26:55.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:55 smithi016 bash[77892]: cluster 2023-10-09T09:26:54.388187+0000 mgr.y (mgr.25421) 1494 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:55.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:55 smithi016 bash[75179]: cluster 2023-10-09T09:26:54.388187+0000 mgr.y (mgr.25421) 1494 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:55.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:55 smithi094 bash[59677]: cluster 2023-10-09T09:26:54.388187+0000 mgr.y (mgr.25421) 1494 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:57.793 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:57 smithi016 bash[75179]: cluster 2023-10-09T09:26:56.389011+0000 mgr.y (mgr.25421) 1495 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:57.794 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:57 smithi016 bash[77892]: cluster 2023-10-09T09:26:56.389011+0000 mgr.y (mgr.25421) 1495 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:57.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:57 smithi094 bash[59677]: cluster 2023-10-09T09:26:56.389011+0000 mgr.y (mgr.25421) 1495 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:58.528 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T09:26:58.815 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:58 smithi016 bash[75179]: audit 2023-10-09T09:26:58.169859+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.16:0/936039905' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:58.815 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:58 smithi016 bash[75179]: audit 2023-10-09T09:26:58.170607+0000 mon.a (mon.0) 2039 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:58.815 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:58 smithi016 bash[77892]: audit 2023-10-09T09:26:58.169859+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.16:0/936039905' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:58.816 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:58 smithi016 bash[77892]: audit 2023-10-09T09:26:58.170607+0000 mon.a (mon.0) 2039 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:58.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:58 smithi094 bash[59677]: audit 2023-10-09T09:26:58.169859+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.16:0/936039905' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:58.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:58 smithi094 bash[59677]: audit 2023-10-09T09:26:58.170607+0000 mon.a (mon.0) 2039 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:26:59.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:26:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:59 smithi094 bash[59677]: cluster 2023-10-09T09:26:58.389894+0000 mgr.y (mgr.25421) 1496 : cluster 0 pgmap v1720: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:26:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:59 smithi094 bash[59677]: cluster 2023-10-09T09:26:58.517945+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:26:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:59 smithi094 bash[59677]: audit 2023-10-09T09:26:58.521815+0000 mon.a (mon.0) 2040 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:26:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:59 smithi094 bash[59677]: cluster 2023-10-09T09:26:58.521900+0000 mon.a (mon.0) 2041 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2023-10-09T09:26:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:59 smithi094 bash[59677]: cluster 2023-10-09T09:26:58.846304+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:26:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:59 smithi094 bash[59677]: cluster 2023-10-09T09:26:58.846448+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:26:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:26:59 smithi094 bash[59677]: cluster 2023-10-09T09:26:58.846643+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[75179]: cluster 2023-10-09T09:26:58.389894+0000 mgr.y (mgr.25421) 1496 : cluster 0 pgmap v1720: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:27:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[75179]: cluster 2023-10-09T09:26:58.517945+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[75179]: audit 2023-10-09T09:26:58.521815+0000 mon.a (mon.0) 2040 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:27:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[75179]: cluster 2023-10-09T09:26:58.521900+0000 mon.a (mon.0) 2041 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2023-10-09T09:27:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[75179]: cluster 2023-10-09T09:26:58.846304+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[75179]: cluster 2023-10-09T09:26:58.846448+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[75179]: cluster 2023-10-09T09:26:58.846643+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[77892]: cluster 2023-10-09T09:26:58.389894+0000 mgr.y (mgr.25421) 1496 : cluster 0 pgmap v1720: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:27:00.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[77892]: cluster 2023-10-09T09:26:58.517945+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[77892]: audit 2023-10-09T09:26:58.521815+0000 mon.a (mon.0) 2040 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:27:00.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[77892]: cluster 2023-10-09T09:26:58.521900+0000 mon.a (mon.0) 2041 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2023-10-09T09:27:00.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[77892]: cluster 2023-10-09T09:26:58.846304+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[77892]: cluster 2023-10-09T09:26:58.846448+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:00.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:26:59 smithi016 bash[77892]: cluster 2023-10-09T09:26:58.846643+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T09:27:01.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:00 smithi094 bash[59677]: cluster 2023-10-09T09:27:00.390767+0000 mgr.y (mgr.25421) 1497 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:01.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:00 smithi016 bash[75179]: cluster 2023-10-09T09:27:00.390767+0000 mgr.y (mgr.25421) 1497 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:01.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:00 smithi016 bash[77892]: cluster 2023-10-09T09:27:00.390767+0000 mgr.y (mgr.25421) 1497 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:03.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:03 smithi094 bash[59677]: cluster 2023-10-09T09:27:02.391547+0000 mgr.y (mgr.25421) 1498 : cluster 0 pgmap v1723: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:03 smithi016 bash[75179]: cluster 2023-10-09T09:27:02.391547+0000 mgr.y (mgr.25421) 1498 : cluster 0 pgmap v1723: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:03.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:03 smithi016 bash[77892]: cluster 2023-10-09T09:27:02.391547+0000 mgr.y (mgr.25421) 1498 : cluster 0 pgmap v1723: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:04.375 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:04.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:04 smithi094 bash[59677]: audit 2023-10-09T09:27:04.271233+0000 mon.a (mon.0) 2042 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:04 smithi016 bash[75179]: audit 2023-10-09T09:27:04.271233+0000 mon.a (mon.0) 2042 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:04.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:04 smithi016 bash[77892]: audit 2023-10-09T09:27:04.271233+0000 mon.a (mon.0) 2042 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:05.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:05 smithi016 bash[75179]: cluster 2023-10-09T09:27:04.392216+0000 mgr.y (mgr.25421) 1499 : cluster 0 pgmap v1724: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:05.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:05 smithi016 bash[77892]: cluster 2023-10-09T09:27:04.392216+0000 mgr.y (mgr.25421) 1499 : cluster 0 pgmap v1724: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:05.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:05 smithi094 bash[59677]: cluster 2023-10-09T09:27:04.392216+0000 mgr.y (mgr.25421) 1499 : cluster 0 pgmap v1724: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:27:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:27:05] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.33.4" 2023-10-09T09:27:06.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:06 smithi016 bash[77892]: audit 2023-10-09T09:27:06.022802+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:27:06.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:06 smithi016 bash[77892]: audit 2023-10-09T09:27:06.397458+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:06.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:06 smithi016 bash[77892]: audit 2023-10-09T09:27:06.403728+0000 mon.a (mon.0) 2045 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:06.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:06 smithi094 bash[59677]: audit 2023-10-09T09:27:06.022802+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:27:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:06 smithi094 bash[59677]: audit 2023-10-09T09:27:06.397458+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:06.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:06 smithi094 bash[59677]: audit 2023-10-09T09:27:06.403728+0000 mon.a (mon.0) 2045 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:06 smithi016 bash[75179]: audit 2023-10-09T09:27:06.022802+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:27:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:06 smithi016 bash[75179]: audit 2023-10-09T09:27:06.397458+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:06.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:06 smithi016 bash[75179]: audit 2023-10-09T09:27:06.403728+0000 mon.a (mon.0) 2045 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:07.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:07 smithi094 bash[59677]: cluster 2023-10-09T09:27:06.393098+0000 mgr.y (mgr.25421) 1500 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:07 smithi016 bash[75179]: cluster 2023-10-09T09:27:06.393098+0000 mgr.y (mgr.25421) 1500 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:07 smithi016 bash[77892]: cluster 2023-10-09T09:27:06.393098+0000 mgr.y (mgr.25421) 1500 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:09.485 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:27:09.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:09 smithi094 bash[59677]: cluster 2023-10-09T09:27:08.393651+0000 mgr.y (mgr.25421) 1501 : cluster 0 pgmap v1726: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:09.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:09 smithi094 bash[59677]: audit 2023-10-09T09:27:08.505608+0000 mon.a (mon.0) 2046 : audit 1 from='client.? 172.21.15.16:0/1270724770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:09 smithi016 bash[75179]: cluster 2023-10-09T09:27:08.393651+0000 mgr.y (mgr.25421) 1501 : cluster 0 pgmap v1726: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:09.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:09 smithi016 bash[75179]: audit 2023-10-09T09:27:08.505608+0000 mon.a (mon.0) 2046 : audit 1 from='client.? 172.21.15.16:0/1270724770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:09 smithi016 bash[77892]: cluster 2023-10-09T09:27:08.393651+0000 mgr.y (mgr.25421) 1501 : cluster 0 pgmap v1726: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:09.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:09 smithi016 bash[77892]: audit 2023-10-09T09:27:08.505608+0000 mon.a (mon.0) 2046 : audit 1 from='client.? 172.21.15.16:0/1270724770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:10.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T09:27:10.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:10 smithi094 bash[59677]: audit 2023-10-09T09:27:09.477991+0000 mon.a (mon.0) 2047 : audit 1 from='client.? 172.21.15.16:0/1270724770' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:10 smithi094 bash[59677]: cluster 2023-10-09T09:27:09.478097+0000 mon.a (mon.0) 2048 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2023-10-09T09:27:10.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:10 smithi094 bash[59677]: cluster 2023-10-09T09:27:10.449390+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:10 smithi016 bash[75179]: audit 2023-10-09T09:27:09.477991+0000 mon.a (mon.0) 2047 : audit 1 from='client.? 172.21.15.16:0/1270724770' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:10 smithi016 bash[75179]: cluster 2023-10-09T09:27:09.478097+0000 mon.a (mon.0) 2048 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2023-10-09T09:27:10.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:10 smithi016 bash[75179]: cluster 2023-10-09T09:27:10.449390+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:10.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:10 smithi016 bash[77892]: audit 2023-10-09T09:27:09.477991+0000 mon.a (mon.0) 2047 : audit 1 from='client.? 172.21.15.16:0/1270724770' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:10.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:10 smithi016 bash[77892]: cluster 2023-10-09T09:27:09.478097+0000 mon.a (mon.0) 2048 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2023-10-09T09:27:10.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:10 smithi016 bash[77892]: cluster 2023-10-09T09:27:10.449390+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:11 smithi094 bash[59677]: cluster 2023-10-09T09:27:10.366125+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:11 smithi094 bash[59677]: cluster 2023-10-09T09:27:10.368902+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:11 smithi094 bash[59677]: cluster 2023-10-09T09:27:10.394314+0000 mgr.y (mgr.25421) 1502 : cluster 0 pgmap v1728: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:11 smithi094 bash[59677]: cluster 2023-10-09T09:27:10.449605+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:11 smithi094 bash[59677]: cluster 2023-10-09T09:27:10.449732+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[75179]: cluster 2023-10-09T09:27:10.366125+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[75179]: cluster 2023-10-09T09:27:10.368902+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[75179]: cluster 2023-10-09T09:27:10.394314+0000 mgr.y (mgr.25421) 1502 : cluster 0 pgmap v1728: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[75179]: cluster 2023-10-09T09:27:10.449605+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[75179]: cluster 2023-10-09T09:27:10.449732+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[77892]: cluster 2023-10-09T09:27:10.366125+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[77892]: cluster 2023-10-09T09:27:10.368902+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[77892]: cluster 2023-10-09T09:27:10.394314+0000 mgr.y (mgr.25421) 1502 : cluster 0 pgmap v1728: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:11.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[77892]: cluster 2023-10-09T09:27:10.449605+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:11.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:11 smithi016 bash[77892]: cluster 2023-10-09T09:27:10.449732+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T09:27:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:13 smithi094 bash[59677]: audit 2023-10-09T09:27:12.272988+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:13 smithi094 bash[59677]: audit 2023-10-09T09:27:12.280431+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:13.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:13 smithi094 bash[59677]: cluster 2023-10-09T09:27:12.395052+0000 mgr.y (mgr.25421) 1503 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:13 smithi016 bash[75179]: audit 2023-10-09T09:27:12.272988+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:13 smithi016 bash[75179]: audit 2023-10-09T09:27:12.280431+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:13.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:13 smithi016 bash[75179]: cluster 2023-10-09T09:27:12.395052+0000 mgr.y (mgr.25421) 1503 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:13.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:13 smithi016 bash[77892]: audit 2023-10-09T09:27:12.272988+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:13.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:13 smithi016 bash[77892]: audit 2023-10-09T09:27:12.280431+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:13.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:13 smithi016 bash[77892]: cluster 2023-10-09T09:27:12.395052+0000 mgr.y (mgr.25421) 1503 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:15.038 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[75179]: audit 2023-10-09T09:27:14.260256+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[75179]: audit 2023-10-09T09:27:14.268416+0000 mon.a (mon.0) 2052 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[75179]: cluster 2023-10-09T09:27:14.395861+0000 mgr.y (mgr.25421) 1504 : cluster 0 pgmap v1730: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[75179]: audit 2023-10-09T09:27:14.738678+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:27:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[75179]: audit 2023-10-09T09:27:14.740462+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:27:15.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[75179]: audit 2023-10-09T09:27:14.749224+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[77892]: audit 2023-10-09T09:27:14.260256+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[77892]: audit 2023-10-09T09:27:14.268416+0000 mon.a (mon.0) 2052 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[77892]: cluster 2023-10-09T09:27:14.395861+0000 mgr.y (mgr.25421) 1504 : cluster 0 pgmap v1730: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:15.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[77892]: audit 2023-10-09T09:27:14.738678+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:27:15.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[77892]: audit 2023-10-09T09:27:14.740462+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:27:15.501 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[77892]: audit 2023-10-09T09:27:14.749224+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:15 smithi094 bash[59677]: audit 2023-10-09T09:27:14.260256+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:15 smithi094 bash[59677]: audit 2023-10-09T09:27:14.268416+0000 mon.a (mon.0) 2052 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:15 smithi094 bash[59677]: cluster 2023-10-09T09:27:14.395861+0000 mgr.y (mgr.25421) 1504 : cluster 0 pgmap v1730: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:15 smithi094 bash[59677]: audit 2023-10-09T09:27:14.738678+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:27:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:15 smithi094 bash[59677]: audit 2023-10-09T09:27:14.740462+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:27:15.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:15 smithi094 bash[59677]: audit 2023-10-09T09:27:14.749224+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:27:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:27:15] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.33.4" 2023-10-09T09:27:17.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:17 smithi094 bash[59677]: cluster 2023-10-09T09:27:16.396818+0000 mgr.y (mgr.25421) 1505 : cluster 0 pgmap v1731: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:17.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:17 smithi016 bash[75179]: cluster 2023-10-09T09:27:16.396818+0000 mgr.y (mgr.25421) 1505 : cluster 0 pgmap v1731: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:17.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:17 smithi016 bash[77892]: cluster 2023-10-09T09:27:16.396818+0000 mgr.y (mgr.25421) 1505 : cluster 0 pgmap v1731: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:19.469 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T09:27:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:19 smithi094 bash[59677]: cluster 2023-10-09T09:27:18.397335+0000 mgr.y (mgr.25421) 1506 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:19 smithi094 bash[59677]: audit 2023-10-09T09:27:19.271637+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:19.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:19 smithi094 bash[59677]: audit 2023-10-09T09:27:19.315126+0000 mon.a (mon.0) 2057 : audit 1 from='client.? 172.21.15.16:0/717454095' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:19 smithi016 bash[75179]: cluster 2023-10-09T09:27:18.397335+0000 mgr.y (mgr.25421) 1506 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:19.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:19 smithi016 bash[75179]: audit 2023-10-09T09:27:19.271637+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:19.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:19 smithi016 bash[75179]: audit 2023-10-09T09:27:19.315126+0000 mon.a (mon.0) 2057 : audit 1 from='client.? 172.21.15.16:0/717454095' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:19 smithi016 bash[77892]: cluster 2023-10-09T09:27:18.397335+0000 mgr.y (mgr.25421) 1506 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:19 smithi016 bash[77892]: audit 2023-10-09T09:27:19.271637+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:19.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:19 smithi016 bash[77892]: audit 2023-10-09T09:27:19.315126+0000 mon.a (mon.0) 2057 : audit 1 from='client.? 172.21.15.16:0/717454095' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:20.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T09:27:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:20 smithi094 bash[59677]: cluster 2023-10-09T09:27:19.459073+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T09:27:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:20 smithi094 bash[59677]: cluster 2023-10-09T09:27:19.459651+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T09:27:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:20 smithi094 bash[59677]: audit 2023-10-09T09:27:19.462067+0000 mon.a (mon.0) 2058 : audit 1 from='client.? 172.21.15.16:0/717454095' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:20.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:20 smithi094 bash[59677]: cluster 2023-10-09T09:27:19.462178+0000 mon.a (mon.0) 2059 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2023-10-09T09:27:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[75179]: cluster 2023-10-09T09:27:19.459073+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T09:27:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[75179]: cluster 2023-10-09T09:27:19.459651+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T09:27:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[75179]: audit 2023-10-09T09:27:19.462067+0000 mon.a (mon.0) 2058 : audit 1 from='client.? 172.21.15.16:0/717454095' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:20.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[75179]: cluster 2023-10-09T09:27:19.462178+0000 mon.a (mon.0) 2059 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2023-10-09T09:27:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[77892]: cluster 2023-10-09T09:27:19.459073+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T09:27:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[77892]: cluster 2023-10-09T09:27:19.459651+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T09:27:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[77892]: audit 2023-10-09T09:27:19.462067+0000 mon.a (mon.0) 2058 : audit 1 from='client.? 172.21.15.16:0/717454095' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:20.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:20 smithi016 bash[77892]: cluster 2023-10-09T09:27:19.462178+0000 mon.a (mon.0) 2059 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2023-10-09T09:27:21.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:21 smithi094 bash[59677]: cluster 2023-10-09T09:27:20.398235+0000 mgr.y (mgr.25421) 1507 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:21.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:21 smithi016 bash[75179]: cluster 2023-10-09T09:27:20.398235+0000 mgr.y (mgr.25421) 1507 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:21.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:21 smithi016 bash[77892]: cluster 2023-10-09T09:27:20.398235+0000 mgr.y (mgr.25421) 1507 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:23.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:23 smithi094 bash[59677]: cluster 2023-10-09T09:27:22.399158+0000 mgr.y (mgr.25421) 1508 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:23.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:23 smithi016 bash[75179]: cluster 2023-10-09T09:27:22.399158+0000 mgr.y (mgr.25421) 1508 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:23.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:23 smithi016 bash[77892]: cluster 2023-10-09T09:27:22.399158+0000 mgr.y (mgr.25421) 1508 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:25.234 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:25.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:25 smithi016 bash[75179]: cluster 2023-10-09T09:27:24.400160+0000 mgr.y (mgr.25421) 1509 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:25.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:25 smithi016 bash[77892]: cluster 2023-10-09T09:27:24.400160+0000 mgr.y (mgr.25421) 1509 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:25.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:25 smithi094 bash[59677]: cluster 2023-10-09T09:27:24.400160+0000 mgr.y (mgr.25421) 1509 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:26.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:27:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:27:25] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.33.4" 2023-10-09T09:27:27.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:27 smithi094 bash[59677]: cluster 2023-10-09T09:27:26.401212+0000 mgr.y (mgr.25421) 1510 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:27.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:27 smithi016 bash[75179]: cluster 2023-10-09T09:27:26.401212+0000 mgr.y (mgr.25421) 1510 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:27.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:27 smithi016 bash[77892]: cluster 2023-10-09T09:27:26.401212+0000 mgr.y (mgr.25421) 1510 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:29.502 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T09:27:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:29 smithi016 bash[75179]: cluster 2023-10-09T09:27:28.402095+0000 mgr.y (mgr.25421) 1511 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:29.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:29 smithi016 bash[75179]: audit 2023-10-09T09:27:29.477273+0000 mon.a (mon.0) 2060 : audit 1 from='client.? 172.21.15.16:0/1364242372' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:29 smithi016 bash[77892]: cluster 2023-10-09T09:27:28.402095+0000 mgr.y (mgr.25421) 1511 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:29.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:29 smithi016 bash[77892]: audit 2023-10-09T09:27:29.477273+0000 mon.a (mon.0) 2060 : audit 1 from='client.? 172.21.15.16:0/1364242372' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:29.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:29 smithi094 bash[59677]: cluster 2023-10-09T09:27:28.402095+0000 mgr.y (mgr.25421) 1511 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:29.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:29 smithi094 bash[59677]: audit 2023-10-09T09:27:29.477273+0000 mon.a (mon.0) 2060 : audit 1 from='client.? 172.21.15.16:0/1364242372' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:30.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T09:27:30.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[75179]: cluster 2023-10-09T09:27:29.493739+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[75179]: audit 2023-10-09T09:27:29.495685+0000 mon.a (mon.0) 2061 : audit 1 from='client.? 172.21.15.16:0/1364242372' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:27:30.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[75179]: cluster 2023-10-09T09:27:29.495761+0000 mon.a (mon.0) 2062 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2023-10-09T09:27:30.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[75179]: cluster 2023-10-09T09:27:29.570076+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[75179]: cluster 2023-10-09T09:27:29.570149+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[75179]: cluster 2023-10-09T09:27:29.570222+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[77892]: cluster 2023-10-09T09:27:29.493739+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[77892]: audit 2023-10-09T09:27:29.495685+0000 mon.a (mon.0) 2061 : audit 1 from='client.? 172.21.15.16:0/1364242372' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[77892]: cluster 2023-10-09T09:27:29.495761+0000 mon.a (mon.0) 2062 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[77892]: cluster 2023-10-09T09:27:29.570076+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[77892]: cluster 2023-10-09T09:27:29.570149+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:30 smithi016 bash[77892]: cluster 2023-10-09T09:27:29.570222+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:30 smithi094 bash[59677]: cluster 2023-10-09T09:27:29.493739+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:30 smithi094 bash[59677]: audit 2023-10-09T09:27:29.495685+0000 mon.a (mon.0) 2061 : audit 1 from='client.? 172.21.15.16:0/1364242372' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:27:30.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:30 smithi094 bash[59677]: cluster 2023-10-09T09:27:29.495761+0000 mon.a (mon.0) 2062 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2023-10-09T09:27:30.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:30 smithi094 bash[59677]: cluster 2023-10-09T09:27:29.570076+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:30 smithi094 bash[59677]: cluster 2023-10-09T09:27:29.570149+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:30.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:30 smithi094 bash[59677]: cluster 2023-10-09T09:27:29.570222+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T09:27:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:31 smithi094 bash[59677]: cluster 2023-10-09T09:27:30.403046+0000 mgr.y (mgr.25421) 1512 : cluster 0 pgmap v1740: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:31 smithi094 bash[59677]: cluster 2023-10-09T09:27:30.502505+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T09:27:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:31 smithi094 bash[59677]: cluster 2023-10-09T09:27:30.504737+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2023-10-09T09:27:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:31 smithi094 bash[59677]: cluster 2023-10-09T09:27:30.509970+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T09:27:32.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[75179]: cluster 2023-10-09T09:27:30.403046+0000 mgr.y (mgr.25421) 1512 : cluster 0 pgmap v1740: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:32.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[75179]: cluster 2023-10-09T09:27:30.502505+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T09:27:32.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[75179]: cluster 2023-10-09T09:27:30.504737+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2023-10-09T09:27:32.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[75179]: cluster 2023-10-09T09:27:30.509970+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T09:27:32.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[77892]: cluster 2023-10-09T09:27:30.403046+0000 mgr.y (mgr.25421) 1512 : cluster 0 pgmap v1740: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:32.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[77892]: cluster 2023-10-09T09:27:30.502505+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T09:27:32.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[77892]: cluster 2023-10-09T09:27:30.504737+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2023-10-09T09:27:32.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:31 smithi016 bash[77892]: cluster 2023-10-09T09:27:30.509970+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T09:27:33.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:33 smithi094 bash[59677]: cluster 2023-10-09T09:27:32.404018+0000 mgr.y (mgr.25421) 1513 : cluster 0 pgmap v1742: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:34.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:33 smithi016 bash[75179]: cluster 2023-10-09T09:27:32.404018+0000 mgr.y (mgr.25421) 1513 : cluster 0 pgmap v1742: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:34.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:33 smithi016 bash[77892]: cluster 2023-10-09T09:27:32.404018+0000 mgr.y (mgr.25421) 1513 : cluster 0 pgmap v1742: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:34.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:34 smithi094 bash[59677]: audit 2023-10-09T09:27:34.272051+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:35.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:34 smithi016 bash[75179]: audit 2023-10-09T09:27:34.272051+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:35.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:34 smithi016 bash[77892]: audit 2023-10-09T09:27:34.272051+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:35.290 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:35.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:35 smithi016 bash[75179]: cluster 2023-10-09T09:27:34.404804+0000 mgr.y (mgr.25421) 1514 : cluster 0 pgmap v1743: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:35.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:35 smithi016 bash[77892]: cluster 2023-10-09T09:27:34.404804+0000 mgr.y (mgr.25421) 1514 : cluster 0 pgmap v1743: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:35.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:35 smithi094 bash[59677]: cluster 2023-10-09T09:27:34.404804+0000 mgr.y (mgr.25421) 1514 : cluster 0 pgmap v1743: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:36.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:27:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:27:35] "GET /metrics HTTP/1.1" 200 34068 "" "Prometheus/2.33.4" 2023-10-09T09:27:37.904 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:37 smithi016 bash[75179]: cluster 2023-10-09T09:27:36.405808+0000 mgr.y (mgr.25421) 1515 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:37.941 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:37 smithi016 bash[77892]: cluster 2023-10-09T09:27:36.405808+0000 mgr.y (mgr.25421) 1515 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:37.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:37 smithi094 bash[59677]: cluster 2023-10-09T09:27:36.405808+0000 mgr.y (mgr.25421) 1515 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:39.544 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T09:27:39.796 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:39 smithi016 bash[75179]: cluster 2023-10-09T09:27:38.406443+0000 mgr.y (mgr.25421) 1516 : cluster 0 pgmap v1745: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:39.796 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:39 smithi016 bash[75179]: audit 2023-10-09T09:27:39.505598+0000 mon.a (mon.0) 2065 : audit 1 from='client.? 172.21.15.16:0/1097202925' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:39.796 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:39 smithi016 bash[77892]: cluster 2023-10-09T09:27:38.406443+0000 mgr.y (mgr.25421) 1516 : cluster 0 pgmap v1745: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:39.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:39 smithi016 bash[77892]: audit 2023-10-09T09:27:39.505598+0000 mon.a (mon.0) 2065 : audit 1 from='client.? 172.21.15.16:0/1097202925' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:39.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:39 smithi094 bash[59677]: cluster 2023-10-09T09:27:38.406443+0000 mgr.y (mgr.25421) 1516 : cluster 0 pgmap v1745: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:39.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:39 smithi094 bash[59677]: audit 2023-10-09T09:27:39.505598+0000 mon.a (mon.0) 2065 : audit 1 from='client.? 172.21.15.16:0/1097202925' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:40.567 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T09:27:40.567 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T09:27:40.567 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T09:27:40.567 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T09:27:40.567 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:27:40.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:27:40.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:40 smithi094 bash[59677]: cluster 2023-10-09T09:27:39.535982+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T09:27:40.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:40 smithi094 bash[59677]: audit 2023-10-09T09:27:39.537621+0000 mon.a (mon.0) 2066 : audit 1 from='client.? 172.21.15.16:0/1097202925' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:27:40.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:40 smithi094 bash[59677]: cluster 2023-10-09T09:27:39.537799+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2023-10-09T09:27:40.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:40 smithi094 bash[59677]: cluster 2023-10-09T09:27:39.538695+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T09:27:41.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[75179]: cluster 2023-10-09T09:27:39.535982+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T09:27:41.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[75179]: audit 2023-10-09T09:27:39.537621+0000 mon.a (mon.0) 2066 : audit 1 from='client.? 172.21.15.16:0/1097202925' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:27:41.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[75179]: cluster 2023-10-09T09:27:39.537799+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2023-10-09T09:27:41.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[75179]: cluster 2023-10-09T09:27:39.538695+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T09:27:41.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[77892]: cluster 2023-10-09T09:27:39.535982+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T09:27:41.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[77892]: audit 2023-10-09T09:27:39.537621+0000 mon.a (mon.0) 2066 : audit 1 from='client.? 172.21.15.16:0/1097202925' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:27:41.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[77892]: cluster 2023-10-09T09:27:39.537799+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2023-10-09T09:27:41.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:40 smithi016 bash[77892]: cluster 2023-10-09T09:27:39.538695+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T09:27:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:41 smithi094 bash[59677]: cluster 2023-10-09T09:27:40.407332+0000 mgr.y (mgr.25421) 1517 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:41 smithi094 bash[59677]: cluster 2023-10-09T09:27:40.554009+0000 mon.a (mon.0) 2068 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-10-09T09:27:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:41 smithi094 bash[59677]: cluster 2023-10-09T09:27:40.554459+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:41 smithi094 bash[59677]: cluster 2023-10-09T09:27:40.556951+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:41.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:41 smithi094 bash[59677]: cluster 2023-10-09T09:27:40.672716+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:41.992 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:41 smithi094 bash[59677]: cluster 2023-10-09T09:27:40.675631+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[75179]: cluster 2023-10-09T09:27:40.407332+0000 mgr.y (mgr.25421) 1517 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[75179]: cluster 2023-10-09T09:27:40.554009+0000 mon.a (mon.0) 2068 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-10-09T09:27:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[75179]: cluster 2023-10-09T09:27:40.554459+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[75179]: cluster 2023-10-09T09:27:40.556951+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[75179]: cluster 2023-10-09T09:27:40.672716+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[75179]: cluster 2023-10-09T09:27:40.675631+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[77892]: cluster 2023-10-09T09:27:40.407332+0000 mgr.y (mgr.25421) 1517 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[77892]: cluster 2023-10-09T09:27:40.554009+0000 mon.a (mon.0) 2068 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[77892]: cluster 2023-10-09T09:27:40.554459+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[77892]: cluster 2023-10-09T09:27:40.556951+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[77892]: cluster 2023-10-09T09:27:40.672716+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:41 smithi016 bash[77892]: cluster 2023-10-09T09:27:40.675631+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T09:27:42.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:42 smithi094 bash[59677]: cluster 2023-10-09T09:27:42.551667+0000 mon.a (mon.0) 2069 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T09:27:43.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:42 smithi016 bash[75179]: cluster 2023-10-09T09:27:42.551667+0000 mon.a (mon.0) 2069 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T09:27:43.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:42 smithi016 bash[77892]: cluster 2023-10-09T09:27:42.551667+0000 mon.a (mon.0) 2069 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T09:27:43.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:43 smithi094 bash[59677]: cluster 2023-10-09T09:27:42.408160+0000 mgr.y (mgr.25421) 1518 : cluster 0 pgmap v1749: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:27:44.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:43 smithi016 bash[75179]: cluster 2023-10-09T09:27:42.408160+0000 mgr.y (mgr.25421) 1518 : cluster 0 pgmap v1749: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:27:44.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:43 smithi016 bash[77892]: cluster 2023-10-09T09:27:42.408160+0000 mgr.y (mgr.25421) 1518 : cluster 0 pgmap v1749: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:27:45.342 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:45.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:45 smithi016 bash[77892]: cluster 2023-10-09T09:27:44.409003+0000 mgr.y (mgr.25421) 1519 : cluster 0 pgmap v1750: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:45.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:45 smithi016 bash[77892]: audit 2023-10-09T09:27:45.589594+0000 mon.a (mon.0) 2070 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:45.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:45 smithi016 bash[75179]: cluster 2023-10-09T09:27:44.409003+0000 mgr.y (mgr.25421) 1519 : cluster 0 pgmap v1750: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:45.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:45 smithi016 bash[75179]: audit 2023-10-09T09:27:45.589594+0000 mon.a (mon.0) 2070 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:45.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:45 smithi094 bash[59677]: cluster 2023-10-09T09:27:44.409003+0000 mgr.y (mgr.25421) 1519 : cluster 0 pgmap v1750: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:45.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:45 smithi094 bash[59677]: audit 2023-10-09T09:27:45.589594+0000 mon.a (mon.0) 2070 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:46.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:27:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:27:45] "GET /metrics HTTP/1.1" 200 34068 "" "Prometheus/2.33.4" 2023-10-09T09:27:46.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:46 smithi016 bash[77892]: cluster 2023-10-09T09:27:46.609921+0000 mon.a (mon.0) 2071 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T09:27:46.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:46 smithi016 bash[77892]: cluster 2023-10-09T09:27:46.609980+0000 mon.a (mon.0) 2072 : cluster 1 Cluster is now healthy 2023-10-09T09:27:46.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:46 smithi016 bash[75179]: cluster 2023-10-09T09:27:46.609921+0000 mon.a (mon.0) 2071 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T09:27:46.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:46 smithi016 bash[75179]: cluster 2023-10-09T09:27:46.609980+0000 mon.a (mon.0) 2072 : cluster 1 Cluster is now healthy 2023-10-09T09:27:46.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:46 smithi094 bash[59677]: cluster 2023-10-09T09:27:46.609921+0000 mon.a (mon.0) 2071 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T09:27:46.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:46 smithi094 bash[59677]: cluster 2023-10-09T09:27:46.609980+0000 mon.a (mon.0) 2072 : cluster 1 Cluster is now healthy 2023-10-09T09:27:47.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:47 smithi094 bash[59677]: cluster 2023-10-09T09:27:46.410078+0000 mgr.y (mgr.25421) 1520 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:48.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:47 smithi016 bash[77892]: cluster 2023-10-09T09:27:46.410078+0000 mgr.y (mgr.25421) 1520 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:48.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:47 smithi016 bash[75179]: cluster 2023-10-09T09:27:46.410078+0000 mgr.y (mgr.25421) 1520 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:49.542 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:27:49.542 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":414,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T09:27:40.540944+0000","last_up_change":"2023-10-09T08:47:15.937734+0000","last_in_change":"2023-10-09T09:20:18.723367+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"390","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":390,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T08:34:26.406514+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T08:34:27.172366+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":74,"up_from":78,"up_thru":370,"down_at":75,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6803","nonce":2159748916}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6805","nonce":2159748916}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6809","nonce":2159748916}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6807","nonce":2159748916}]},"public_addr":"172.21.15.16:6803/2159748916","cluster_addr":"172.21.15.16:6805/2159748916","heartbeat_back_addr":"172.21.15.16:6809/2159748916","heartbeat_front_addr":"172.21.15.16:6807/2159748916","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":79,"up_from":83,"up_thru":406,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6811","nonce":3704674354}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6813","nonce":3704674354}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6817","nonce":3704674354}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6815","nonce":3704674354}]},"public_addr":"172.21.15.16:6811/3704674354","cluster_addr":"172.21.15.16:6813/3704674354","heartbeat_back_addr":"172.21.15.16:6817/3704674354","heartbeat_front_addr":"172.21.15.16:6815/3704674354","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":84,"up_from":88,"up_thru":370,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6819","nonce":1227549489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6821","nonce":1227549489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6825","nonce":1227549489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6823","nonce":1227549489}]},"public_addr":"172.21.15.16:6819/1227549489","cluster_addr":"172.21.15.16:6821/1227549489","heartbeat_back_addr":"172.21.15.16:6825/1227549489","heartbeat_front_addr":"172.21.15.16:6823/1227549489","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":89,"up_from":93,"up_thru":402,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6827","nonce":1617049985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6829","nonce":1617049985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6833","nonce":1617049985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6831","nonce":1617049985}]},"public_addr":"172.21.15.16:6827/1617049985","cluster_addr":"172.21.15.16:6829/1617049985","heartbeat_back_addr":"172.21.15.16:6833/1617049985","heartbeat_front_addr":"172.21.15.16:6831/1617049985","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":370,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":406,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":411,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":413,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":74},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":79},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":84},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":89},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1a","mappings":[{"from":3,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2419669477":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/1478513102":"2023-10-10T08:38:06.212642+0000","172.21.15.16:0/3538653132":"2023-10-10T08:40:49.059520+0000","172.21.15.94:0/3558864062":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.94:0/1375017130":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/3686891819":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.94:0/2012987098":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.94:0/727051339":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/3915272728":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/1021312597":"2023-10-10T08:38:06.212642+0000","172.21.15.16:0/3387393375":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.16:6801/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.94:6834/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.94:6835/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/2254945868":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:27:49.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:49 smithi094 bash[59677]: cluster 2023-10-09T09:27:48.410854+0000 mgr.y (mgr.25421) 1521 : cluster 0 pgmap v1752: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:49 smithi094 bash[59677]: audit 2023-10-09T09:27:49.272382+0000 mon.a (mon.0) 2073 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:49 smithi094 bash[59677]: audit 2023-10-09T09:27:49.537668+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.16:0/2852996241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:27:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:49 smithi016 bash[77892]: cluster 2023-10-09T09:27:48.410854+0000 mgr.y (mgr.25421) 1521 : cluster 0 pgmap v1752: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:49 smithi016 bash[77892]: audit 2023-10-09T09:27:49.272382+0000 mon.a (mon.0) 2073 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:50.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:49 smithi016 bash[77892]: audit 2023-10-09T09:27:49.537668+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.16:0/2852996241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:27:50.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:49 smithi016 bash[75179]: cluster 2023-10-09T09:27:48.410854+0000 mgr.y (mgr.25421) 1521 : cluster 0 pgmap v1752: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:50.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:49 smithi016 bash[75179]: audit 2023-10-09T09:27:49.272382+0000 mon.a (mon.0) 2073 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:50.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:49 smithi016 bash[75179]: audit 2023-10-09T09:27:49.537668+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.16:0/2852996241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:27:50.558 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:27:50.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:27:51.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:50 smithi016 bash[75179]: cluster 2023-10-09T09:27:50.411706+0000 mgr.y (mgr.25421) 1522 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:51.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:50 smithi016 bash[77892]: cluster 2023-10-09T09:27:50.411706+0000 mgr.y (mgr.25421) 1522 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:51.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:50 smithi094 bash[59677]: cluster 2023-10-09T09:27:50.411706+0000 mgr.y (mgr.25421) 1522 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:53.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:53 smithi094 bash[59677]: cluster 2023-10-09T09:27:52.412510+0000 mgr.y (mgr.25421) 1523 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:53.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:53 smithi016 bash[75179]: cluster 2023-10-09T09:27:52.412510+0000 mgr.y (mgr.25421) 1523 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:53.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:53 smithi016 bash[77892]: cluster 2023-10-09T09:27:52.412510+0000 mgr.y (mgr.25421) 1523 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:55.349 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:55.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:55 smithi016 bash[75179]: cluster 2023-10-09T09:27:54.413319+0000 mgr.y (mgr.25421) 1524 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:55.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:55 smithi016 bash[77892]: cluster 2023-10-09T09:27:54.413319+0000 mgr.y (mgr.25421) 1524 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:55.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:55 smithi094 bash[59677]: cluster 2023-10-09T09:27:54.413319+0000 mgr.y (mgr.25421) 1524 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:56.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:27:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:27:55] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.33.4" 2023-10-09T09:27:56.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:56 smithi016 bash[75179]: audit 2023-10-09T09:27:55.613072+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:56.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:56 smithi016 bash[77892]: audit 2023-10-09T09:27:55.613072+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:56.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:56 smithi094 bash[59677]: audit 2023-10-09T09:27:55.613072+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:27:57.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:57 smithi094 bash[59677]: cluster 2023-10-09T09:27:56.414316+0000 mgr.y (mgr.25421) 1525 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:58.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:57 smithi016 bash[75179]: cluster 2023-10-09T09:27:56.414316+0000 mgr.y (mgr.25421) 1525 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:58.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:57 smithi016 bash[77892]: cluster 2023-10-09T09:27:56.414316+0000 mgr.y (mgr.25421) 1525 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:59.608 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:27:59.609 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":414,"fsid":"f674ade6-667c-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:23:18.622123+0000","modified":"2023-10-09T09:27:40.540944+0000","last_up_change":"2023-10-09T08:47:15.937734+0000","last_in_change":"2023-10-09T09:20:18.723367+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:27:38.062560+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T08:32:36.561904+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"390","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":390,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T08:34:26.406514+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T08:34:27.172366+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0334a36b-f8d4-486c-a3f7-6b5278148938","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":74,"up_from":78,"up_thru":370,"down_at":75,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6803","nonce":2159748916}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6805","nonce":2159748916}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6809","nonce":2159748916}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2159748916},{"type":"v1","addr":"172.21.15.16:6807","nonce":2159748916}]},"public_addr":"172.21.15.16:6803/2159748916","cluster_addr":"172.21.15.16:6805/2159748916","heartbeat_back_addr":"172.21.15.16:6809/2159748916","heartbeat_front_addr":"172.21.15.16:6807/2159748916","state":["exists","up"]},{"osd":1,"uuid":"fc3d1f7f-18df-4428-90b3-68c1d8296b11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":79,"up_from":83,"up_thru":406,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6811","nonce":3704674354}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6813","nonce":3704674354}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6817","nonce":3704674354}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3704674354},{"type":"v1","addr":"172.21.15.16:6815","nonce":3704674354}]},"public_addr":"172.21.15.16:6811/3704674354","cluster_addr":"172.21.15.16:6813/3704674354","heartbeat_back_addr":"172.21.15.16:6817/3704674354","heartbeat_front_addr":"172.21.15.16:6815/3704674354","state":["exists","up"]},{"osd":2,"uuid":"19af46cb-b0d8-410d-bf62-d4352bafb831","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":84,"up_from":88,"up_thru":370,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6819","nonce":1227549489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6821","nonce":1227549489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6825","nonce":1227549489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":1227549489},{"type":"v1","addr":"172.21.15.16:6823","nonce":1227549489}]},"public_addr":"172.21.15.16:6819/1227549489","cluster_addr":"172.21.15.16:6821/1227549489","heartbeat_back_addr":"172.21.15.16:6825/1227549489","heartbeat_front_addr":"172.21.15.16:6823/1227549489","state":["exists","up"]},{"osd":3,"uuid":"9ee2e425-a2d1-432d-838c-e246649de096","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":89,"up_from":93,"up_thru":402,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6827","nonce":1617049985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6829","nonce":1617049985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6833","nonce":1617049985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":1617049985},{"type":"v1","addr":"172.21.15.16:6831","nonce":1617049985}]},"public_addr":"172.21.15.16:6827/1617049985","cluster_addr":"172.21.15.16:6829/1617049985","heartbeat_back_addr":"172.21.15.16:6833/1617049985","heartbeat_front_addr":"172.21.15.16:6831/1617049985","state":["exists","up"]},{"osd":4,"uuid":"44c51083-8ea0-4d32-8849-c5524f94afa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":370,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6801","nonce":2620675189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6803","nonce":2620675189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6807","nonce":2620675189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":2620675189},{"type":"v1","addr":"172.21.15.94:6805","nonce":2620675189}]},"public_addr":"172.21.15.94:6801/2620675189","cluster_addr":"172.21.15.94:6803/2620675189","heartbeat_back_addr":"172.21.15.94:6807/2620675189","heartbeat_front_addr":"172.21.15.94:6805/2620675189","state":["exists","up"]},{"osd":5,"uuid":"fc3fc2f1-b5f5-4986-9260-98c23cd9c49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":406,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6809","nonce":1438595646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6811","nonce":1438595646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6815","nonce":1438595646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1438595646},{"type":"v1","addr":"172.21.15.94:6813","nonce":1438595646}]},"public_addr":"172.21.15.94:6809/1438595646","cluster_addr":"172.21.15.94:6811/1438595646","heartbeat_back_addr":"172.21.15.94:6815/1438595646","heartbeat_front_addr":"172.21.15.94:6813/1438595646","state":["exists","up"]},{"osd":6,"uuid":"deb8f0b9-0b80-4dea-b180-353b84fb5daf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":411,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6817","nonce":4105872278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6819","nonce":4105872278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6823","nonce":4105872278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":4105872278},{"type":"v1","addr":"172.21.15.94:6821","nonce":4105872278}]},"public_addr":"172.21.15.94:6817/4105872278","cluster_addr":"172.21.15.94:6819/4105872278","heartbeat_back_addr":"172.21.15.94:6823/4105872278","heartbeat_front_addr":"172.21.15.94:6821/4105872278","state":["exists","up"]},{"osd":7,"uuid":"84cd66c8-c28e-4d58-ac4d-3836315c04f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":413,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6825","nonce":3615932234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6827","nonce":3615932234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6831","nonce":3615932234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":3615932234},{"type":"v1","addr":"172.21.15.94:6829","nonce":3615932234}]},"public_addr":"172.21.15.94:6825/3615932234","cluster_addr":"172.21.15.94:6827/3615932234","heartbeat_back_addr":"172.21.15.94:6831/3615932234","heartbeat_front_addr":"172.21.15.94:6829/3615932234","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:07.764758+0000","dead_epoch":74},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:26:50.992494+0000","dead_epoch":79},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:27:35.963585+0000","dead_epoch":84},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:28:20.549982+0000","dead_epoch":89},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:04.411948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:29:52.489248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:30:40.258073+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:31:28.263782+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1a","mappings":[{"from":3,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:0/147602172":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/2419669477":"2023-10-10T08:40:49.059520+0000","172.21.15.16:0/1478513102":"2023-10-10T08:38:06.212642+0000","172.21.15.16:0/3538653132":"2023-10-10T08:40:49.059520+0000","172.21.15.94:0/3558864062":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/1403279027":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/1291849630":"2023-10-10T08:32:04.579457+0000","172.21.15.94:0/1375017130":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/1383471245":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/1247249547":"2023-10-10T08:24:16.162112+0000","172.21.15.16:0/3686891819":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.94:0/2012987098":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6801/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:6801/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.94:0/727051339":"2023-10-10T08:39:43.331826+0000","172.21.15.16:0/2576600022":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/3915272728":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6801/4047990666":"2023-10-10T08:24:16.162112+0000","172.21.15.16:6800/243905061":"2023-10-10T08:23:46.198487+0000","172.21.15.16:0/1021312597":"2023-10-10T08:38:06.212642+0000","172.21.15.16:0/3387393375":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6800/1710200957":"2023-10-10T08:38:06.212642+0000","172.21.15.16:6801/2495526270":"2023-10-10T08:40:49.059520+0000","172.21.15.94:6834/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.94:6835/2010618240":"2023-10-10T08:39:43.331826+0000","172.21.15.16:6800/3038487937":"2023-10-10T08:32:04.579457+0000","172.21.15.16:0/2254945868":"2023-10-10T08:40:49.059520+0000","172.21.15.16:6801/3038487937":"2023-10-10T08:32:04.579457+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:27:59.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:59 smithi094 bash[59677]: cluster 2023-10-09T09:27:58.415037+0000 mgr.y (mgr.25421) 1526 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:27:59.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:27:59 smithi094 bash[59677]: audit 2023-10-09T09:27:59.603763+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.16:0/2938060613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:28:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:59 smithi016 bash[75179]: cluster 2023-10-09T09:27:58.415037+0000 mgr.y (mgr.25421) 1526 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:27:59 smithi016 bash[75179]: audit 2023-10-09T09:27:59.603763+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.16:0/2938060613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:28:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:59 smithi016 bash[77892]: cluster 2023-10-09T09:27:58.415037+0000 mgr.y (mgr.25421) 1526 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:27:59 smithi016 bash[77892]: audit 2023-10-09T09:27:59.603763+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.16:0/2938060613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:28:00.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T09:28:00.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T09:28:00.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T09:28:00.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T09:28:00.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T09:28:00.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T09:28:00.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T09:28:00.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T09:28:01.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:00 smithi016 bash[75179]: cluster 2023-10-09T09:28:00.415613+0000 mgr.y (mgr.25421) 1527 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:01.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:00 smithi016 bash[77892]: cluster 2023-10-09T09:28:00.415613+0000 mgr.y (mgr.25421) 1527 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:01.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:00 smithi094 bash[59677]: cluster 2023-10-09T09:28:00.415613+0000 mgr.y (mgr.25421) 1527 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:03.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:03 smithi094 bash[59677]: cluster 2023-10-09T09:28:02.416428+0000 mgr.y (mgr.25421) 1528 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:03 smithi016 bash[75179]: cluster 2023-10-09T09:28:02.416428+0000 mgr.y (mgr.25421) 1528 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:03.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:03 smithi016 bash[77892]: cluster 2023-10-09T09:28:02.416428+0000 mgr.y (mgr.25421) 1528 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:04.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:04 smithi094 bash[59677]: audit 2023-10-09T09:28:04.272782+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:04 smithi016 bash[75179]: audit 2023-10-09T09:28:04.272782+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:04.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:04 smithi016 bash[77892]: audit 2023-10-09T09:28:04.272782+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:05.464 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.465 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.465 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.471 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.471 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.472 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.472 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.472 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:05.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:05 smithi016 bash[77892]: cluster 2023-10-09T09:28:04.417195+0000 mgr.y (mgr.25421) 1529 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:05.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:05 smithi016 bash[75179]: cluster 2023-10-09T09:28:04.417195+0000 mgr.y (mgr.25421) 1529 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:05.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:05 smithi094 bash[59677]: cluster 2023-10-09T09:28:04.417195+0000 mgr.y (mgr.25421) 1529 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:06.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:28:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:28:05] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.33.4" 2023-10-09T09:28:07.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:07 smithi094 bash[59677]: cluster 2023-10-09T09:28:06.418076+0000 mgr.y (mgr.25421) 1530 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:07.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:07 smithi016 bash[75179]: cluster 2023-10-09T09:28:06.418076+0000 mgr.y (mgr.25421) 1530 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:07.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:07 smithi016 bash[77892]: cluster 2023-10-09T09:28:06.418076+0000 mgr.y (mgr.25421) 1530 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:09.734 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:09 smithi016 bash[75179]: cluster 2023-10-09T09:28:08.418837+0000 mgr.y (mgr.25421) 1531 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:09.735 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:09 smithi016 bash[77892]: cluster 2023-10-09T09:28:08.418837+0000 mgr.y (mgr.25421) 1531 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:09.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:09 smithi094 bash[59677]: cluster 2023-10-09T09:28:08.418837+0000 mgr.y (mgr.25421) 1531 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:11.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:10 smithi016 bash[75179]: cluster 2023-10-09T09:28:10.419708+0000 mgr.y (mgr.25421) 1532 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:11.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:11 smithi016 bash[77892]: cluster 2023-10-09T09:28:10.419708+0000 mgr.y (mgr.25421) 1532 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:11.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:10 smithi094 bash[59677]: cluster 2023-10-09T09:28:10.419708+0000 mgr.y (mgr.25421) 1532 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:13.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:13 smithi094 bash[59677]: cluster 2023-10-09T09:28:12.420565+0000 mgr.y (mgr.25421) 1533 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:13.743 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:13 smithi016 bash[75179]: cluster 2023-10-09T09:28:12.420565+0000 mgr.y (mgr.25421) 1533 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:13.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:13 smithi016 bash[77892]: cluster 2023-10-09T09:28:12.420565+0000 mgr.y (mgr.25421) 1533 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:15.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[75179]: cluster 2023-10-09T09:28:14.421352+0000 mgr.y (mgr.25421) 1534 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[75179]: audit 2023-10-09T09:28:14.834236+0000 mon.a (mon.0) 2076 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:28:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[75179]: audit 2023-10-09T09:28:15.210886+0000 mon.a (mon.0) 2077 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[75179]: audit 2023-10-09T09:28:15.217283+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[77892]: cluster 2023-10-09T09:28:14.421352+0000 mgr.y (mgr.25421) 1534 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[77892]: audit 2023-10-09T09:28:14.834236+0000 mon.a (mon.0) 2076 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:28:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[77892]: audit 2023-10-09T09:28:15.210886+0000 mon.a (mon.0) 2077 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[77892]: audit 2023-10-09T09:28:15.217283+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:15.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:15 smithi094 bash[59677]: cluster 2023-10-09T09:28:14.421352+0000 mgr.y (mgr.25421) 1534 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:15.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:15 smithi094 bash[59677]: audit 2023-10-09T09:28:14.834236+0000 mon.a (mon.0) 2076 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:28:15.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:15 smithi094 bash[59677]: audit 2023-10-09T09:28:15.210886+0000 mon.a (mon.0) 2077 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:15.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:15 smithi094 bash[59677]: audit 2023-10-09T09:28:15.217283+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:28:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:28:15] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.33.4" 2023-10-09T09:28:17.741 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:17 smithi094 bash[59677]: cluster 2023-10-09T09:28:16.422271+0000 mgr.y (mgr.25421) 1535 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:17.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:17 smithi016 bash[75179]: cluster 2023-10-09T09:28:16.422271+0000 mgr.y (mgr.25421) 1535 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:17.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:17 smithi016 bash[77892]: cluster 2023-10-09T09:28:16.422271+0000 mgr.y (mgr.25421) 1535 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:19.736 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:19 smithi016 bash[75179]: cluster 2023-10-09T09:28:18.423103+0000 mgr.y (mgr.25421) 1536 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:19.737 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:19 smithi016 bash[75179]: audit 2023-10-09T09:28:19.273345+0000 mon.a (mon.0) 2079 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:19.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:19 smithi016 bash[77892]: cluster 2023-10-09T09:28:18.423103+0000 mgr.y (mgr.25421) 1536 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:19.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:19 smithi016 bash[77892]: audit 2023-10-09T09:28:19.273345+0000 mon.a (mon.0) 2079 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:19.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:19 smithi094 bash[59677]: cluster 2023-10-09T09:28:18.423103+0000 mgr.y (mgr.25421) 1536 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:19.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:19 smithi094 bash[59677]: audit 2023-10-09T09:28:19.273345+0000 mon.a (mon.0) 2079 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:21.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:21 smithi016 bash[75179]: cluster 2023-10-09T09:28:20.423828+0000 mgr.y (mgr.25421) 1537 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:21 smithi016 bash[77892]: cluster 2023-10-09T09:28:20.423828+0000 mgr.y (mgr.25421) 1537 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:21.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:21 smithi094 bash[59677]: cluster 2023-10-09T09:28:20.423828+0000 mgr.y (mgr.25421) 1537 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:22.414 INFO:teuthology.orchestra.run.smithi016.stdout:154618823361 2023-10-09T09:28:22.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T09:28:22.580 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:22 smithi016 bash[75179]: audit 2023-10-09T09:28:21.505290+0000 mon.a (mon.0) 2080 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:22.581 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:22 smithi016 bash[75179]: audit 2023-10-09T09:28:21.510993+0000 mon.a (mon.0) 2081 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:22.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:22 smithi016 bash[77892]: audit 2023-10-09T09:28:21.505290+0000 mon.a (mon.0) 2080 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:22.692 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:22 smithi016 bash[77892]: audit 2023-10-09T09:28:21.510993+0000 mon.a (mon.0) 2081 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:22.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:22 smithi094 bash[59677]: audit 2023-10-09T09:28:21.505290+0000 mon.a (mon.0) 2080 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:22.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:22 smithi094 bash[59677]: audit 2023-10-09T09:28:21.510993+0000 mon.a (mon.0) 2081 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:23.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:23 smithi016 bash[77892]: cluster 2023-10-09T09:28:22.424408+0000 mgr.y (mgr.25421) 1538 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:23.921 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:23 smithi016 bash[75179]: cluster 2023-10-09T09:28:22.424408+0000 mgr.y (mgr.25421) 1538 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:23.921 INFO:teuthology.orchestra.run.smithi016.stdout:377957122549 2023-10-09T09:28:23.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T09:28:23.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:23 smithi094 bash[59677]: cluster 2023-10-09T09:28:22.424408+0000 mgr.y (mgr.25421) 1538 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:25.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:25 smithi094 bash[59677]: cluster 2023-10-09T09:28:24.424936+0000 mgr.y (mgr.25421) 1539 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:26.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:25 smithi016 bash[75179]: cluster 2023-10-09T09:28:24.424936+0000 mgr.y (mgr.25421) 1539 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:26.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:28:25 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:28:25] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.33.4" 2023-10-09T09:28:26.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:25 smithi016 bash[77892]: cluster 2023-10-09T09:28:24.424936+0000 mgr.y (mgr.25421) 1539 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:27.156 INFO:teuthology.orchestra.run.smithi016.stdout:335007449601 2023-10-09T09:28:27.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T09:28:27.364 INFO:teuthology.orchestra.run.smithi016.stdout:180388627128 2023-10-09T09:28:27.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T09:28:27.664 INFO:teuthology.orchestra.run.smithi016.stdout:356482286076 2023-10-09T09:28:27.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T09:28:27.961 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:27 smithi016 bash[75179]: cluster 2023-10-09T09:28:26.425818+0000 mgr.y (mgr.25421) 1540 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:27.961 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:27 smithi016 bash[77892]: cluster 2023-10-09T09:28:26.425818+0000 mgr.y (mgr.25421) 1540 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:27.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:27 smithi094 bash[59677]: cluster 2023-10-09T09:28:26.425818+0000 mgr.y (mgr.25421) 1540 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:28.074 INFO:teuthology.orchestra.run.smithi016.stdout:206158430895 2023-10-09T09:28:28.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T09:28:28.830 INFO:teuthology.orchestra.run.smithi016.stdout:399431959024 2023-10-09T09:28:28.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T09:28:28.874 INFO:teuthology.orchestra.run.smithi016.stdout:128849019592 2023-10-09T09:28:28.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T09:28:29.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:29 smithi094 bash[59677]: cluster 2023-10-09T09:28:28.426621+0000 mgr.y (mgr.25421) 1541 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:30.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:29 smithi016 bash[75179]: cluster 2023-10-09T09:28:28.426621+0000 mgr.y (mgr.25421) 1541 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:30.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:29 smithi016 bash[77892]: cluster 2023-10-09T09:28:28.426621+0000 mgr.y (mgr.25421) 1541 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:31.914 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:31.916 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:31.917 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:31.917 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:31.917 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:31.918 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:31.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[75179]: cluster 2023-10-09T09:28:30.427386+0000 mgr.y (mgr.25421) 1542 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:31.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[75179]: audit 2023-10-09T09:28:30.560429+0000 mon.a (mon.0) 2082 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[75179]: audit 2023-10-09T09:28:30.568526+0000 mon.a (mon.0) 2083 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[75179]: audit 2023-10-09T09:28:31.037909+0000 mon.a (mon.0) 2084 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:28:31.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[75179]: audit 2023-10-09T09:28:31.039923+0000 mon.a (mon.0) 2085 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:28:31.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[75179]: audit 2023-10-09T09:28:31.051720+0000 mon.a (mon.0) 2086 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.948 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[77892]: cluster 2023-10-09T09:28:30.427386+0000 mgr.y (mgr.25421) 1542 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:31.948 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[77892]: audit 2023-10-09T09:28:30.560429+0000 mon.a (mon.0) 2082 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.948 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[77892]: audit 2023-10-09T09:28:30.568526+0000 mon.a (mon.0) 2083 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.948 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[77892]: audit 2023-10-09T09:28:31.037909+0000 mon.a (mon.0) 2084 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:28:31.948 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[77892]: audit 2023-10-09T09:28:31.039923+0000 mon.a (mon.0) 2085 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:28:31.948 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:31 smithi016 bash[77892]: audit 2023-10-09T09:28:31.051720+0000 mon.a (mon.0) 2086 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:31 smithi094 bash[59677]: cluster 2023-10-09T09:28:30.427386+0000 mgr.y (mgr.25421) 1542 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:31 smithi094 bash[59677]: audit 2023-10-09T09:28:30.560429+0000 mon.a (mon.0) 2082 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:31 smithi094 bash[59677]: audit 2023-10-09T09:28:30.568526+0000 mon.a (mon.0) 2083 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:31 smithi094 bash[59677]: audit 2023-10-09T09:28:31.037909+0000 mon.a (mon.0) 2084 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:28:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:31 smithi094 bash[59677]: audit 2023-10-09T09:28:31.039923+0000 mon.a (mon.0) 2085 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:28:31.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:31 smithi094 bash[59677]: audit 2023-10-09T09:28:31.051720+0000 mon.a (mon.0) 2086 : audit 1 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' 2023-10-09T09:28:32.924 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:32.924 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:33.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:33 smithi094 bash[59677]: cluster 2023-10-09T09:28:32.428019+0000 mgr.y (mgr.25421) 1543 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:34.020 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:33 smithi016 bash[75179]: cluster 2023-10-09T09:28:32.428019+0000 mgr.y (mgr.25421) 1543 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:34.020 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:33 smithi016 bash[77892]: cluster 2023-10-09T09:28:32.428019+0000 mgr.y (mgr.25421) 1543 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:34.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:34 smithi094 bash[59677]: audit 2023-10-09T09:28:34.273094+0000 mon.a (mon.0) 2087 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:35.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:34 smithi016 bash[75179]: audit 2023-10-09T09:28:34.273094+0000 mon.a (mon.0) 2087 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:35.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:34 smithi016 bash[77892]: audit 2023-10-09T09:28:34.273094+0000 mon.a (mon.0) 2087 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:35.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:35 smithi016 bash[75179]: cluster 2023-10-09T09:28:34.428666+0000 mgr.y (mgr.25421) 1544 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:35.870 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:35 smithi016 bash[77892]: cluster 2023-10-09T09:28:34.428666+0000 mgr.y (mgr.25421) 1544 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:35.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:35 smithi094 bash[59677]: cluster 2023-10-09T09:28:34.428666+0000 mgr.y (mgr.25421) 1544 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:36.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:28:35 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:28:35] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.33.4" 2023-10-09T09:28:37.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:37 smithi016 bash[77892]: cluster 2023-10-09T09:28:36.429675+0000 mgr.y (mgr.25421) 1545 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:37.959 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:37 smithi016 bash[75179]: cluster 2023-10-09T09:28:36.429675+0000 mgr.y (mgr.25421) 1545 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:37.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:37 smithi094 bash[59677]: cluster 2023-10-09T09:28:36.429675+0000 mgr.y (mgr.25421) 1545 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:39.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:39 smithi094 bash[59677]: cluster 2023-10-09T09:28:38.430510+0000 mgr.y (mgr.25421) 1546 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:40.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:39 smithi016 bash[77892]: cluster 2023-10-09T09:28:38.430510+0000 mgr.y (mgr.25421) 1546 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:40.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:39 smithi016 bash[75179]: cluster 2023-10-09T09:28:38.430510+0000 mgr.y (mgr.25421) 1546 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:41.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:41 smithi016 bash[75179]: cluster 2023-10-09T09:28:40.431118+0000 mgr.y (mgr.25421) 1547 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:41.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:41 smithi016 bash[77892]: cluster 2023-10-09T09:28:40.431118+0000 mgr.y (mgr.25421) 1547 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:41.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:41 smithi094 bash[59677]: cluster 2023-10-09T09:28:40.431118+0000 mgr.y (mgr.25421) 1547 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:43.740 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:43 smithi094 bash[59677]: cluster 2023-10-09T09:28:42.431816+0000 mgr.y (mgr.25421) 1548 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:43.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:43 smithi016 bash[75179]: cluster 2023-10-09T09:28:42.431816+0000 mgr.y (mgr.25421) 1548 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:43.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:43 smithi016 bash[77892]: cluster 2023-10-09T09:28:42.431816+0000 mgr.y (mgr.25421) 1548 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:45.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:45 smithi016 bash[75179]: cluster 2023-10-09T09:28:44.432637+0000 mgr.y (mgr.25421) 1549 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:45.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:45 smithi016 bash[77892]: cluster 2023-10-09T09:28:44.432637+0000 mgr.y (mgr.25421) 1549 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:45.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:45 smithi094 bash[59677]: cluster 2023-10-09T09:28:44.432637+0000 mgr.y (mgr.25421) 1549 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:46.072 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:28:45 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:28:45] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.33.4" 2023-10-09T09:28:46.226 INFO:teuthology.orchestra.run.smithi016.stdout:335007449606 2023-10-09T09:28:46.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:46 smithi016 bash[75179]: audit 2023-10-09T09:28:46.225006+0000 mon.a (mon.0) 2088 : audit 0 from='client.? 172.21.15.16:0/903522580' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:28:46.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:46 smithi016 bash[77892]: audit 2023-10-09T09:28:46.225006+0000 mon.a (mon.0) 2088 : audit 0 from='client.? 172.21.15.16:0/903522580' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:28:46.890 INFO:teuthology.orchestra.run.smithi016.stdout:206158430900 2023-10-09T09:28:46.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:46 smithi094 bash[59677]: audit 2023-10-09T09:28:46.225006+0000 mon.a (mon.0) 2088 : audit 0 from='client.? 172.21.15.16:0/903522580' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:28:47.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:47 smithi094 bash[59677]: cluster 2023-10-09T09:28:46.433419+0000 mgr.y (mgr.25421) 1550 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:47.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:47 smithi094 bash[59677]: audit 2023-10-09T09:28:46.888333+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.16:0/1764032928' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:28:48.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:47 smithi016 bash[75179]: cluster 2023-10-09T09:28:46.433419+0000 mgr.y (mgr.25421) 1550 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:48.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:47 smithi016 bash[75179]: audit 2023-10-09T09:28:46.888333+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.16:0/1764032928' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:28:48.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:47 smithi016 bash[77892]: cluster 2023-10-09T09:28:46.433419+0000 mgr.y (mgr.25421) 1550 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:48.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:47 smithi016 bash[77892]: audit 2023-10-09T09:28:46.888333+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.16:0/1764032928' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:28:49.307 INFO:tasks.cephadm.ceph_manager.ceph:need seq 335007449601 got 335007449606 for osd.0 2023-10-09T09:28:49.307 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:49.591 INFO:teuthology.orchestra.run.smithi016.stdout:154618823367 2023-10-09T09:28:49.832 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:49 smithi016 bash[75179]: cluster 2023-10-09T09:28:48.433955+0000 mgr.y (mgr.25421) 1551 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:49.833 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:49 smithi016 bash[75179]: audit 2023-10-09T09:28:49.273859+0000 mon.a (mon.0) 2089 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:49.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:49 smithi016 bash[77892]: cluster 2023-10-09T09:28:48.433955+0000 mgr.y (mgr.25421) 1551 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:49.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:49 smithi016 bash[77892]: audit 2023-10-09T09:28:49.273859+0000 mon.a (mon.0) 2089 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:49.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:49 smithi094 bash[59677]: cluster 2023-10-09T09:28:48.433955+0000 mgr.y (mgr.25421) 1551 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:49.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:49 smithi094 bash[59677]: audit 2023-10-09T09:28:49.273859+0000 mon.a (mon.0) 2089 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:50.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:50 smithi016 bash[75179]: audit 2023-10-09T09:28:49.590187+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.16:0/3567023327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:28:50.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:50 smithi016 bash[77892]: audit 2023-10-09T09:28:49.590187+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.16:0/3567023327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:28:50.794 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430895 got 206158430900 for osd.7 2023-10-09T09:28:50.795 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:50.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:50 smithi094 bash[59677]: audit 2023-10-09T09:28:49.590187+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.16:0/3567023327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:28:51.166 INFO:teuthology.orchestra.run.smithi016.stdout:377957122555 2023-10-09T09:28:51.360 INFO:teuthology.orchestra.run.smithi016.stdout:356482286081 2023-10-09T09:28:51.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:51 smithi016 bash[75179]: cluster 2023-10-09T09:28:50.434559+0000 mgr.y (mgr.25421) 1552 : cluster 0 pgmap v1783: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:51.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:51 smithi016 bash[75179]: audit 2023-10-09T09:28:51.165256+0000 mon.a (mon.0) 2090 : audit 0 from='client.? 172.21.15.16:0/2829162911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:28:51.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:51 smithi016 bash[75179]: audit 2023-10-09T09:28:51.358393+0000 mon.a (mon.0) 2091 : audit 0 from='client.? 172.21.15.16:0/3709469021' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:28:51.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:51 smithi016 bash[77892]: cluster 2023-10-09T09:28:50.434559+0000 mgr.y (mgr.25421) 1552 : cluster 0 pgmap v1783: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:51.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:51 smithi016 bash[77892]: audit 2023-10-09T09:28:51.165256+0000 mon.a (mon.0) 2090 : audit 0 from='client.? 172.21.15.16:0/2829162911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:28:51.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:51 smithi016 bash[77892]: audit 2023-10-09T09:28:51.358393+0000 mon.a (mon.0) 2091 : audit 0 from='client.? 172.21.15.16:0/3709469021' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:28:51.902 INFO:teuthology.orchestra.run.smithi016.stdout:399431959029 2023-10-09T09:28:51.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:51 smithi094 bash[59677]: cluster 2023-10-09T09:28:50.434559+0000 mgr.y (mgr.25421) 1552 : cluster 0 pgmap v1783: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:51.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:51 smithi094 bash[59677]: audit 2023-10-09T09:28:51.165256+0000 mon.a (mon.0) 2090 : audit 0 from='client.? 172.21.15.16:0/2829162911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:28:51.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:51 smithi094 bash[59677]: audit 2023-10-09T09:28:51.358393+0000 mon.a (mon.0) 2091 : audit 0 from='client.? 172.21.15.16:0/3709469021' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:28:52.056 INFO:teuthology.orchestra.run.smithi016.stdout:180388627134 2023-10-09T09:28:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:52 smithi016 bash[75179]: audit 2023-10-09T09:28:51.898391+0000 mon.a (mon.0) 2092 : audit 0 from='client.? 172.21.15.16:0/4083734946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:28:52.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:52 smithi016 bash[75179]: audit 2023-10-09T09:28:52.054358+0000 mon.c (mon.2) 145 : audit 0 from='client.? 172.21.15.16:0/368752274' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:28:52.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:52 smithi016 bash[77892]: audit 2023-10-09T09:28:51.898391+0000 mon.a (mon.0) 2092 : audit 0 from='client.? 172.21.15.16:0/4083734946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:28:52.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:52 smithi016 bash[77892]: audit 2023-10-09T09:28:52.054358+0000 mon.c (mon.2) 145 : audit 0 from='client.? 172.21.15.16:0/368752274' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:28:52.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:52 smithi094 bash[59677]: audit 2023-10-09T09:28:51.898391+0000 mon.a (mon.0) 2092 : audit 0 from='client.? 172.21.15.16:0/4083734946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:28:52.991 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:52 smithi094 bash[59677]: audit 2023-10-09T09:28:52.054358+0000 mon.c (mon.2) 145 : audit 0 from='client.? 172.21.15.16:0/368752274' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:28:53.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:53 smithi016 bash[75179]: cluster 2023-10-09T09:28:52.435150+0000 mgr.y (mgr.25421) 1553 : cluster 0 pgmap v1784: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:53.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:53 smithi016 bash[77892]: cluster 2023-10-09T09:28:52.435150+0000 mgr.y (mgr.25421) 1553 : cluster 0 pgmap v1784: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:53.941 INFO:teuthology.orchestra.run.smithi016.stdout:128849019598 2023-10-09T09:28:53.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:53 smithi094 bash[59677]: cluster 2023-10-09T09:28:52.435150+0000 mgr.y (mgr.25421) 1553 : cluster 0 pgmap v1784: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:54.137 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823361 got 154618823367 for osd.5 2023-10-09T09:28:54.138 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:54.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:54 smithi016 bash[75179]: audit 2023-10-09T09:28:53.939877+0000 mon.a (mon.0) 2093 : audit 0 from='client.? 172.21.15.16:0/3239075017' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:28:54.811 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:54 smithi016 bash[77892]: audit 2023-10-09T09:28:53.939877+0000 mon.a (mon.0) 2093 : audit 0 from='client.? 172.21.15.16:0/3239075017' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:28:54.811 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122549 got 377957122555 for osd.2 2023-10-09T09:28:54.812 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:54.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:54 smithi094 bash[59677]: audit 2023-10-09T09:28:53.939877+0000 mon.a (mon.0) 2093 : audit 0 from='client.? 172.21.15.16:0/3239075017' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:28:55.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:55 smithi094 bash[59677]: cluster 2023-10-09T09:28:54.435897+0000 mgr.y (mgr.25421) 1554 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:56.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:55 smithi016 bash[75179]: cluster 2023-10-09T09:28:54.435897+0000 mgr.y (mgr.25421) 1554 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:56.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:55 smithi016 bash[77892]: cluster 2023-10-09T09:28:54.435897+0000 mgr.y (mgr.25421) 1554 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:56.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:28:55 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:28:55] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.33.4" 2023-10-09T09:28:56.476 INFO:tasks.cephadm.ceph_manager.ceph:need seq 356482286076 got 356482286081 for osd.1 2023-10-09T09:28:56.476 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:56.613 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431959024 got 399431959029 for osd.3 2023-10-09T09:28:56.613 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:56.635 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388627128 got 180388627134 for osd.6 2023-10-09T09:28:56.635 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:56.739 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019592 got 128849019598 for osd.4 2023-10-09T09:28:56.739 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:56.739 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T09:28:56.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:28:57.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:57 smithi094 bash[59677]: cluster 2023-10-09T09:28:56.436910+0000 mgr.y (mgr.25421) 1555 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:58.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:57 smithi016 bash[75179]: cluster 2023-10-09T09:28:56.436910+0000 mgr.y (mgr.25421) 1555 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:58.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:57 smithi016 bash[77892]: cluster 2023-10-09T09:28:56.436910+0000 mgr.y (mgr.25421) 1555 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:28:59.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:28:59 smithi094 bash[59677]: cluster 2023-10-09T09:28:58.437842+0000 mgr.y (mgr.25421) 1556 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:00.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:28:59 smithi016 bash[75179]: cluster 2023-10-09T09:28:58.437842+0000 mgr.y (mgr.25421) 1556 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:00.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:28:59 smithi016 bash[77892]: cluster 2023-10-09T09:28:58.437842+0000 mgr.y (mgr.25421) 1556 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:01.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:01 smithi094 bash[59677]: cluster 2023-10-09T09:29:00.438705+0000 mgr.y (mgr.25421) 1557 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:01.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:01 smithi016 bash[75179]: cluster 2023-10-09T09:29:00.438705+0000 mgr.y (mgr.25421) 1557 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:01.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:01 smithi016 bash[77892]: cluster 2023-10-09T09:29:00.438705+0000 mgr.y (mgr.25421) 1557 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:01.524 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:03.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:03 smithi016 bash[75179]: cluster 2023-10-09T09:29:02.439568+0000 mgr.y (mgr.25421) 1558 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:03.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:03 smithi016 bash[77892]: cluster 2023-10-09T09:29:02.439568+0000 mgr.y (mgr.25421) 1558 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:03.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:03 smithi094 bash[59677]: cluster 2023-10-09T09:29:02.439568+0000 mgr.y (mgr.25421) 1558 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:04.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:04 smithi094 bash[59677]: audit 2023-10-09T09:29:04.274286+0000 mon.a (mon.0) 2094 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:05.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:04 smithi016 bash[75179]: audit 2023-10-09T09:29:04.274286+0000 mon.a (mon.0) 2094 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:05.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:04 smithi016 bash[77892]: audit 2023-10-09T09:29:04.274286+0000 mon.a (mon.0) 2094 : audit 0 from='mgr.25421 172.21.15.16:0/333398764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:05.547 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:29:05.549 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:29:05.802 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:05 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:29:05] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.33.4" 2023-10-09T09:29:05.802 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:05 smithi016 bash[75179]: cluster 2023-10-09T09:29:04.440038+0000 mgr.y (mgr.25421) 1559 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:05.803 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:05 smithi016 bash[77892]: cluster 2023-10-09T09:29:04.440038+0000 mgr.y (mgr.25421) 1559 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:05.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:05 smithi094 bash[59677]: cluster 2023-10-09T09:29:04.440038+0000 mgr.y (mgr.25421) 1559 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:06.594 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1790,"stamp":"2023-10-09T09:29:04.439775+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6288,"num_read_kb":12724,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":2972600,"num_keys_recovered":70,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7827860,"kb_used_data":46292,"kb_used_omap":126,"kb_used_meta":7781377,"kb_avail":742133356,"statfs":{"total":767960285184,"available":759944556544,"internally_reserved":0,"allocated":47403008,"data_stored":38979357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":7968130683},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004930"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":321,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547613+0000","last_change":"2023-10-09T09:20:19.894893+0000","last_active":"2023-10-09T09:27:39.547613+0000","last_peered":"2023-10-09T09:27:39.547613+0000","last_clean":"2023-10-09T09:27:39.547613+0000","last_became_active":"2023-10-09T09:20:19.893953+0000","last_became_peered":"2023-10-09T09:20:19.893953+0000","last_unstale":"2023-10-09T09:27:39.547613+0000","last_undegraded":"2023-10-09T09:27:39.547613+0000","last_fullsized":"2023-10-09T09:27:39.547613+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":259,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673574+0000","last_change":"2023-10-09T09:26:19.643487+0000","last_active":"2023-10-09T09:27:40.673574+0000","last_peered":"2023-10-09T09:27:40.673574+0000","last_clean":"2023-10-09T09:27:40.673574+0000","last_became_active":"2023-10-09T09:26:19.641799+0000","last_became_peered":"2023-10-09T09:26:19.641799+0000","last_unstale":"2023-10-09T09:27:40.673574+0000","last_undegraded":"2023-10-09T09:27:40.673574+0000","last_fullsized":"2023-10-09T09:27:40.673574+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:32.569466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":165,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549236+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:27:39.549236+0000","last_peered":"2023-10-09T09:27:39.549236+0000","last_clean":"2023-10-09T09:27:39.549236+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:27:39.549236+0000","last_undegraded":"2023-10-09T09:27:39.549236+0000","last_fullsized":"2023-10-09T09:27:39.549236+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":454,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562334+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:27:40.562334+0000","last_peered":"2023-10-09T09:27:40.562334+0000","last_clean":"2023-10-09T09:27:40.562334+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:27:40.562334+0000","last_undegraded":"2023-10-09T09:27:40.562334+0000","last_fullsized":"2023-10-09T09:27:40.562334+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673116+0000","last_change":"2023-10-09T09:26:19.644393+0000","last_active":"2023-10-09T09:27:40.673116+0000","last_peered":"2023-10-09T09:27:40.673116+0000","last_clean":"2023-10-09T09:27:40.673116+0000","last_became_active":"2023-10-09T09:26:19.643611+0000","last_became_peered":"2023-10-09T09:26:19.643611+0000","last_unstale":"2023-10-09T09:27:40.673116+0000","last_undegraded":"2023-10-09T09:27:40.673116+0000","last_fullsized":"2023-10-09T09:27:40.673116+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:50.548997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":326,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549815+0000","last_change":"2023-10-09T09:19:42.872407+0000","last_active":"2023-10-09T09:27:39.549815+0000","last_peered":"2023-10-09T09:27:39.549815+0000","last_clean":"2023-10-09T09:27:39.549815+0000","last_became_active":"2023-10-09T09:19:42.871637+0000","last_became_peered":"2023-10-09T09:19:42.871637+0000","last_unstale":"2023-10-09T09:27:39.549815+0000","last_undegraded":"2023-10-09T09:27:39.549815+0000","last_fullsized":"2023-10-09T09:27:39.549815+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":405,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536423+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:27:39.536423+0000","last_peered":"2023-10-09T09:27:39.536423+0000","last_clean":"2023-10-09T09:27:39.536423+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:27:39.536423+0000","last_undegraded":"2023-10-09T09:27:39.536423+0000","last_fullsized":"2023-10-09T09:27:39.536423+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":260,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673205+0000","last_change":"2023-10-09T09:26:19.644374+0000","last_active":"2023-10-09T09:27:40.673205+0000","last_peered":"2023-10-09T09:27:40.673205+0000","last_clean":"2023-10-09T09:27:40.673205+0000","last_became_active":"2023-10-09T09:26:19.642350+0000","last_became_peered":"2023-10-09T09:26:19.642350+0000","last_unstale":"2023-10-09T09:27:40.673205+0000","last_undegraded":"2023-10-09T09:27:40.673205+0000","last_fullsized":"2023-10-09T09:27:40.673205+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550028+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:27:39.550028+0000","last_peered":"2023-10-09T09:27:39.550028+0000","last_clean":"2023-10-09T09:27:39.550028+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:27:39.550028+0000","last_undegraded":"2023-10-09T09:27:39.550028+0000","last_fullsized":"2023-10-09T09:27:39.550028+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":236,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673097+0000","last_change":"2023-10-09T09:26:19.493403+0000","last_active":"2023-10-09T09:27:40.673097+0000","last_peered":"2023-10-09T09:27:40.673097+0000","last_clean":"2023-10-09T09:27:40.673097+0000","last_became_active":"2023-10-09T09:26:19.493049+0000","last_became_peered":"2023-10-09T09:26:19.493049+0000","last_unstale":"2023-10-09T09:27:40.673097+0000","last_undegraded":"2023-10-09T09:27:40.673097+0000","last_fullsized":"2023-10-09T09:27:40.673097+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:30.756826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":200,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.544118+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:27:39.544118+0000","last_peered":"2023-10-09T09:27:39.544118+0000","last_clean":"2023-10-09T09:27:39.544118+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:27:39.544118+0000","last_undegraded":"2023-10-09T09:27:39.544118+0000","last_fullsized":"2023-10-09T09:27:39.544118+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":193,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.553130+0000","last_change":"2023-10-09T09:20:19.898526+0000","last_active":"2023-10-09T09:27:40.553130+0000","last_peered":"2023-10-09T09:27:40.553130+0000","last_clean":"2023-10-09T09:27:40.553130+0000","last_became_active":"2023-10-09T09:20:19.898197+0000","last_became_peered":"2023-10-09T09:20:19.898197+0000","last_unstale":"2023-10-09T09:27:40.553130+0000","last_undegraded":"2023-10-09T09:27:40.553130+0000","last_fullsized":"2023-10-09T09:27:40.553130+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:30.135339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":315,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548354+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:27:39.548354+0000","last_peered":"2023-10-09T09:27:39.548354+0000","last_clean":"2023-10-09T09:27:39.548354+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:27:39.548354+0000","last_undegraded":"2023-10-09T09:27:39.548354+0000","last_fullsized":"2023-10-09T09:27:39.548354+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":303,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547600+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:27:39.547600+0000","last_peered":"2023-10-09T09:27:39.547600+0000","last_clean":"2023-10-09T09:27:39.547600+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:27:39.547600+0000","last_undegraded":"2023-10-09T09:27:39.547600+0000","last_fullsized":"2023-10-09T09:27:39.547600+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":260,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.543717+0000","last_change":"2023-10-09T09:20:19.876172+0000","last_active":"2023-10-09T09:27:39.543717+0000","last_peered":"2023-10-09T09:27:39.543717+0000","last_clean":"2023-10-09T09:27:39.543717+0000","last_became_active":"2023-10-09T09:20:19.765721+0000","last_became_peered":"2023-10-09T09:20:19.765721+0000","last_unstale":"2023-10-09T09:27:39.543717+0000","last_undegraded":"2023-10-09T09:27:39.543717+0000","last_fullsized":"2023-10-09T09:27:39.543717+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":256,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673613+0000","last_change":"2023-10-09T09:26:19.643064+0000","last_active":"2023-10-09T09:27:40.673613+0000","last_peered":"2023-10-09T09:27:40.673613+0000","last_clean":"2023-10-09T09:27:40.673613+0000","last_became_active":"2023-10-09T09:26:19.642261+0000","last_became_peered":"2023-10-09T09:26:19.642261+0000","last_unstale":"2023-10-09T09:27:40.673613+0000","last_undegraded":"2023-10-09T09:27:40.673613+0000","last_fullsized":"2023-10-09T09:27:40.673613+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:14.490696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":433,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562041+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:27:40.562041+0000","last_peered":"2023-10-09T09:27:40.562041+0000","last_clean":"2023-10-09T09:27:40.562041+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:27:40.562041+0000","last_undegraded":"2023-10-09T09:27:40.562041+0000","last_fullsized":"2023-10-09T09:27:40.562041+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549880+0000","last_change":"2023-10-09T09:26:49.502769+0000","last_active":"2023-10-09T09:27:39.549880+0000","last_peered":"2023-10-09T09:27:39.549880+0000","last_clean":"2023-10-09T09:27:39.549880+0000","last_became_active":"2023-10-09T09:26:49.502432+0000","last_became_peered":"2023-10-09T09:26:49.502432+0000","last_unstale":"2023-10-09T09:27:39.549880+0000","last_undegraded":"2023-10-09T09:27:39.549880+0000","last_fullsized":"2023-10-09T09:27:39.549880+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":104,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.682120+0000","last_change":"2023-10-09T09:27:40.682120+0000","last_active":"2023-10-09T09:27:40.682120+0000","last_peered":"2023-10-09T09:27:40.682120+0000","last_clean":"2023-10-09T09:27:40.682120+0000","last_became_active":"2023-10-09T09:27:40.681426+0000","last_became_peered":"2023-10-09T09:27:40.681426+0000","last_unstale":"2023-10-09T09:27:40.682120+0000","last_undegraded":"2023-10-09T09:27:40.682120+0000","last_fullsized":"2023-10-09T09:27:40.682120+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":405,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.537193+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:27:39.537193+0000","last_peered":"2023-10-09T09:27:39.537193+0000","last_clean":"2023-10-09T09:27:39.537193+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:27:39.537193+0000","last_undegraded":"2023-10-09T09:27:39.537193+0000","last_fullsized":"2023-10-09T09:27:39.537193+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":349,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548985+0000","last_change":"2023-10-09T09:26:49.513269+0000","last_active":"2023-10-09T09:27:39.548985+0000","last_peered":"2023-10-09T09:27:39.548985+0000","last_clean":"2023-10-09T09:27:39.548985+0000","last_became_active":"2023-10-09T09:26:49.503720+0000","last_became_peered":"2023-10-09T09:26:49.503720+0000","last_unstale":"2023-10-09T09:27:39.548985+0000","last_undegraded":"2023-10-09T09:27:39.548985+0000","last_fullsized":"2023-10-09T09:27:39.548985+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":179,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673791+0000","last_change":"2023-10-09T09:26:19.494282+0000","last_active":"2023-10-09T09:27:40.673791+0000","last_peered":"2023-10-09T09:27:40.673791+0000","last_clean":"2023-10-09T09:27:40.673791+0000","last_became_active":"2023-10-09T09:26:19.493851+0000","last_became_peered":"2023-10-09T09:26:19.493851+0000","last_unstale":"2023-10-09T09:27:40.673791+0000","last_undegraded":"2023-10-09T09:27:40.673791+0000","last_fullsized":"2023-10-09T09:27:40.673791+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:44.212093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":211,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.544116+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:27:39.544116+0000","last_peered":"2023-10-09T09:27:39.544116+0000","last_clean":"2023-10-09T09:27:39.544116+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:27:39.544116+0000","last_undegraded":"2023-10-09T09:27:39.544116+0000","last_fullsized":"2023-10-09T09:27:39.544116+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.546405+0000","last_change":"2023-10-09T09:26:49.502858+0000","last_active":"2023-10-09T09:27:39.546405+0000","last_peered":"2023-10-09T09:27:39.546405+0000","last_clean":"2023-10-09T09:27:39.546405+0000","last_became_active":"2023-10-09T09:26:49.502402+0000","last_became_peered":"2023-10-09T09:26:49.502402+0000","last_unstale":"2023-10-09T09:27:39.546405+0000","last_undegraded":"2023-10-09T09:27:39.546405+0000","last_fullsized":"2023-10-09T09:27:39.546405+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":211,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.544271+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:27:39.544271+0000","last_peered":"2023-10-09T09:27:39.544271+0000","last_clean":"2023-10-09T09:27:39.544271+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:27:39.544271+0000","last_undegraded":"2023-10-09T09:27:39.544271+0000","last_fullsized":"2023-10-09T09:27:39.544271+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":303,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547629+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:27:39.547629+0000","last_peered":"2023-10-09T09:27:39.547629+0000","last_clean":"2023-10-09T09:27:39.547629+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:27:39.547629+0000","last_undegraded":"2023-10-09T09:27:39.547629+0000","last_fullsized":"2023-10-09T09:27:39.547629+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":51,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.569130+0000","last_change":"2023-10-09T09:27:40.569130+0000","last_active":"2023-10-09T09:27:40.569130+0000","last_peered":"2023-10-09T09:27:40.569130+0000","last_clean":"2023-10-09T09:27:40.569130+0000","last_became_active":"2023-10-09T09:27:40.568730+0000","last_became_peered":"2023-10-09T09:27:40.568730+0000","last_unstale":"2023-10-09T09:27:40.569130+0000","last_undegraded":"2023-10-09T09:27:40.569130+0000","last_fullsized":"2023-10-09T09:27:40.569130+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":317,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549501+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:27:39.549501+0000","last_peered":"2023-10-09T09:27:39.549501+0000","last_clean":"2023-10-09T09:27:39.549501+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:27:39.549501+0000","last_undegraded":"2023-10-09T09:27:39.549501+0000","last_fullsized":"2023-10-09T09:27:39.549501+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":201,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549939+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:27:39.549939+0000","last_peered":"2023-10-09T09:27:39.549939+0000","last_clean":"2023-10-09T09:27:39.549939+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:27:39.549939+0000","last_undegraded":"2023-10-09T09:27:39.549939+0000","last_fullsized":"2023-10-09T09:27:39.549939+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":434,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562012+0000","last_change":"2023-10-09T09:18:51.869617+0000","last_active":"2023-10-09T09:27:40.562012+0000","last_peered":"2023-10-09T09:27:40.562012+0000","last_clean":"2023-10-09T09:27:40.562012+0000","last_became_active":"2023-10-09T09:18:51.869212+0000","last_became_peered":"2023-10-09T09:18:51.869212+0000","last_unstale":"2023-10-09T09:27:40.562012+0000","last_undegraded":"2023-10-09T09:27:40.562012+0000","last_fullsized":"2023-10-09T09:27:40.562012+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:36.753958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":249,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673407+0000","last_change":"2023-10-09T09:26:19.488721+0000","last_active":"2023-10-09T09:27:40.673407+0000","last_peered":"2023-10-09T09:27:40.673407+0000","last_clean":"2023-10-09T09:27:40.673407+0000","last_became_active":"2023-10-09T09:26:19.488304+0000","last_became_peered":"2023-10-09T09:26:19.488304+0000","last_unstale":"2023-10-09T09:27:40.673407+0000","last_undegraded":"2023-10-09T09:27:40.673407+0000","last_fullsized":"2023-10-09T09:27:40.673407+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:02.529432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":276,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549089+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:27:39.549089+0000","last_peered":"2023-10-09T09:27:39.549089+0000","last_clean":"2023-10-09T09:27:39.549089+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:27:39.549089+0000","last_undegraded":"2023-10-09T09:27:39.549089+0000","last_fullsized":"2023-10-09T09:27:39.549089+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":281,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549346+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:27:39.549346+0000","last_peered":"2023-10-09T09:27:39.549346+0000","last_clean":"2023-10-09T09:27:39.549346+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:27:39.549346+0000","last_undegraded":"2023-10-09T09:27:39.549346+0000","last_fullsized":"2023-10-09T09:27:39.549346+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":321,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548587+0000","last_change":"2023-10-09T09:20:19.935938+0000","last_active":"2023-10-09T09:27:39.548587+0000","last_peered":"2023-10-09T09:27:39.548587+0000","last_clean":"2023-10-09T09:27:39.548587+0000","last_became_active":"2023-10-09T09:20:19.935423+0000","last_became_peered":"2023-10-09T09:20:19.935423+0000","last_unstale":"2023-10-09T09:27:39.548587+0000","last_undegraded":"2023-10-09T09:27:39.548587+0000","last_fullsized":"2023-10-09T09:27:39.548587+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":348,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547275+0000","last_change":"2023-10-09T09:20:19.755858+0000","last_active":"2023-10-09T09:27:39.547275+0000","last_peered":"2023-10-09T09:27:39.547275+0000","last_clean":"2023-10-09T09:27:39.547275+0000","last_became_active":"2023-10-09T09:20:19.755442+0000","last_became_peered":"2023-10-09T09:20:19.755442+0000","last_unstale":"2023-10-09T09:27:39.547275+0000","last_undegraded":"2023-10-09T09:27:39.547275+0000","last_fullsized":"2023-10-09T09:27:39.547275+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":294,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548569+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:27:39.548569+0000","last_peered":"2023-10-09T09:27:39.548569+0000","last_clean":"2023-10-09T09:27:39.548569+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:27:39.548569+0000","last_undegraded":"2023-10-09T09:27:39.548569+0000","last_fullsized":"2023-10-09T09:27:39.548569+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":226,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.672979+0000","last_change":"2023-10-09T09:26:19.644787+0000","last_active":"2023-10-09T09:27:40.672979+0000","last_peered":"2023-10-09T09:27:40.672979+0000","last_clean":"2023-10-09T09:27:40.672979+0000","last_became_active":"2023-10-09T09:26:19.643767+0000","last_became_peered":"2023-10-09T09:26:19.643767+0000","last_unstale":"2023-10-09T09:27:40.672979+0000","last_undegraded":"2023-10-09T09:27:40.672979+0000","last_fullsized":"2023-10-09T09:27:40.672979+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:26.488695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.554641+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:27:40.554641+0000","last_peered":"2023-10-09T09:27:40.554641+0000","last_clean":"2023-10-09T09:27:40.554641+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:27:40.554641+0000","last_undegraded":"2023-10-09T09:27:40.554641+0000","last_fullsized":"2023-10-09T09:27:40.554641+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"391'120","reported_seq":527,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549981+0000","last_change":"2023-10-09T09:24:22.950512+0000","last_active":"2023-10-09T09:27:39.549981+0000","last_peered":"2023-10-09T09:27:39.549981+0000","last_clean":"2023-10-09T09:27:39.549981+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:27:39.549981+0000","last_undegraded":"2023-10-09T09:27:39.549981+0000","last_fullsized":"2023-10-09T09:27:39.549981+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040695899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":2231,"num_write":134,"num_write_kb":1460,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550142+0000","last_change":"2023-10-09T09:20:19.757594+0000","last_active":"2023-10-09T09:27:39.550142+0000","last_peered":"2023-10-09T09:27:39.550142+0000","last_clean":"2023-10-09T09:27:39.550142+0000","last_became_active":"2023-10-09T09:20:19.757216+0000","last_became_peered":"2023-10-09T09:20:19.757216+0000","last_unstale":"2023-10-09T09:27:39.550142+0000","last_undegraded":"2023-10-09T09:27:39.550142+0000","last_fullsized":"2023-10-09T09:27:39.550142+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549419+0000","last_change":"2023-10-09T09:26:49.501845+0000","last_active":"2023-10-09T09:27:39.549419+0000","last_peered":"2023-10-09T09:27:39.549419+0000","last_clean":"2023-10-09T09:27:39.549419+0000","last_became_active":"2023-10-09T09:26:49.501350+0000","last_became_peered":"2023-10-09T09:26:49.501350+0000","last_unstale":"2023-10-09T09:27:39.549419+0000","last_undegraded":"2023-10-09T09:27:39.549419+0000","last_fullsized":"2023-10-09T09:27:39.549419+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:18.155759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"391'94","reported_seq":637,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.546517+0000","last_change":"2023-10-09T09:24:22.939657+0000","last_active":"2023-10-09T09:27:39.546517+0000","last_peered":"2023-10-09T09:27:39.546517+0000","last_clean":"2023-10-09T09:27:39.546517+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:27:39.546517+0000","last_undegraded":"2023-10-09T09:27:39.546517+0000","last_fullsized":"2023-10-09T09:27:39.546517+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":94,"log_dups_size":0,"ondisk_log_size":94,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":536,"num_read_kb":1670,"num_write":97,"num_write_kb":5264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":276,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548141+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:27:39.548141+0000","last_peered":"2023-10-09T09:27:39.548141+0000","last_clean":"2023-10-09T09:27:39.548141+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:27:39.548141+0000","last_undegraded":"2023-10-09T09:27:39.548141+0000","last_fullsized":"2023-10-09T09:27:39.548141+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":193,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549860+0000","last_change":"2023-10-09T09:20:19.932428+0000","last_active":"2023-10-09T09:27:39.549860+0000","last_peered":"2023-10-09T09:27:39.549860+0000","last_clean":"2023-10-09T09:27:39.549860+0000","last_became_active":"2023-10-09T09:20:19.931684+0000","last_became_peered":"2023-10-09T09:20:19.931684+0000","last_unstale":"2023-10-09T09:27:39.549860+0000","last_undegraded":"2023-10-09T09:27:39.549860+0000","last_fullsized":"2023-10-09T09:27:39.549860+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:40.774870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"391'108","reported_seq":551,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.554712+0000","last_change":"2023-10-09T09:24:22.935939+0000","last_active":"2023-10-09T09:27:40.554712+0000","last_peered":"2023-10-09T09:27:40.554712+0000","last_clean":"2023-10-09T09:27:40.554712+0000","last_became_active":"2023-10-09T09:20:19.757450+0000","last_became_peered":"2023-10-09T09:20:19.757450+0000","last_unstale":"2023-10-09T09:27:40.554712+0000","last_undegraded":"2023-10-09T09:27:40.554712+0000","last_fullsized":"2023-10-09T09:27:40.554712+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:57.053230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068192000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":697,"num_read_kb":1551,"num_write":95,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":194,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.552422+0000","last_change":"2023-10-09T09:20:19.902503+0000","last_active":"2023-10-09T09:27:40.552422+0000","last_peered":"2023-10-09T09:27:40.552422+0000","last_clean":"2023-10-09T09:27:40.552422+0000","last_became_active":"2023-10-09T09:20:19.902131+0000","last_became_peered":"2023-10-09T09:20:19.902131+0000","last_unstale":"2023-10-09T09:27:40.552422+0000","last_undegraded":"2023-10-09T09:27:40.552422+0000","last_fullsized":"2023-10-09T09:27:40.552422+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:43.897708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.553463+0000","last_change":"2023-10-09T09:21:51.904699+0000","last_active":"2023-10-09T09:27:40.553463+0000","last_peered":"2023-10-09T09:27:40.553463+0000","last_clean":"2023-10-09T09:27:40.553463+0000","last_became_active":"2023-10-09T09:21:51.904381+0000","last_became_peered":"2023-10-09T09:21:51.904381+0000","last_unstale":"2023-10-09T09:27:40.553463+0000","last_undegraded":"2023-10-09T09:27:40.553463+0000","last_fullsized":"2023-10-09T09:27:40.553463+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:46.915626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"391'140","reported_seq":523,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549130+0000","last_change":"2023-10-09T09:24:22.940815+0000","last_active":"2023-10-09T09:27:39.549130+0000","last_peered":"2023-10-09T09:27:39.549130+0000","last_clean":"2023-10-09T09:27:39.549130+0000","last_became_active":"2023-10-09T09:20:19.759254+0000","last_became_peered":"2023-10-09T09:20:19.759254+0000","last_unstale":"2023-10-09T09:27:39.549130+0000","last_undegraded":"2023-10-09T09:27:39.549130+0000","last_fullsized":"2023-10-09T09:27:39.549130+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":267,"num_read_kb":1259,"num_write":146,"num_write_kb":2143,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":652,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549147+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:27:39.549147+0000","last_peered":"2023-10-09T09:27:39.549147+0000","last_clean":"2023-10-09T09:27:39.549147+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:27:39.549147+0000","last_undegraded":"2023-10-09T09:27:39.549147+0000","last_fullsized":"2023-10-09T09:27:39.549147+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":75,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673394+0000","last_change":"2023-10-09T09:26:19.491386+0000","last_active":"2023-10-09T09:27:40.673394+0000","last_peered":"2023-10-09T09:27:40.673394+0000","last_clean":"2023-10-09T09:27:40.673394+0000","last_became_active":"2023-10-09T09:26:19.491104+0000","last_became_peered":"2023-10-09T09:26:19.491104+0000","last_unstale":"2023-10-09T09:27:40.673394+0000","last_undegraded":"2023-10-09T09:27:40.673394+0000","last_fullsized":"2023-10-09T09:27:40.673394+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:14.546226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":272,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550536+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:27:39.550536+0000","last_peered":"2023-10-09T09:27:39.550536+0000","last_clean":"2023-10-09T09:27:39.550536+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:27:39.550536+0000","last_undegraded":"2023-10-09T09:27:39.550536+0000","last_fullsized":"2023-10-09T09:27:39.550536+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"391'86","reported_seq":481,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.565407+0000","last_change":"2023-10-09T09:27:40.565407+0000","last_active":"2023-10-09T09:27:40.565407+0000","last_peered":"2023-10-09T09:27:40.565407+0000","last_clean":"2023-10-09T09:27:40.565407+0000","last_became_active":"2023-10-09T09:27:40.564527+0000","last_became_peered":"2023-10-09T09:27:40.564527+0000","last_unstale":"2023-10-09T09:27:40.565407+0000","last_undegraded":"2023-10-09T09:27:40.565407+0000","last_fullsized":"2023-10-09T09:27:40.565407+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":615,"num_read_kb":534,"num_write":66,"num_write_kb":1068,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":453,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.535870+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:27:39.535870+0000","last_peered":"2023-10-09T09:27:39.535870+0000","last_clean":"2023-10-09T09:27:39.535870+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:27:39.535870+0000","last_undegraded":"2023-10-09T09:27:39.535870+0000","last_fullsized":"2023-10-09T09:27:39.535870+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":476,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536540+0000","last_change":"2023-10-09T09:20:19.890699+0000","last_active":"2023-10-09T09:27:39.536540+0000","last_peered":"2023-10-09T09:27:39.536540+0000","last_clean":"2023-10-09T09:27:39.536540+0000","last_became_active":"2023-10-09T09:20:19.889949+0000","last_became_peered":"2023-10-09T09:20:19.889949+0000","last_unstale":"2023-10-09T09:27:39.536540+0000","last_undegraded":"2023-10-09T09:27:39.536540+0000","last_fullsized":"2023-10-09T09:27:39.536540+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:59.920953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"391'161","reported_seq":1030,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.543650+0000","last_change":"2023-10-09T09:24:22.945624+0000","last_active":"2023-10-09T09:27:39.543650+0000","last_peered":"2023-10-09T09:27:39.543650+0000","last_clean":"2023-10-09T09:27:39.543650+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:27:39.543650+0000","last_undegraded":"2023-10-09T09:27:39.543650+0000","last_fullsized":"2023-10-09T09:27:39.543650+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1582,"num_read_kb":1930,"num_write":111,"num_write_kb":341,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":193,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549724+0000","last_change":"2023-10-09T09:20:19.912177+0000","last_active":"2023-10-09T09:27:39.549724+0000","last_peered":"2023-10-09T09:27:39.549724+0000","last_clean":"2023-10-09T09:27:39.549724+0000","last_became_active":"2023-10-09T09:20:19.908966+0000","last_became_peered":"2023-10-09T09:20:19.908966+0000","last_unstale":"2023-10-09T09:27:39.549724+0000","last_undegraded":"2023-10-09T09:27:39.549724+0000","last_fullsized":"2023-10-09T09:27:39.549724+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:54.778943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":429,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536841+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:27:39.536841+0000","last_peered":"2023-10-09T09:27:39.536841+0000","last_clean":"2023-10-09T09:27:39.536841+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:27:39.536841+0000","last_undegraded":"2023-10-09T09:27:39.536841+0000","last_fullsized":"2023-10-09T09:27:39.536841+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":37,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673720+0000","last_change":"2023-10-09T09:26:19.494388+0000","last_active":"2023-10-09T09:27:40.673720+0000","last_peered":"2023-10-09T09:27:40.673720+0000","last_clean":"2023-10-09T09:27:40.673720+0000","last_became_active":"2023-10-09T09:26:19.493921+0000","last_became_peered":"2023-10-09T09:26:19.493921+0000","last_unstale":"2023-10-09T09:27:40.673720+0000","last_undegraded":"2023-10-09T09:27:40.673720+0000","last_fullsized":"2023-10-09T09:27:40.673720+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:31.372332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"391'176","reported_seq":833,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.555204+0000","last_change":"2023-10-09T09:24:22.951420+0000","last_active":"2023-10-09T09:27:40.555204+0000","last_peered":"2023-10-09T09:27:40.555204+0000","last_clean":"2023-10-09T09:27:40.555204+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:27:40.555204+0000","last_undegraded":"2023-10-09T09:27:40.555204+0000","last_fullsized":"2023-10-09T09:27:40.555204+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":176,"log_dups_size":0,"ondisk_log_size":176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084346699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1202,"num_read_kb":2193,"num_write":134,"num_write_kb":1170,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":182,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.672886+0000","last_change":"2023-10-09T09:26:19.493412+0000","last_active":"2023-10-09T09:27:40.672886+0000","last_peered":"2023-10-09T09:27:40.672886+0000","last_clean":"2023-10-09T09:27:40.672886+0000","last_became_active":"2023-10-09T09:26:19.492949+0000","last_became_peered":"2023-10-09T09:26:19.492949+0000","last_unstale":"2023-10-09T09:27:40.672886+0000","last_undegraded":"2023-10-09T09:27:40.672886+0000","last_fullsized":"2023-10-09T09:27:40.672886+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:02.951627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"391'91","reported_seq":718,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549943+0000","last_change":"2023-10-09T09:27:30.535381+0000","last_active":"2023-10-09T09:27:39.549943+0000","last_peered":"2023-10-09T09:27:39.549943+0000","last_clean":"2023-10-09T09:27:39.549943+0000","last_became_active":"2023-10-09T09:27:30.521550+0000","last_became_peered":"2023-10-09T09:27:30.521550+0000","last_unstale":"2023-10-09T09:27:39.549943+0000","last_undegraded":"2023-10-09T09:27:39.549943+0000","last_fullsized":"2023-10-09T09:27:39.549943+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":572,"num_read_kb":856,"num_write":63,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":471,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536783+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:27:39.536783+0000","last_peered":"2023-10-09T09:27:39.536783+0000","last_clean":"2023-10-09T09:27:39.536783+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:27:39.536783+0000","last_undegraded":"2023-10-09T09:27:39.536783+0000","last_fullsized":"2023-10-09T09:27:39.536783+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":294,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547584+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:27:39.547584+0000","last_peered":"2023-10-09T09:27:39.547584+0000","last_clean":"2023-10-09T09:27:39.547584+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:27:39.547584+0000","last_undegraded":"2023-10-09T09:27:39.547584+0000","last_fullsized":"2023-10-09T09:27:39.547584+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":439,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562200+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:27:40.562200+0000","last_peered":"2023-10-09T09:27:40.562200+0000","last_clean":"2023-10-09T09:27:40.562200+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:27:40.562200+0000","last_undegraded":"2023-10-09T09:27:40.562200+0000","last_fullsized":"2023-10-09T09:27:40.562200+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":120,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.553681+0000","last_change":"2023-10-09T09:20:19.916151+0000","last_active":"2023-10-09T09:27:40.553681+0000","last_peered":"2023-10-09T09:27:40.553681+0000","last_clean":"2023-10-09T09:27:40.553681+0000","last_became_active":"2023-10-09T09:20:19.915607+0000","last_became_peered":"2023-10-09T09:20:19.915607+0000","last_unstale":"2023-10-09T09:27:40.553681+0000","last_undegraded":"2023-10-09T09:27:40.553681+0000","last_fullsized":"2023-10-09T09:27:40.553681+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:57:51.697949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":218,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549830+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:27:39.549830+0000","last_peered":"2023-10-09T09:27:39.549830+0000","last_clean":"2023-10-09T09:27:39.549830+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:27:39.549830+0000","last_undegraded":"2023-10-09T09:27:39.549830+0000","last_fullsized":"2023-10-09T09:27:39.549830+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":294,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547932+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:27:39.547932+0000","last_peered":"2023-10-09T09:27:39.547932+0000","last_clean":"2023-10-09T09:27:39.547932+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:27:39.547932+0000","last_undegraded":"2023-10-09T09:27:39.547932+0000","last_fullsized":"2023-10-09T09:27:39.547932+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":205,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550063+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:27:39.550063+0000","last_peered":"2023-10-09T09:27:39.550063+0000","last_clean":"2023-10-09T09:27:39.550063+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:27:39.550063+0000","last_undegraded":"2023-10-09T09:27:39.550063+0000","last_fullsized":"2023-10-09T09:27:39.550063+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":276,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547511+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:27:39.547511+0000","last_peered":"2023-10-09T09:27:39.547511+0000","last_clean":"2023-10-09T09:27:39.547511+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:27:39.547511+0000","last_undegraded":"2023-10-09T09:27:39.547511+0000","last_fullsized":"2023-10-09T09:27:39.547511+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":249,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673691+0000","last_change":"2023-10-09T09:26:19.491307+0000","last_active":"2023-10-09T09:27:40.673691+0000","last_peered":"2023-10-09T09:27:40.673691+0000","last_clean":"2023-10-09T09:27:40.673691+0000","last_became_active":"2023-10-09T09:26:19.490921+0000","last_became_peered":"2023-10-09T09:26:19.490921+0000","last_unstale":"2023-10-09T09:27:40.673691+0000","last_undegraded":"2023-10-09T09:27:40.673691+0000","last_fullsized":"2023-10-09T09:27:40.673691+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:20.144589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":260,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673278+0000","last_change":"2023-10-09T09:26:19.488134+0000","last_active":"2023-10-09T09:27:40.673278+0000","last_peered":"2023-10-09T09:27:40.673278+0000","last_clean":"2023-10-09T09:27:40.673278+0000","last_became_active":"2023-10-09T09:26:19.487724+0000","last_became_peered":"2023-10-09T09:26:19.487724+0000","last_unstale":"2023-10-09T09:27:40.673278+0000","last_undegraded":"2023-10-09T09:27:40.673278+0000","last_fullsized":"2023-10-09T09:27:40.673278+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:25.687144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":559,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536731+0000","last_change":"2023-10-09T09:20:19.788084+0000","last_active":"2023-10-09T09:27:39.536731+0000","last_peered":"2023-10-09T09:27:39.536731+0000","last_clean":"2023-10-09T09:27:39.536731+0000","last_became_active":"2023-10-09T09:20:19.762112+0000","last_became_peered":"2023-10-09T09:20:19.762112+0000","last_unstale":"2023-10-09T09:27:39.536731+0000","last_undegraded":"2023-10-09T09:27:39.536731+0000","last_fullsized":"2023-10-09T09:27:39.536731+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:47.406781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":303,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.546810+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:27:39.546810+0000","last_peered":"2023-10-09T09:27:39.546810+0000","last_clean":"2023-10-09T09:27:39.546810+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:27:39.546810+0000","last_undegraded":"2023-10-09T09:27:39.546810+0000","last_fullsized":"2023-10-09T09:27:39.546810+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":20760,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12224,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430903,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907472,"kb_used_data":4168,"kb_used_omap":0,"kb_used_meta":903296,"kb_avail":92837680,"statfs":{"total":95995035648,"available":95065784320,"internally_reserved":0,"allocated":4268032,"data_stored":3617173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924975104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97399999999999998,"15min":1.766},"min":{"1min":0.68799999999999994,"5min":0.39800000000000002,"15min":0.23499999999999999},"max":{"1min":1.319,"5min":1.464,"15min":51.859999999999999},"last":1.1890000000000001},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.96399999999999997,"15min":1.784},"min":{"1min":0.63100000000000001,"5min":0.503,"15min":0.28899999999999998},"max":{"1min":1.2350000000000001,"5min":1.5369999999999999,"15min":51.957999999999998},"last":1.173}]},{"osd":1,"last update":"Mon Oct 9 09:28:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":2.7810000000000001,"15min":2.153},"min":{"1min":0.61699999999999999,"5min":0.55200000000000005,"15min":0.27400000000000002},"max":{"1min":1.2769999999999999,"5min":142.703,"15min":142.703},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":2.762,"15min":2.2000000000000002},"min":{"1min":0.66200000000000003,"5min":0.628,"15min":0.316},"max":{"1min":1.3200000000000001,"5min":142.67500000000001,"15min":142.67500000000001},"last":1.016}]},{"osd":2,"last update":"Mon Oct 9 09:28:52 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.94199999999999995,"15min":1.486},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.32800000000000001},"max":{"1min":1.147,"5min":1.2529999999999999,"15min":51.914000000000001},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.97599999999999998,"15min":1.3959999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.33200000000000002},"max":{"1min":1.3380000000000001,"5min":1.383,"15min":51.902999999999999},"last":0.83599999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:28:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.93300000000000005,"15min":1.379},"min":{"1min":0.68999999999999995,"5min":0.65000000000000002,"15min":0.374},"max":{"1min":1.159,"5min":1.506,"15min":18.178999999999998},"last":0.871},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0629999999999999,"15min":1.3129999999999999},"min":{"1min":0.61899999999999999,"5min":0.54900000000000004,"15min":0.31},"max":{"1min":1.349,"5min":1.5149999999999999,"15min":17.931999999999999},"last":1.0740000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.96299999999999997,"15min":1.325},"min":{"1min":0.70199999999999996,"5min":0.52400000000000002,"15min":0.28699999999999998},"max":{"1min":1.335,"5min":1.3500000000000001,"15min":11.371},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.0449999999999999,"15min":1.3129999999999999},"min":{"1min":0.84799999999999998,"5min":0.52600000000000002,"15min":0.30599999999999999},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":8.9879999999999995},"last":0.97399999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":0.999,"15min":1.286},"min":{"1min":0.80200000000000005,"5min":0.48099999999999998,"15min":0.32200000000000001},"max":{"1min":1.242,"5min":1.395,"15min":15.616},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.119,"5min":1.081,"15min":1.5129999999999999},"min":{"1min":0.88200000000000001,"5min":0.57399999999999995,"15min":0.36599999999999999},"max":{"1min":1.3,"5min":1.494,"15min":18.501000000000001},"last":0.94599999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.1020000000000001,"15min":1.5209999999999999},"min":{"1min":0.82199999999999995,"5min":0.65400000000000003,"15min":0.34899999999999998},"max":{"1min":1.391,"5min":1.5669999999999999,"15min":21.145},"last":1.093},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.2130000000000001,"15min":1.5189999999999999},"min":{"1min":0.92000000000000004,"5min":0.45800000000000002,"15min":0.34999999999999998},"max":{"1min":1.573,"5min":1.573,"15min":12.269},"last":1.2450000000000001}]}]},{"osd":6,"up_from":42,"seq":180388627136,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515464,"kb_used_data":8256,"kb_used_omap":0,"kb_used_meta":1507200,"kb_avail":92229688,"statfs":{"total":95995035648,"available":94443200512,"internally_reserved":0,"allocated":8454144,"data_stored":7811300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1543372800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.98199999999999998,"15min":2.7629999999999999},"min":{"1min":0.51700000000000002,"5min":0.48899999999999999,"15min":0.26100000000000001},"max":{"1min":1.3759999999999999,"5min":1.4690000000000001,"15min":120.812},"last":1.175},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0580000000000001,"15min":2.6499999999999999},"min":{"1min":0.69699999999999995,"5min":0.68700000000000006,"15min":0.25600000000000001},"max":{"1min":1.4339999999999999,"5min":1.853,"15min":120.857},"last":1.135}]},{"osd":1,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.6180000000000001,"15min":2.4159999999999999},"min":{"1min":0.60799999999999998,"5min":0.48599999999999999,"15min":0.28100000000000003},"max":{"1min":1.9470000000000001,"5min":59.509,"15min":142.31100000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.651,"15min":2.367},"min":{"1min":0.63300000000000001,"5min":0.53100000000000003,"15min":0.25700000000000001},"max":{"1min":1.9139999999999999,"5min":59.482999999999997,"15min":132.48099999999999},"last":1.089}]},{"osd":2,"last update":"Mon Oct 9 09:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.95299999999999996,"15min":1.6890000000000001},"min":{"1min":0.72499999999999998,"5min":0.49399999999999999,"15min":0.28999999999999998},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":82.170000000000002},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.026,"15min":1.611},"min":{"1min":0.755,"5min":0.63,"15min":0.28699999999999998},"max":{"1min":1.581,"5min":1.581,"15min":82.352000000000004},"last":1.0289999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.94099999999999995,"15min":1.548},"min":{"1min":0.751,"5min":0.59099999999999997,"15min":0.33700000000000002},"max":{"1min":1.3460000000000001,"5min":1.8140000000000001,"15min":14.525},"last":1.006},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.96299999999999997,"15min":1.216},"min":{"1min":0.70399999999999996,"5min":0.629,"15min":0.32100000000000001},"max":{"1min":1.2849999999999999,"5min":1.5,"15min":9.0839999999999996},"last":0.68300000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:28:19 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.006,"15min":1.534},"min":{"1min":0.68500000000000005,"5min":0.45700000000000002,"15min":0.27600000000000002},"max":{"1min":1.444,"5min":2.1389999999999998,"15min":29.585999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.98399999999999999,"15min":1.4410000000000001},"min":{"1min":0.69899999999999995,"5min":0.38900000000000001,"15min":0.316},"max":{"1min":1.383,"5min":1.7769999999999999,"15min":20.318000000000001},"last":0.92400000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:28:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.052,"15min":1.4610000000000001},"min":{"1min":0.75700000000000001,"5min":0.53900000000000003,"15min":0.312},"max":{"1min":2.1549999999999998,"5min":2.1549999999999998,"15min":18.103000000000002},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.131,"5min":1.0580000000000001,"15min":1.401},"min":{"1min":0.77900000000000003,"5min":0.55600000000000005,"15min":0.33300000000000002},"max":{"1min":2.629,"5min":2.629,"15min":11.567},"last":1.149}]},{"osd":7,"last update":"Mon Oct 9 09:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.1559999999999999,"15min":1.5840000000000001},"min":{"1min":0.91500000000000004,"5min":0.55100000000000005,"15min":0.33700000000000002},"max":{"1min":1.994,"5min":1.994,"15min":29.530000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.2,"5min":1.111,"15min":1.4299999999999999},"min":{"1min":0.98499999999999999,"5min":0.497,"15min":0.32900000000000001},"max":{"1min":1.821,"5min":1.8779999999999999,"15min":11.477},"last":1.0549999999999999}]}]},{"osd":1,"up_from":83,"seq":356482286084,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":656324,"kb_used_data":8572,"kb_used_omap":9,"kb_used_meta":647734,"kb_avail":93088828,"statfs":{"total":95995035648,"available":95322959872,"internally_reserved":0,"allocated":8777728,"data_stored":7324979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":663280617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.70199999999999996,"15min":1.9790000000000001},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.215},"max":{"1min":0.83099999999999996,"5min":1.569,"15min":133.76499999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.76100000000000001,"15min":1.7330000000000001},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.22900000000000001},"max":{"1min":1.0389999999999999,"5min":1.381,"15min":127.395},"last":0.90000000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.76800000000000002,"15min":1.78},"min":{"1min":0.42399999999999999,"5min":0.377,"15min":0.26500000000000001},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":236.29599999999999},"last":1.1659999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.77800000000000002,"15min":1.7270000000000001},"min":{"1min":0.52800000000000002,"5min":0.39400000000000002,"15min":0.27500000000000002},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":236.21000000000001},"last":1.4510000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.89100000000000001,"15min":1.744},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.308},"max":{"1min":1.165,"5min":1.486,"15min":152.637},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.85699999999999998,"15min":2.306},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.25800000000000001},"max":{"1min":1.0509999999999999,"5min":1.421,"15min":152.374},"last":1.0960000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.99399999999999999,"15min":1.462},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.38},"max":{"1min":1.2509999999999999,"5min":1.6830000000000001,"15min":14.56},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.008,"15min":1.8779999999999999},"min":{"1min":0.63100000000000001,"5min":0.55800000000000005,"15min":0.33200000000000002},"max":{"1min":1.349,"5min":1.792,"15min":133.702},"last":1.206}]},{"osd":5,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0640000000000001,"15min":1.298},"min":{"1min":0.67400000000000004,"5min":0.61299999999999999,"15min":0.34699999999999998},"max":{"1min":1.347,"5min":1.768,"15min":9.4290000000000003},"last":1.6910000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.119,"15min":1.3129999999999999},"min":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.35199999999999998},"max":{"1min":1.4199999999999999,"5min":1.8819999999999999,"15min":14.464},"last":1.294}]},{"osd":6,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.149,"15min":1.984},"min":{"1min":0.70199999999999996,"5min":0.60399999999999998,"15min":0.38300000000000001},"max":{"1min":1.5669999999999999,"5min":1.9179999999999999,"15min":133.739},"last":1.579},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.155,"15min":1.4059999999999999},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.377},"max":{"1min":1.609,"5min":1.954,"15min":9.4619999999999997},"last":1.768}]},{"osd":7,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.1579999999999999,"15min":1.518},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.41499999999999998},"max":{"1min":1.3120000000000001,"5min":1.9870000000000001,"15min":19.861999999999998},"last":1.3819999999999999},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.1419999999999999,"15min":1.9610000000000001},"min":{"1min":0.71899999999999997,"5min":0.66200000000000003,"15min":0.40899999999999997},"max":{"1min":1.5189999999999999,"5min":1.784,"15min":133.66499999999999},"last":1.486}]}]},{"osd":0,"up_from":78,"seq":335007449610,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":993368,"kb_used_data":5072,"kb_used_omap":28,"kb_used_meta":988259,"kb_avail":92751784,"statfs":{"total":95995035648,"available":94977826816,"internally_reserved":0,"allocated":5193728,"data_stored":3722137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1011978064},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:28:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.73799999999999999,"15min":1.347},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.19700000000000001},"max":{"1min":1.006,"5min":3.0139999999999998,"15min":83.671999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.76500000000000001,"5min":0.79000000000000004,"15min":1.3560000000000001},"min":{"1min":0.34799999999999998,"5min":0.29599999999999999,"15min":0.20300000000000001},"max":{"1min":1.1850000000000001,"5min":3.052,"15min":83.953000000000003},"last":0.85399999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.85099999999999998,"15min":1.472},"min":{"1min":0.56899999999999995,"5min":0.36899999999999999,"15min":0.26800000000000002},"max":{"1min":1.3959999999999999,"5min":2.9129999999999998,"15min":74.200999999999993},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.96099999999999997,"15min":1.524},"min":{"1min":0.68600000000000005,"5min":0.46000000000000002,"15min":0.26800000000000002},"max":{"1min":1.427,"5min":1.7849999999999999,"15min":73.983999999999995},"last":1.2749999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.89500000000000002,"15min":1.29},"min":{"1min":0.45900000000000002,"5min":0.33100000000000002,"15min":0.253},"max":{"1min":1.198,"5min":2.9460000000000002,"15min":9.8239999999999998},"last":0.623},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.91800000000000004,"15min":1.171},"min":{"1min":0.438,"5min":0.40699999999999997,"15min":0.29399999999999998},"max":{"1min":1.2569999999999999,"5min":1.716,"15min":9.9269999999999996},"last":1.099}]},{"osd":4,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.095,"15min":1.522},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.34000000000000002},"max":{"1min":1.597,"5min":1.597,"15min":14.163},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.96099999999999997,"15min":1.5640000000000001},"min":{"1min":0.66400000000000003,"5min":0.54300000000000004,"15min":0.314},"max":{"1min":1.2210000000000001,"5min":1.4139999999999999,"15min":11.346},"last":0.90800000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.121,"15min":1.4330000000000001},"min":{"1min":0.61599999999999999,"5min":0.57299999999999995,"15min":0.316},"max":{"1min":1.5289999999999999,"5min":2.988,"15min":9.9770000000000003},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.1850000000000001,"15min":1.3460000000000001},"min":{"1min":0.77200000000000002,"5min":0.66500000000000004,"15min":0.35499999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":9.5909999999999993},"last":1.1879999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.204,"15min":1.605},"min":{"1min":0.83699999999999997,"5min":0.53300000000000003,"15min":0.34799999999999998},"max":{"1min":1.6379999999999999,"5min":2.968,"15min":11.314},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.254,"15min":1.6879999999999999},"min":{"1min":0.85799999999999998,"5min":0.70899999999999996,"15min":0.39400000000000002},"max":{"1min":1.8400000000000001,"5min":1.8400000000000001,"15min":16.911000000000001},"last":1.3560000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.28,"15min":1.4990000000000001},"min":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.41699999999999998},"max":{"1min":1.79,"5min":1.79,"15min":9.6780000000000008},"last":1.431},{"interface":"front","average":{"1min":1.113,"5min":1.0880000000000001,"15min":1.6719999999999999},"min":{"1min":0.82199999999999995,"5min":0.623,"15min":0.35899999999999999},"max":{"1min":1.26,"5min":1.4330000000000001,"15min":10.734},"last":1.0409999999999999}]}]},{"osd":2,"up_from":88,"seq":377957122558,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":971160,"kb_used_data":4496,"kb_used_omap":81,"kb_used_meta":966574,"kb_avail":92773992,"statfs":{"total":95995035648,"available":95000567808,"internally_reserved":0,"allocated":4603904,"data_stored":3130325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":989771994},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.71299999999999997,"15min":1.419},"min":{"1min":0.56100000000000005,"5min":0.35599999999999998,"15min":0.23899999999999999},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":83.421999999999997},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.79800000000000004,"15min":1.3400000000000001},"min":{"1min":0.38500000000000001,"5min":0.36699999999999999,"15min":0.23499999999999999},"max":{"1min":1.085,"5min":1.3999999999999999,"15min":83.373999999999995},"last":0.72099999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.82399999999999995,"15min":1.7410000000000001},"min":{"1min":0.49399999999999999,"5min":0.38400000000000001,"15min":0.245},"max":{"1min":1.0760000000000001,"5min":2.1840000000000002,"15min":122.367},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.878,"15min":1.7050000000000001},"min":{"1min":0.46000000000000002,"5min":0.35699999999999998,"15min":0.27800000000000002},"max":{"1min":1.262,"5min":3.5680000000000001,"15min":122.777},"last":1.004}]},{"osd":3,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.87,"15min":2.1549999999999998},"min":{"1min":0.51300000000000001,"5min":0.42699999999999999,"15min":0.26500000000000001},"max":{"1min":1.1439999999999999,"5min":1.456,"15min":152.29300000000001},"last":1.0840000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.79700000000000004,"15min":2.5339999999999998},"min":{"1min":0.56200000000000006,"5min":0.41599999999999998,"15min":0.27100000000000002},"max":{"1min":1.073,"5min":1.9259999999999999,"15min":153.70400000000001},"last":1.1080000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0029999999999999,"15min":1.2889999999999999},"min":{"1min":0.78100000000000003,"5min":0.64400000000000002,"15min":0.32600000000000001},"max":{"1min":1.631,"5min":1.6850000000000001,"15min":14.343999999999999},"last":1.218},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.089,"15min":1.4259999999999999},"min":{"1min":0.749,"5min":0.60899999999999999,"15min":0.35799999999999998},"max":{"1min":1.4890000000000001,"5min":3.7090000000000001,"15min":12.351000000000001},"last":1.206}]},{"osd":5,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.1299999999999999,"15min":1.228},"min":{"1min":0.81200000000000006,"5min":0.624,"15min":0.35999999999999999},"max":{"1min":1.5469999999999999,"5min":3.9079999999999999,"15min":10.135},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.0629999999999999,"15min":1.369},"min":{"1min":0.81699999999999995,"5min":0.55900000000000005,"15min":0.371},"max":{"1min":1.8460000000000001,"5min":2.2189999999999999,"15min":13.303000000000001},"last":1.3109999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.151,"15min":1.6080000000000001},"min":{"1min":0.78500000000000003,"5min":0.64500000000000002,"15min":0.35899999999999999},"max":{"1min":1.7090000000000001,"5min":1.7090000000000001,"15min":14.714},"last":1.252},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.2310000000000001,"15min":1.629},"min":{"1min":0.83599999999999997,"5min":0.69299999999999995,"15min":0.432},"max":{"1min":1.6259999999999999,"5min":5.9130000000000003,"15min":14.571999999999999},"last":1.4119999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.238,"15min":1.5509999999999999},"min":{"1min":0.82199999999999995,"5min":0.67600000000000005,"15min":0.41599999999999998},"max":{"1min":1.6499999999999999,"5min":5.9500000000000002,"15min":29.513999999999999},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.206,"15min":1.4259999999999999},"min":{"1min":0.90300000000000002,"5min":0.65900000000000003,"15min":0.40200000000000002},"max":{"1min":1.738,"5min":1.883,"15min":30.216999999999999},"last":1.349}]}]},{"osd":3,"up_from":93,"seq":399431959032,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":973308,"kb_used_data":4468,"kb_used_omap":7,"kb_used_meta":968824,"kb_avail":92771844,"statfs":{"total":95995035648,"available":94998368256,"internally_reserved":0,"allocated":4575232,"data_stored":3129376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":992076392},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.72299999999999998,"15min":1.4750000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.20499999999999999},"max":{"1min":0.93500000000000005,"5min":1.228,"15min":46.433999999999997},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.78800000000000003,"15min":1.5800000000000001},"min":{"1min":0.40999999999999998,"5min":0.39300000000000002,"15min":0.20100000000000001},"max":{"1min":1.208,"5min":1.246,"15min":46.387},"last":0.82899999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.81200000000000006,"15min":1.2729999999999999},"min":{"1min":0.55200000000000005,"5min":0.376,"15min":0.23599999999999999},"max":{"1min":1.4259999999999999,"5min":1.429,"15min":32.520000000000003},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.874,"5min":0.876,"15min":1.1919999999999999},"min":{"1min":0.44,"5min":0.313,"15min":0.23100000000000001},"max":{"1min":1.4490000000000001,"5min":1.4750000000000001,"15min":32.585999999999999},"last":0.88800000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.77800000000000002,"15min":1.1910000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.26900000000000002},"max":{"1min":1.1699999999999999,"5min":1.444,"15min":37.997},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.87,"5min":0.84699999999999998,"15min":1.236},"min":{"1min":0.44,"5min":0.39700000000000002,"15min":0.254},"max":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":37.499000000000002},"last":0.67400000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.022,"15min":1.3169999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.316},"max":{"1min":1.236,"5min":1.6120000000000001,"15min":10.445},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.0029999999999999,"15min":1.4259999999999999},"min":{"1min":0.61499999999999999,"5min":0.60599999999999998,"15min":0.33300000000000002},"max":{"1min":1.2390000000000001,"5min":1.8089999999999999,"15min":12.617000000000001},"last":1.0980000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.02,"15min":1.383},"min":{"1min":0.59099999999999997,"5min":0.53900000000000003,"15min":0.34100000000000003},"max":{"1min":1.298,"5min":1.601,"15min":8.5540000000000003},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0640000000000001,"15min":1.4019999999999999},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.36899999999999999},"max":{"1min":1.363,"5min":1.8300000000000001,"15min":9.5210000000000008},"last":1.135}]},{"osd":6,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.1060000000000001,"15min":1.6060000000000001},"min":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":0.37},"max":{"1min":1.393,"5min":1.8700000000000001,"15min":14.124000000000001},"last":1.2829999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.1619999999999999,"15min":1.595},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.38100000000000001},"max":{"1min":1.4770000000000001,"5min":2.0259999999999998,"15min":13.778},"last":1.0429999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.1499999999999999,"15min":1.4430000000000001},"min":{"1min":0.79000000000000004,"5min":0.59499999999999997,"15min":0.42699999999999999},"max":{"1min":1.4930000000000001,"5min":1.7150000000000001,"15min":9.673},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.181,"15min":1.399},"min":{"1min":0.83299999999999996,"5min":0.748,"15min":0.377},"max":{"1min":1.3680000000000001,"5min":1.944,"15min":9.7850000000000001},"last":0.996}]}]},{"osd":4,"up_from":30,"seq":128849019600,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":893580,"kb_used_data":3588,"kb_used_omap":0,"kb_used_meta":889984,"kb_avail":92851572,"statfs":{"total":95995035648,"available":95080009728,"internally_reserved":0,"allocated":3674112,"data_stored":3027000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911343616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:06 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.89800000000000002,"15min":1.909},"min":{"1min":0.67100000000000004,"5min":0.47499999999999998,"15min":0.247},"max":{"1min":1.204,"5min":1.268,"15min":116.59999999999999},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.92100000000000004,"15min":1.74},"min":{"1min":0.64800000000000002,"5min":0.33800000000000002,"15min":0.24199999999999999},"max":{"1min":1.6180000000000001,"5min":1.6180000000000001,"15min":116.843},"last":1.288}]},{"osd":1,"last update":"Mon Oct 9 09:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.88600000000000001,"15min":1.3839999999999999},"min":{"1min":0.61899999999999999,"5min":0.49299999999999999,"15min":0.27700000000000002},"max":{"1min":1.371,"5min":1.5129999999999999,"15min":37.481999999999999},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98099999999999998,"15min":1.5880000000000001},"min":{"1min":0.626,"5min":0.55600000000000005,"15min":0.28100000000000003},"max":{"1min":1.3149999999999999,"5min":1.466,"15min":37.332999999999998},"last":1.0429999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.03,"15min":1.796},"min":{"1min":0.67200000000000004,"5min":0.53500000000000003,"15min":0.32200000000000001},"max":{"1min":1.351,"5min":1.4139999999999999,"15min":116.7},"last":1.266},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.97599999999999998,"15min":1.655},"min":{"1min":0.54100000000000004,"5min":0.50900000000000001,"15min":0.29499999999999998},"max":{"1min":1.2290000000000001,"5min":1.585,"15min":116.63800000000001},"last":1.085}]},{"osd":3,"last update":"Mon Oct 9 09:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.95399999999999996,"15min":1.5489999999999999},"min":{"1min":0.73799999999999999,"5min":0.62,"15min":0.29999999999999999},"max":{"1min":1.208,"5min":1.45,"15min":55.299999999999997},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.042,"15min":1.659},"min":{"1min":0.79700000000000004,"5min":0.60699999999999998,"15min":0.30399999999999999},"max":{"1min":1.4119999999999999,"5min":1.544,"15min":55.622},"last":1.151}]},{"osd":5,"last update":"Mon Oct 9 09:28:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0149999999999999,"15min":1.52},"min":{"1min":0.74399999999999999,"5min":0.64300000000000002,"15min":0.314},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":13.907},"last":1.329},{"interface":"front","average":{"1min":1.103,"5min":1.042,"15min":1.2929999999999999},"min":{"1min":0.77400000000000002,"5min":0.61699999999999999,"15min":0.33300000000000002},"max":{"1min":1.837,"5min":1.837,"15min":11.042999999999999},"last":0.95699999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.012,"15min":1.5229999999999999},"min":{"1min":0.78300000000000003,"5min":0.45100000000000001,"15min":0.29999999999999999},"max":{"1min":1.3500000000000001,"5min":1.5149999999999999,"15min":18.103999999999999},"last":0.872},{"interface":"front","average":{"1min":1.141,"5min":1.1399999999999999,"15min":1.679},"min":{"1min":0.79200000000000004,"5min":0.42799999999999999,"15min":0.34000000000000002},"max":{"1min":1.46,"5min":1.637,"15min":13.444000000000001},"last":1.196}]},{"osd":7,"last update":"Mon Oct 9 09:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.139,"15min":1.6120000000000001},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.33000000000000002},"max":{"1min":1.454,"5min":1.6399999999999999,"15min":15.484999999999999},"last":1.2230000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.21,"15min":1.5700000000000001},"min":{"1min":0.64500000000000002,"5min":0.54600000000000004,"15min":0.33900000000000002},"max":{"1min":1.476,"5min":11.106999999999999,"15min":13.916},"last":1.1120000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823370,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917184,"kb_used_data":7672,"kb_used_omap":0,"kb_used_meta":909504,"kb_avail":92827968,"statfs":{"total":95995035648,"available":95055839232,"internally_reserved":0,"allocated":7856128,"data_stored":7217067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931332096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.321,"15min":2.5489999999999999},"min":{"1min":0.629,"5min":0.44400000000000001,"15min":0.24299999999999999},"max":{"1min":1.252,"5min":36.121000000000002,"15min":139.81700000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.872,"5min":1.2989999999999999,"15min":2.589},"min":{"1min":0.66600000000000004,"5min":0.55100000000000005,"15min":0.251},"max":{"1min":1.323,"5min":36.067999999999998,"15min":141.66900000000001},"last":1.2909999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.91200000000000003,"15min":1.6220000000000001},"min":{"1min":0.60099999999999998,"5min":0.49399999999999999,"15min":0.26800000000000002},"max":{"1min":1.246,"5min":1.486,"15min":36.609999999999999},"last":1.04},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.89600000000000002,"15min":1.5740000000000001},"min":{"1min":0.745,"5min":0.55200000000000005,"15min":0.29699999999999999},"max":{"1min":1.333,"5min":1.333,"15min":36.856999999999999},"last":0.80400000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.95199999999999996,"15min":1.9570000000000001},"min":{"1min":0.60399999999999998,"5min":0.58399999999999996,"15min":0.29299999999999998},"max":{"1min":1.2010000000000001,"5min":3.556,"15min":128.77199999999999},"last":1.087},{"interface":"front","average":{"1min":1.014,"5min":0.99299999999999999,"15min":2.0190000000000001},"min":{"1min":0.67100000000000004,"5min":0.45200000000000001,"15min":0.28599999999999998},"max":{"1min":1.292,"5min":1.399,"15min":128.72999999999999},"last":1.1950000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.98899999999999999,"15min":1.419},"min":{"1min":0.57099999999999995,"5min":0.56399999999999995,"15min":0.32000000000000001},"max":{"1min":1.3080000000000001,"5min":2.9670000000000001,"15min":11.427},"last":1.129},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.96599999999999997,"15min":1.3779999999999999},"min":{"1min":0.71299999999999997,"5min":0.64700000000000002,"15min":0.31},"max":{"1min":1.448,"5min":1.448,"15min":11.019},"last":1.1519999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.95799999999999996,"15min":1.4570000000000001},"min":{"1min":0.68600000000000005,"5min":0.48699999999999999,"15min":0.26600000000000001},"max":{"1min":1.4570000000000001,"5min":1.4570000000000001,"15min":13.904},"last":0.97999999999999998},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0249999999999999,"15min":1.5800000000000001},"min":{"1min":0.65900000000000003,"5min":0.504,"15min":0.36899999999999999},"max":{"1min":1.294,"5min":1.518,"15min":22.228000000000002},"last":1.2470000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.999,"15min":1.54},"min":{"1min":0.66100000000000003,"5min":0.56200000000000006,"15min":0.32000000000000001},"max":{"1min":1.399,"5min":1.536,"15min":53.840000000000003},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.109,"5min":1.121,"15min":1.7589999999999999},"min":{"1min":0.69499999999999995,"5min":0.50800000000000001,"15min":0.29999999999999999},"max":{"1min":1.401,"5min":1.7789999999999999,"15min":50.404000000000003},"last":1.3320000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.145,"15min":1.468},"min":{"1min":0.88700000000000001,"5min":0.55800000000000005,"15min":0.34799999999999998},"max":{"1min":1.4490000000000001,"5min":3.2450000000000001,"15min":12.159000000000001},"last":1.3779999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.093,"15min":1.458},"min":{"1min":0.74099999999999999,"5min":0.53500000000000003,"15min":0.36399999999999999},"max":{"1min":1.4930000000000001,"5min":1.746,"15min":11.331},"last":1.1120000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:29:06.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:29:06.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:06 smithi094 bash[59677]: audit 2023-10-09T09:29:05.525432+0000 mgr.y (mgr.25421) 1560 : audit 0 from='client.36999 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:07.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:06 smithi016 bash[75179]: audit 2023-10-09T09:29:05.525432+0000 mgr.y (mgr.25421) 1560 : audit 0 from='client.36999 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:07.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:06 smithi016 bash[77892]: audit 2023-10-09T09:29:05.525432+0000 mgr.y (mgr.25421) 1560 : audit 0 from='client.36999 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:07.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:07 smithi094 bash[59677]: cluster 2023-10-09T09:29:06.440922+0000 mgr.y (mgr.25421) 1561 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:08.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:07 smithi016 bash[75179]: cluster 2023-10-09T09:29:06.440922+0000 mgr.y (mgr.25421) 1561 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:08.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:07 smithi016 bash[77892]: cluster 2023-10-09T09:29:06.440922+0000 mgr.y (mgr.25421) 1561 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:09.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:09 smithi094 bash[59677]: cluster 2023-10-09T09:29:08.441762+0000 mgr.y (mgr.25421) 1562 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:10.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:09 smithi016 bash[75179]: cluster 2023-10-09T09:29:08.441762+0000 mgr.y (mgr.25421) 1562 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:10.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:09 smithi016 bash[77892]: cluster 2023-10-09T09:29:08.441762+0000 mgr.y (mgr.25421) 1562 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:11.418 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:11.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:11 smithi016 bash[75179]: cluster 2023-10-09T09:29:10.442377+0000 mgr.y (mgr.25421) 1563 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:11.448 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:11 smithi016 bash[77892]: cluster 2023-10-09T09:29:10.442377+0000 mgr.y (mgr.25421) 1563 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:11.490 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:11 smithi094 bash[59677]: cluster 2023-10-09T09:29:10.442377+0000 mgr.y (mgr.25421) 1563 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:13.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:13 smithi016 bash[75179]: cluster 2023-10-09T09:29:12.443225+0000 mgr.y (mgr.25421) 1564 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:13.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:13 smithi016 bash[77892]: cluster 2023-10-09T09:29:12.443225+0000 mgr.y (mgr.25421) 1564 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:13.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:13 smithi094 bash[59677]: cluster 2023-10-09T09:29:12.443225+0000 mgr.y (mgr.25421) 1564 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:15.421 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-10-09T09:29:15.424 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-10-09T09:29:15.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:15 smithi016 bash[75179]: cluster 2023-10-09T09:29:14.444064+0000 mgr.y (mgr.25421) 1565 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:15.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:15 smithi016 bash[77892]: cluster 2023-10-09T09:29:14.444064+0000 mgr.y (mgr.25421) 1565 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:15.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:15 smithi094 bash[59677]: cluster 2023-10-09T09:29:14.444064+0000 mgr.y (mgr.25421) 1565 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-09T09:29:16.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:15 smithi016 bash[71225]: ::ffff:172.21.15.94 - - [09/Oct/2023:09:29:15] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.33.4" 2023-10-09T09:29:16.411 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1795,"stamp":"2023-10-09T09:29:14.443539+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6288,"num_read_kb":12724,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":2972600,"num_keys_recovered":70,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7827860,"kb_used_data":46292,"kb_used_omap":126,"kb_used_meta":7781377,"kb_avail":742133356,"statfs":{"total":767960285184,"available":759944556544,"internally_reserved":0,"allocated":47403008,"data_stored":38979357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":7968130683},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004532"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":321,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547613+0000","last_change":"2023-10-09T09:20:19.894893+0000","last_active":"2023-10-09T09:27:39.547613+0000","last_peered":"2023-10-09T09:27:39.547613+0000","last_clean":"2023-10-09T09:27:39.547613+0000","last_became_active":"2023-10-09T09:20:19.893953+0000","last_became_peered":"2023-10-09T09:20:19.893953+0000","last_unstale":"2023-10-09T09:27:39.547613+0000","last_undegraded":"2023-10-09T09:27:39.547613+0000","last_fullsized":"2023-10-09T09:27:39.547613+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":259,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673574+0000","last_change":"2023-10-09T09:26:19.643487+0000","last_active":"2023-10-09T09:27:40.673574+0000","last_peered":"2023-10-09T09:27:40.673574+0000","last_clean":"2023-10-09T09:27:40.673574+0000","last_became_active":"2023-10-09T09:26:19.641799+0000","last_became_peered":"2023-10-09T09:26:19.641799+0000","last_unstale":"2023-10-09T09:27:40.673574+0000","last_undegraded":"2023-10-09T09:27:40.673574+0000","last_fullsized":"2023-10-09T09:27:40.673574+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:32.569466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":165,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549236+0000","last_change":"2023-10-09T09:14:48.312902+0000","last_active":"2023-10-09T09:27:39.549236+0000","last_peered":"2023-10-09T09:27:39.549236+0000","last_clean":"2023-10-09T09:27:39.549236+0000","last_became_active":"2023-10-09T09:14:48.312532+0000","last_became_peered":"2023-10-09T09:14:48.312532+0000","last_unstale":"2023-10-09T09:27:39.549236+0000","last_undegraded":"2023-10-09T09:27:39.549236+0000","last_fullsized":"2023-10-09T09:27:39.549236+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:00.352257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":454,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562334+0000","last_change":"2023-10-09T09:09:44.392294+0000","last_active":"2023-10-09T09:27:40.562334+0000","last_peered":"2023-10-09T09:27:40.562334+0000","last_clean":"2023-10-09T09:27:40.562334+0000","last_became_active":"2023-10-09T09:09:44.390680+0000","last_became_peered":"2023-10-09T09:09:44.390680+0000","last_unstale":"2023-10-09T09:27:40.562334+0000","last_undegraded":"2023-10-09T09:27:40.562334+0000","last_fullsized":"2023-10-09T09:27:40.562334+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:57.729341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673116+0000","last_change":"2023-10-09T09:26:19.644393+0000","last_active":"2023-10-09T09:27:40.673116+0000","last_peered":"2023-10-09T09:27:40.673116+0000","last_clean":"2023-10-09T09:27:40.673116+0000","last_became_active":"2023-10-09T09:26:19.643611+0000","last_became_peered":"2023-10-09T09:26:19.643611+0000","last_unstale":"2023-10-09T09:27:40.673116+0000","last_undegraded":"2023-10-09T09:27:40.673116+0000","last_fullsized":"2023-10-09T09:27:40.673116+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:50.548997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":326,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549815+0000","last_change":"2023-10-09T09:19:42.872407+0000","last_active":"2023-10-09T09:27:39.549815+0000","last_peered":"2023-10-09T09:27:39.549815+0000","last_clean":"2023-10-09T09:27:39.549815+0000","last_became_active":"2023-10-09T09:19:42.871637+0000","last_became_peered":"2023-10-09T09:19:42.871637+0000","last_unstale":"2023-10-09T09:27:39.549815+0000","last_undegraded":"2023-10-09T09:27:39.549815+0000","last_fullsized":"2023-10-09T09:27:39.549815+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":405,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536423+0000","last_change":"2023-10-09T09:08:24.657022+0000","last_active":"2023-10-09T09:27:39.536423+0000","last_peered":"2023-10-09T09:27:39.536423+0000","last_clean":"2023-10-09T09:27:39.536423+0000","last_became_active":"2023-10-09T09:08:24.656570+0000","last_became_peered":"2023-10-09T09:08:24.656570+0000","last_unstale":"2023-10-09T09:27:39.536423+0000","last_undegraded":"2023-10-09T09:27:39.536423+0000","last_fullsized":"2023-10-09T09:27:39.536423+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:09.236685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":260,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673205+0000","last_change":"2023-10-09T09:26:19.644374+0000","last_active":"2023-10-09T09:27:40.673205+0000","last_peered":"2023-10-09T09:27:40.673205+0000","last_clean":"2023-10-09T09:27:40.673205+0000","last_became_active":"2023-10-09T09:26:19.642350+0000","last_became_peered":"2023-10-09T09:26:19.642350+0000","last_unstale":"2023-10-09T09:27:40.673205+0000","last_undegraded":"2023-10-09T09:27:40.673205+0000","last_fullsized":"2023-10-09T09:27:40.673205+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550028+0000","last_change":"2023-10-09T09:14:48.324272+0000","last_active":"2023-10-09T09:27:39.550028+0000","last_peered":"2023-10-09T09:27:39.550028+0000","last_clean":"2023-10-09T09:27:39.550028+0000","last_became_active":"2023-10-09T09:14:48.323879+0000","last_became_peered":"2023-10-09T09:14:48.323879+0000","last_unstale":"2023-10-09T09:27:39.550028+0000","last_undegraded":"2023-10-09T09:27:39.550028+0000","last_fullsized":"2023-10-09T09:27:39.550028+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":236,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673097+0000","last_change":"2023-10-09T09:26:19.493403+0000","last_active":"2023-10-09T09:27:40.673097+0000","last_peered":"2023-10-09T09:27:40.673097+0000","last_clean":"2023-10-09T09:27:40.673097+0000","last_became_active":"2023-10-09T09:26:19.493049+0000","last_became_peered":"2023-10-09T09:26:19.493049+0000","last_unstale":"2023-10-09T09:27:40.673097+0000","last_undegraded":"2023-10-09T09:27:40.673097+0000","last_fullsized":"2023-10-09T09:27:40.673097+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:30.756826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":200,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.544118+0000","last_change":"2023-10-09T09:09:44.784419+0000","last_active":"2023-10-09T09:27:39.544118+0000","last_peered":"2023-10-09T09:27:39.544118+0000","last_clean":"2023-10-09T09:27:39.544118+0000","last_became_active":"2023-10-09T09:09:44.364993+0000","last_became_peered":"2023-10-09T09:09:44.364993+0000","last_unstale":"2023-10-09T09:27:39.544118+0000","last_undegraded":"2023-10-09T09:27:39.544118+0000","last_fullsized":"2023-10-09T09:27:39.544118+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":193,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.553130+0000","last_change":"2023-10-09T09:20:19.898526+0000","last_active":"2023-10-09T09:27:40.553130+0000","last_peered":"2023-10-09T09:27:40.553130+0000","last_clean":"2023-10-09T09:27:40.553130+0000","last_became_active":"2023-10-09T09:20:19.898197+0000","last_became_peered":"2023-10-09T09:20:19.898197+0000","last_unstale":"2023-10-09T09:27:40.553130+0000","last_undegraded":"2023-10-09T09:27:40.553130+0000","last_fullsized":"2023-10-09T09:27:40.553130+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:30.135339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"74'22","reported_seq":315,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548354+0000","last_change":"2023-10-09T09:09:44.374213+0000","last_active":"2023-10-09T09:27:39.548354+0000","last_peered":"2023-10-09T09:27:39.548354+0000","last_clean":"2023-10-09T09:27:39.548354+0000","last_became_active":"2023-10-09T09:09:44.352336+0000","last_became_peered":"2023-10-09T09:09:44.352336+0000","last_unstale":"2023-10-09T09:27:39.548354+0000","last_undegraded":"2023-10-09T09:27:39.548354+0000","last_fullsized":"2023-10-09T09:27:39.548354+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":303,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547600+0000","last_change":"2023-10-09T09:14:48.318237+0000","last_active":"2023-10-09T09:27:39.547600+0000","last_peered":"2023-10-09T09:27:39.547600+0000","last_clean":"2023-10-09T09:27:39.547600+0000","last_became_active":"2023-10-09T09:14:48.317751+0000","last_became_peered":"2023-10-09T09:14:48.317751+0000","last_unstale":"2023-10-09T09:27:39.547600+0000","last_undegraded":"2023-10-09T09:27:39.547600+0000","last_fullsized":"2023-10-09T09:27:39.547600+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":260,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.543717+0000","last_change":"2023-10-09T09:20:19.876172+0000","last_active":"2023-10-09T09:27:39.543717+0000","last_peered":"2023-10-09T09:27:39.543717+0000","last_clean":"2023-10-09T09:27:39.543717+0000","last_became_active":"2023-10-09T09:20:19.765721+0000","last_became_peered":"2023-10-09T09:20:19.765721+0000","last_unstale":"2023-10-09T09:27:39.543717+0000","last_undegraded":"2023-10-09T09:27:39.543717+0000","last_fullsized":"2023-10-09T09:27:39.543717+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":256,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673613+0000","last_change":"2023-10-09T09:26:19.643064+0000","last_active":"2023-10-09T09:27:40.673613+0000","last_peered":"2023-10-09T09:27:40.673613+0000","last_clean":"2023-10-09T09:27:40.673613+0000","last_became_active":"2023-10-09T09:26:19.642261+0000","last_became_peered":"2023-10-09T09:26:19.642261+0000","last_unstale":"2023-10-09T09:27:40.673613+0000","last_undegraded":"2023-10-09T09:27:40.673613+0000","last_fullsized":"2023-10-09T09:27:40.673613+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:14.490696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":433,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562041+0000","last_change":"2023-10-09T09:09:44.380599+0000","last_active":"2023-10-09T09:27:40.562041+0000","last_peered":"2023-10-09T09:27:40.562041+0000","last_clean":"2023-10-09T09:27:40.562041+0000","last_became_active":"2023-10-09T09:09:44.380307+0000","last_became_peered":"2023-10-09T09:09:44.380307+0000","last_unstale":"2023-10-09T09:27:40.562041+0000","last_undegraded":"2023-10-09T09:27:40.562041+0000","last_fullsized":"2023-10-09T09:27:40.562041+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.934684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549880+0000","last_change":"2023-10-09T09:26:49.502769+0000","last_active":"2023-10-09T09:27:39.549880+0000","last_peered":"2023-10-09T09:27:39.549880+0000","last_clean":"2023-10-09T09:27:39.549880+0000","last_became_active":"2023-10-09T09:26:49.502432+0000","last_became_peered":"2023-10-09T09:26:49.502432+0000","last_unstale":"2023-10-09T09:27:39.549880+0000","last_undegraded":"2023-10-09T09:27:39.549880+0000","last_fullsized":"2023-10-09T09:27:39.549880+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":104,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.682120+0000","last_change":"2023-10-09T09:27:40.682120+0000","last_active":"2023-10-09T09:27:40.682120+0000","last_peered":"2023-10-09T09:27:40.682120+0000","last_clean":"2023-10-09T09:27:40.682120+0000","last_became_active":"2023-10-09T09:27:40.681426+0000","last_became_peered":"2023-10-09T09:27:40.681426+0000","last_unstale":"2023-10-09T09:27:40.682120+0000","last_undegraded":"2023-10-09T09:27:40.682120+0000","last_fullsized":"2023-10-09T09:27:40.682120+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":405,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.537193+0000","last_change":"2023-10-09T09:09:44.392774+0000","last_active":"2023-10-09T09:27:39.537193+0000","last_peered":"2023-10-09T09:27:39.537193+0000","last_clean":"2023-10-09T09:27:39.537193+0000","last_became_active":"2023-10-09T09:09:44.391166+0000","last_became_peered":"2023-10-09T09:09:44.391166+0000","last_unstale":"2023-10-09T09:27:39.537193+0000","last_undegraded":"2023-10-09T09:27:39.537193+0000","last_fullsized":"2023-10-09T09:27:39.537193+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:11.967578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":349,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548985+0000","last_change":"2023-10-09T09:26:49.513269+0000","last_active":"2023-10-09T09:27:39.548985+0000","last_peered":"2023-10-09T09:27:39.548985+0000","last_clean":"2023-10-09T09:27:39.548985+0000","last_became_active":"2023-10-09T09:26:49.503720+0000","last_became_peered":"2023-10-09T09:26:49.503720+0000","last_unstale":"2023-10-09T09:27:39.548985+0000","last_undegraded":"2023-10-09T09:27:39.548985+0000","last_fullsized":"2023-10-09T09:27:39.548985+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":179,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673791+0000","last_change":"2023-10-09T09:26:19.494282+0000","last_active":"2023-10-09T09:27:40.673791+0000","last_peered":"2023-10-09T09:27:40.673791+0000","last_clean":"2023-10-09T09:27:40.673791+0000","last_became_active":"2023-10-09T09:26:19.493851+0000","last_became_peered":"2023-10-09T09:26:19.493851+0000","last_unstale":"2023-10-09T09:27:40.673791+0000","last_undegraded":"2023-10-09T09:27:40.673791+0000","last_fullsized":"2023-10-09T09:27:40.673791+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:44.212093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":211,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.544116+0000","last_change":"2023-10-09T09:14:19.791274+0000","last_active":"2023-10-09T09:27:39.544116+0000","last_peered":"2023-10-09T09:27:39.544116+0000","last_clean":"2023-10-09T09:27:39.544116+0000","last_became_active":"2023-10-09T09:14:19.743867+0000","last_became_peered":"2023-10-09T09:14:19.743867+0000","last_unstale":"2023-10-09T09:27:39.544116+0000","last_undegraded":"2023-10-09T09:27:39.544116+0000","last_fullsized":"2023-10-09T09:27:39.544116+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.546405+0000","last_change":"2023-10-09T09:26:49.502858+0000","last_active":"2023-10-09T09:27:39.546405+0000","last_peered":"2023-10-09T09:27:39.546405+0000","last_clean":"2023-10-09T09:27:39.546405+0000","last_became_active":"2023-10-09T09:26:49.502402+0000","last_became_peered":"2023-10-09T09:26:49.502402+0000","last_unstale":"2023-10-09T09:27:39.546405+0000","last_undegraded":"2023-10-09T09:27:39.546405+0000","last_fullsized":"2023-10-09T09:27:39.546405+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":211,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.544271+0000","last_change":"2023-10-09T09:14:48.387980+0000","last_active":"2023-10-09T09:27:39.544271+0000","last_peered":"2023-10-09T09:27:39.544271+0000","last_clean":"2023-10-09T09:27:39.544271+0000","last_became_active":"2023-10-09T09:14:48.307869+0000","last_became_peered":"2023-10-09T09:14:48.307869+0000","last_unstale":"2023-10-09T09:27:39.544271+0000","last_undegraded":"2023-10-09T09:27:39.544271+0000","last_fullsized":"2023-10-09T09:27:39.544271+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":303,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547629+0000","last_change":"2023-10-09T09:14:48.320346+0000","last_active":"2023-10-09T09:27:39.547629+0000","last_peered":"2023-10-09T09:27:39.547629+0000","last_clean":"2023-10-09T09:27:39.547629+0000","last_became_active":"2023-10-09T09:14:48.319962+0000","last_became_peered":"2023-10-09T09:14:48.319962+0000","last_unstale":"2023-10-09T09:27:39.547629+0000","last_undegraded":"2023-10-09T09:27:39.547629+0000","last_fullsized":"2023-10-09T09:27:39.547629+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":51,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.569130+0000","last_change":"2023-10-09T09:27:40.569130+0000","last_active":"2023-10-09T09:27:40.569130+0000","last_peered":"2023-10-09T09:27:40.569130+0000","last_clean":"2023-10-09T09:27:40.569130+0000","last_became_active":"2023-10-09T09:27:40.568730+0000","last_became_peered":"2023-10-09T09:27:40.568730+0000","last_unstale":"2023-10-09T09:27:40.569130+0000","last_undegraded":"2023-10-09T09:27:40.569130+0000","last_fullsized":"2023-10-09T09:27:40.569130+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":317,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549501+0000","last_change":"2023-10-09T09:14:48.306179+0000","last_active":"2023-10-09T09:27:39.549501+0000","last_peered":"2023-10-09T09:27:39.549501+0000","last_clean":"2023-10-09T09:27:39.549501+0000","last_became_active":"2023-10-09T09:14:48.305024+0000","last_became_peered":"2023-10-09T09:14:48.305024+0000","last_unstale":"2023-10-09T09:27:39.549501+0000","last_undegraded":"2023-10-09T09:27:39.549501+0000","last_fullsized":"2023-10-09T09:27:39.549501+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":201,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549939+0000","last_change":"2023-10-09T09:08:24.730952+0000","last_active":"2023-10-09T09:27:39.549939+0000","last_peered":"2023-10-09T09:27:39.549939+0000","last_clean":"2023-10-09T09:27:39.549939+0000","last_became_active":"2023-10-09T09:08:24.729940+0000","last_became_peered":"2023-10-09T09:08:24.729940+0000","last_unstale":"2023-10-09T09:27:39.549939+0000","last_undegraded":"2023-10-09T09:27:39.549939+0000","last_fullsized":"2023-10-09T09:27:39.549939+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":434,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562012+0000","last_change":"2023-10-09T09:18:51.869617+0000","last_active":"2023-10-09T09:27:40.562012+0000","last_peered":"2023-10-09T09:27:40.562012+0000","last_clean":"2023-10-09T09:27:40.562012+0000","last_became_active":"2023-10-09T09:18:51.869212+0000","last_became_peered":"2023-10-09T09:18:51.869212+0000","last_unstale":"2023-10-09T09:27:40.562012+0000","last_undegraded":"2023-10-09T09:27:40.562012+0000","last_fullsized":"2023-10-09T09:27:40.562012+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:36.753958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":249,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673407+0000","last_change":"2023-10-09T09:26:19.488721+0000","last_active":"2023-10-09T09:27:40.673407+0000","last_peered":"2023-10-09T09:27:40.673407+0000","last_clean":"2023-10-09T09:27:40.673407+0000","last_became_active":"2023-10-09T09:26:19.488304+0000","last_became_peered":"2023-10-09T09:26:19.488304+0000","last_unstale":"2023-10-09T09:27:40.673407+0000","last_undegraded":"2023-10-09T09:27:40.673407+0000","last_fullsized":"2023-10-09T09:27:40.673407+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:02.529432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":276,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549089+0000","last_change":"2023-10-09T09:00:31.860601+0000","last_active":"2023-10-09T09:27:39.549089+0000","last_peered":"2023-10-09T09:27:39.549089+0000","last_clean":"2023-10-09T09:27:39.549089+0000","last_became_active":"2023-10-09T09:00:31.860193+0000","last_became_peered":"2023-10-09T09:00:31.860193+0000","last_unstale":"2023-10-09T09:27:39.549089+0000","last_undegraded":"2023-10-09T09:27:39.549089+0000","last_fullsized":"2023-10-09T09:27:39.549089+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":281,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549346+0000","last_change":"2023-10-09T09:14:48.310098+0000","last_active":"2023-10-09T09:27:39.549346+0000","last_peered":"2023-10-09T09:27:39.549346+0000","last_clean":"2023-10-09T09:27:39.549346+0000","last_became_active":"2023-10-09T09:14:48.309468+0000","last_became_peered":"2023-10-09T09:14:48.309468+0000","last_unstale":"2023-10-09T09:27:39.549346+0000","last_undegraded":"2023-10-09T09:27:39.549346+0000","last_fullsized":"2023-10-09T09:27:39.549346+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":321,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548587+0000","last_change":"2023-10-09T09:20:19.935938+0000","last_active":"2023-10-09T09:27:39.548587+0000","last_peered":"2023-10-09T09:27:39.548587+0000","last_clean":"2023-10-09T09:27:39.548587+0000","last_became_active":"2023-10-09T09:20:19.935423+0000","last_became_peered":"2023-10-09T09:20:19.935423+0000","last_unstale":"2023-10-09T09:27:39.548587+0000","last_undegraded":"2023-10-09T09:27:39.548587+0000","last_fullsized":"2023-10-09T09:27:39.548587+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":348,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547275+0000","last_change":"2023-10-09T09:20:19.755858+0000","last_active":"2023-10-09T09:27:39.547275+0000","last_peered":"2023-10-09T09:27:39.547275+0000","last_clean":"2023-10-09T09:27:39.547275+0000","last_became_active":"2023-10-09T09:20:19.755442+0000","last_became_peered":"2023-10-09T09:20:19.755442+0000","last_unstale":"2023-10-09T09:27:39.547275+0000","last_undegraded":"2023-10-09T09:27:39.547275+0000","last_fullsized":"2023-10-09T09:27:39.547275+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":294,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548569+0000","last_change":"2023-10-09T09:14:48.327442+0000","last_active":"2023-10-09T09:27:39.548569+0000","last_peered":"2023-10-09T09:27:39.548569+0000","last_clean":"2023-10-09T09:27:39.548569+0000","last_became_active":"2023-10-09T09:14:48.323304+0000","last_became_peered":"2023-10-09T09:14:48.323304+0000","last_unstale":"2023-10-09T09:27:39.548569+0000","last_undegraded":"2023-10-09T09:27:39.548569+0000","last_fullsized":"2023-10-09T09:27:39.548569+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":226,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.672979+0000","last_change":"2023-10-09T09:26:19.644787+0000","last_active":"2023-10-09T09:27:40.672979+0000","last_peered":"2023-10-09T09:27:40.672979+0000","last_clean":"2023-10-09T09:27:40.672979+0000","last_became_active":"2023-10-09T09:26:19.643767+0000","last_became_peered":"2023-10-09T09:26:19.643767+0000","last_unstale":"2023-10-09T09:27:40.672979+0000","last_undegraded":"2023-10-09T09:27:40.672979+0000","last_fullsized":"2023-10-09T09:27:40.672979+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:26.488695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.554641+0000","last_change":"2023-10-09T09:14:48.317937+0000","last_active":"2023-10-09T09:27:40.554641+0000","last_peered":"2023-10-09T09:27:40.554641+0000","last_clean":"2023-10-09T09:27:40.554641+0000","last_became_active":"2023-10-09T09:14:48.317565+0000","last_became_peered":"2023-10-09T09:14:48.317565+0000","last_unstale":"2023-10-09T09:27:40.554641+0000","last_undegraded":"2023-10-09T09:27:40.554641+0000","last_fullsized":"2023-10-09T09:27:40.554641+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:31.064234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"391'120","reported_seq":527,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549981+0000","last_change":"2023-10-09T09:24:22.950512+0000","last_active":"2023-10-09T09:27:39.549981+0000","last_peered":"2023-10-09T09:27:39.549981+0000","last_clean":"2023-10-09T09:27:39.549981+0000","last_became_active":"2023-10-09T09:14:48.299261+0000","last_became_peered":"2023-10-09T09:14:48.299261+0000","last_unstale":"2023-10-09T09:27:39.549981+0000","last_undegraded":"2023-10-09T09:27:39.549981+0000","last_fullsized":"2023-10-09T09:27:39.549981+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.210535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040695899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":2231,"num_write":134,"num_write_kb":1460,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550142+0000","last_change":"2023-10-09T09:20:19.757594+0000","last_active":"2023-10-09T09:27:39.550142+0000","last_peered":"2023-10-09T09:27:39.550142+0000","last_clean":"2023-10-09T09:27:39.550142+0000","last_became_active":"2023-10-09T09:20:19.757216+0000","last_became_peered":"2023-10-09T09:20:19.757216+0000","last_unstale":"2023-10-09T09:27:39.550142+0000","last_undegraded":"2023-10-09T09:27:39.550142+0000","last_fullsized":"2023-10-09T09:27:39.550142+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549419+0000","last_change":"2023-10-09T09:26:49.501845+0000","last_active":"2023-10-09T09:27:39.549419+0000","last_peered":"2023-10-09T09:27:39.549419+0000","last_clean":"2023-10-09T09:27:39.549419+0000","last_became_active":"2023-10-09T09:26:49.501350+0000","last_became_peered":"2023-10-09T09:26:49.501350+0000","last_unstale":"2023-10-09T09:27:39.549419+0000","last_undegraded":"2023-10-09T09:27:39.549419+0000","last_fullsized":"2023-10-09T09:27:39.549419+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:18.155759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"391'94","reported_seq":637,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.546517+0000","last_change":"2023-10-09T09:24:22.939657+0000","last_active":"2023-10-09T09:27:39.546517+0000","last_peered":"2023-10-09T09:27:39.546517+0000","last_clean":"2023-10-09T09:27:39.546517+0000","last_became_active":"2023-10-09T09:14:48.318703+0000","last_became_peered":"2023-10-09T09:14:48.318703+0000","last_unstale":"2023-10-09T09:27:39.546517+0000","last_undegraded":"2023-10-09T09:27:39.546517+0000","last_fullsized":"2023-10-09T09:27:39.546517+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":94,"log_dups_size":0,"ondisk_log_size":94,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":536,"num_read_kb":1670,"num_write":97,"num_write_kb":5264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":276,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.548141+0000","last_change":"2023-10-09T09:00:31.860119+0000","last_active":"2023-10-09T09:27:39.548141+0000","last_peered":"2023-10-09T09:27:39.548141+0000","last_clean":"2023-10-09T09:27:39.548141+0000","last_became_active":"2023-10-09T09:00:31.859730+0000","last_became_peered":"2023-10-09T09:00:31.859730+0000","last_unstale":"2023-10-09T09:27:39.548141+0000","last_undegraded":"2023-10-09T09:27:39.548141+0000","last_fullsized":"2023-10-09T09:27:39.548141+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":193,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549860+0000","last_change":"2023-10-09T09:20:19.932428+0000","last_active":"2023-10-09T09:27:39.549860+0000","last_peered":"2023-10-09T09:27:39.549860+0000","last_clean":"2023-10-09T09:27:39.549860+0000","last_became_active":"2023-10-09T09:20:19.931684+0000","last_became_peered":"2023-10-09T09:20:19.931684+0000","last_unstale":"2023-10-09T09:27:39.549860+0000","last_undegraded":"2023-10-09T09:27:39.549860+0000","last_fullsized":"2023-10-09T09:27:39.549860+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:40.774870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"391'108","reported_seq":551,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.554712+0000","last_change":"2023-10-09T09:24:22.935939+0000","last_active":"2023-10-09T09:27:40.554712+0000","last_peered":"2023-10-09T09:27:40.554712+0000","last_clean":"2023-10-09T09:27:40.554712+0000","last_became_active":"2023-10-09T09:20:19.757450+0000","last_became_peered":"2023-10-09T09:20:19.757450+0000","last_unstale":"2023-10-09T09:27:40.554712+0000","last_undegraded":"2023-10-09T09:27:40.554712+0000","last_fullsized":"2023-10-09T09:27:40.554712+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:57.053230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068192000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":697,"num_read_kb":1551,"num_write":95,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":194,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.552422+0000","last_change":"2023-10-09T09:20:19.902503+0000","last_active":"2023-10-09T09:27:40.552422+0000","last_peered":"2023-10-09T09:27:40.552422+0000","last_clean":"2023-10-09T09:27:40.552422+0000","last_became_active":"2023-10-09T09:20:19.902131+0000","last_became_peered":"2023-10-09T09:20:19.902131+0000","last_unstale":"2023-10-09T09:27:40.552422+0000","last_undegraded":"2023-10-09T09:27:40.552422+0000","last_fullsized":"2023-10-09T09:27:40.552422+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:43.897708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.553463+0000","last_change":"2023-10-09T09:21:51.904699+0000","last_active":"2023-10-09T09:27:40.553463+0000","last_peered":"2023-10-09T09:27:40.553463+0000","last_clean":"2023-10-09T09:27:40.553463+0000","last_became_active":"2023-10-09T09:21:51.904381+0000","last_became_peered":"2023-10-09T09:21:51.904381+0000","last_unstale":"2023-10-09T09:27:40.553463+0000","last_undegraded":"2023-10-09T09:27:40.553463+0000","last_fullsized":"2023-10-09T09:27:40.553463+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:46.915626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"391'140","reported_seq":523,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549130+0000","last_change":"2023-10-09T09:24:22.940815+0000","last_active":"2023-10-09T09:27:39.549130+0000","last_peered":"2023-10-09T09:27:39.549130+0000","last_clean":"2023-10-09T09:27:39.549130+0000","last_became_active":"2023-10-09T09:20:19.759254+0000","last_became_peered":"2023-10-09T09:20:19.759254+0000","last_unstale":"2023-10-09T09:27:39.549130+0000","last_undegraded":"2023-10-09T09:27:39.549130+0000","last_fullsized":"2023-10-09T09:27:39.549130+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":267,"num_read_kb":1259,"num_write":146,"num_write_kb":2143,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"74'139","reported_seq":652,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549147+0000","last_change":"2023-10-09T09:11:40.394822+0000","last_active":"2023-10-09T09:27:39.549147+0000","last_peered":"2023-10-09T09:27:39.549147+0000","last_clean":"2023-10-09T09:27:39.549147+0000","last_became_active":"2023-10-09T09:11:40.393105+0000","last_became_peered":"2023-10-09T09:11:40.393105+0000","last_unstale":"2023-10-09T09:27:39.549147+0000","last_undegraded":"2023-10-09T09:27:39.549147+0000","last_fullsized":"2023-10-09T09:27:39.549147+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:27:38.586588+0000","last_clean_scrub_stamp":"2023-10-09T08:27:38.586588+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":75,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673394+0000","last_change":"2023-10-09T09:26:19.491386+0000","last_active":"2023-10-09T09:27:40.673394+0000","last_peered":"2023-10-09T09:27:40.673394+0000","last_clean":"2023-10-09T09:27:40.673394+0000","last_became_active":"2023-10-09T09:26:19.491104+0000","last_became_peered":"2023-10-09T09:26:19.491104+0000","last_unstale":"2023-10-09T09:27:40.673394+0000","last_undegraded":"2023-10-09T09:27:40.673394+0000","last_fullsized":"2023-10-09T09:27:40.673394+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:14.546226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":272,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550536+0000","last_change":"2023-10-09T09:09:44.360770+0000","last_active":"2023-10-09T09:27:39.550536+0000","last_peered":"2023-10-09T09:27:39.550536+0000","last_clean":"2023-10-09T09:27:39.550536+0000","last_became_active":"2023-10-09T09:09:44.360226+0000","last_became_peered":"2023-10-09T09:09:44.360226+0000","last_unstale":"2023-10-09T09:27:39.550536+0000","last_undegraded":"2023-10-09T09:27:39.550536+0000","last_fullsized":"2023-10-09T09:27:39.550536+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"391'86","reported_seq":481,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.565407+0000","last_change":"2023-10-09T09:27:40.565407+0000","last_active":"2023-10-09T09:27:40.565407+0000","last_peered":"2023-10-09T09:27:40.565407+0000","last_clean":"2023-10-09T09:27:40.565407+0000","last_became_active":"2023-10-09T09:27:40.564527+0000","last_became_peered":"2023-10-09T09:27:40.564527+0000","last_unstale":"2023-10-09T09:27:40.565407+0000","last_undegraded":"2023-10-09T09:27:40.565407+0000","last_fullsized":"2023-10-09T09:27:40.565407+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":615,"num_read_kb":534,"num_write":66,"num_write_kb":1068,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":453,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.535870+0000","last_change":"2023-10-09T09:03:19.557726+0000","last_active":"2023-10-09T09:27:39.535870+0000","last_peered":"2023-10-09T09:27:39.535870+0000","last_clean":"2023-10-09T09:27:39.535870+0000","last_became_active":"2023-10-09T09:03:19.557425+0000","last_became_peered":"2023-10-09T09:03:19.557425+0000","last_unstale":"2023-10-09T09:27:39.535870+0000","last_undegraded":"2023-10-09T09:27:39.535870+0000","last_fullsized":"2023-10-09T09:27:39.535870+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:05.251015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":477,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562152+0000","last_change":"2023-10-09T09:20:19.890699+0000","last_active":"2023-10-09T09:27:40.562152+0000","last_peered":"2023-10-09T09:27:40.562152+0000","last_clean":"2023-10-09T09:27:40.562152+0000","last_became_active":"2023-10-09T09:20:19.889949+0000","last_became_peered":"2023-10-09T09:20:19.889949+0000","last_unstale":"2023-10-09T09:27:40.562152+0000","last_undegraded":"2023-10-09T09:27:40.562152+0000","last_fullsized":"2023-10-09T09:27:40.562152+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:59.920953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"391'161","reported_seq":1030,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.543650+0000","last_change":"2023-10-09T09:24:22.945624+0000","last_active":"2023-10-09T09:27:39.543650+0000","last_peered":"2023-10-09T09:27:39.543650+0000","last_clean":"2023-10-09T09:27:39.543650+0000","last_became_active":"2023-10-09T09:09:44.438688+0000","last_became_peered":"2023-10-09T09:09:44.438688+0000","last_unstale":"2023-10-09T09:27:39.543650+0000","last_undegraded":"2023-10-09T09:27:39.543650+0000","last_fullsized":"2023-10-09T09:27:39.543650+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1582,"num_read_kb":1930,"num_write":111,"num_write_kb":341,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":193,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549724+0000","last_change":"2023-10-09T09:20:19.912177+0000","last_active":"2023-10-09T09:27:39.549724+0000","last_peered":"2023-10-09T09:27:39.549724+0000","last_clean":"2023-10-09T09:27:39.549724+0000","last_became_active":"2023-10-09T09:20:19.908966+0000","last_became_peered":"2023-10-09T09:20:19.908966+0000","last_unstale":"2023-10-09T09:27:39.549724+0000","last_undegraded":"2023-10-09T09:27:39.549724+0000","last_fullsized":"2023-10-09T09:27:39.549724+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:54.778943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":429,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536841+0000","last_change":"2023-10-09T09:09:44.392480+0000","last_active":"2023-10-09T09:27:39.536841+0000","last_peered":"2023-10-09T09:27:39.536841+0000","last_clean":"2023-10-09T09:27:39.536841+0000","last_became_active":"2023-10-09T09:09:44.392022+0000","last_became_peered":"2023-10-09T09:09:44.392022+0000","last_unstale":"2023-10-09T09:27:39.536841+0000","last_undegraded":"2023-10-09T09:27:39.536841+0000","last_fullsized":"2023-10-09T09:27:39.536841+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:52.444036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":37,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673720+0000","last_change":"2023-10-09T09:26:19.494388+0000","last_active":"2023-10-09T09:27:40.673720+0000","last_peered":"2023-10-09T09:27:40.673720+0000","last_clean":"2023-10-09T09:27:40.673720+0000","last_became_active":"2023-10-09T09:26:19.493921+0000","last_became_peered":"2023-10-09T09:26:19.493921+0000","last_unstale":"2023-10-09T09:27:40.673720+0000","last_undegraded":"2023-10-09T09:27:40.673720+0000","last_fullsized":"2023-10-09T09:27:40.673720+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:31.372332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"391'176","reported_seq":833,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.555204+0000","last_change":"2023-10-09T09:24:22.951420+0000","last_active":"2023-10-09T09:27:40.555204+0000","last_peered":"2023-10-09T09:27:40.555204+0000","last_clean":"2023-10-09T09:27:40.555204+0000","last_became_active":"2023-10-09T09:14:48.316910+0000","last_became_peered":"2023-10-09T09:14:48.316910+0000","last_unstale":"2023-10-09T09:27:40.555204+0000","last_undegraded":"2023-10-09T09:27:40.555204+0000","last_fullsized":"2023-10-09T09:27:40.555204+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":176,"log_dups_size":0,"ondisk_log_size":176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:01.455754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084346699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1202,"num_read_kb":2193,"num_write":134,"num_write_kb":1170,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":182,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.672886+0000","last_change":"2023-10-09T09:26:19.493412+0000","last_active":"2023-10-09T09:27:40.672886+0000","last_peered":"2023-10-09T09:27:40.672886+0000","last_clean":"2023-10-09T09:27:40.672886+0000","last_became_active":"2023-10-09T09:26:19.492949+0000","last_became_peered":"2023-10-09T09:26:19.492949+0000","last_unstale":"2023-10-09T09:27:40.672886+0000","last_undegraded":"2023-10-09T09:27:40.672886+0000","last_fullsized":"2023-10-09T09:27:40.672886+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:02.951627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"391'91","reported_seq":718,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549943+0000","last_change":"2023-10-09T09:27:30.535381+0000","last_active":"2023-10-09T09:27:39.549943+0000","last_peered":"2023-10-09T09:27:39.549943+0000","last_clean":"2023-10-09T09:27:39.549943+0000","last_became_active":"2023-10-09T09:27:30.521550+0000","last_became_peered":"2023-10-09T09:27:30.521550+0000","last_unstale":"2023-10-09T09:27:39.549943+0000","last_undegraded":"2023-10-09T09:27:39.549943+0000","last_fullsized":"2023-10-09T09:27:39.549943+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:32:36.662932+0000","last_clean_scrub_stamp":"2023-10-09T08:32:36.662932+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":572,"num_read_kb":856,"num_write":63,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":472,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562402+0000","last_change":"2023-10-09T09:14:48.325227+0000","last_active":"2023-10-09T09:27:40.562402+0000","last_peered":"2023-10-09T09:27:40.562402+0000","last_clean":"2023-10-09T09:27:40.562402+0000","last_became_active":"2023-10-09T09:14:48.306631+0000","last_became_peered":"2023-10-09T09:14:48.306631+0000","last_unstale":"2023-10-09T09:27:40.562402+0000","last_undegraded":"2023-10-09T09:27:40.562402+0000","last_fullsized":"2023-10-09T09:27:40.562402+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:32.922943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":294,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547584+0000","last_change":"2023-10-09T09:08:24.652220+0000","last_active":"2023-10-09T09:27:39.547584+0000","last_peered":"2023-10-09T09:27:39.547584+0000","last_clean":"2023-10-09T09:27:39.547584+0000","last_became_active":"2023-10-09T09:08:24.650724+0000","last_became_peered":"2023-10-09T09:08:24.650724+0000","last_unstale":"2023-10-09T09:27:39.547584+0000","last_undegraded":"2023-10-09T09:27:39.547584+0000","last_fullsized":"2023-10-09T09:27:39.547584+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":439,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.562200+0000","last_change":"2023-10-09T09:09:44.378521+0000","last_active":"2023-10-09T09:27:40.562200+0000","last_peered":"2023-10-09T09:27:40.562200+0000","last_clean":"2023-10-09T09:27:40.562200+0000","last_became_active":"2023-10-09T09:09:44.378187+0000","last_became_peered":"2023-10-09T09:09:44.378187+0000","last_unstale":"2023-10-09T09:27:40.562200+0000","last_undegraded":"2023-10-09T09:27:40.562200+0000","last_fullsized":"2023-10-09T09:27:40.562200+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:57.134467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":120,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.553681+0000","last_change":"2023-10-09T09:20:19.916151+0000","last_active":"2023-10-09T09:27:40.553681+0000","last_peered":"2023-10-09T09:27:40.553681+0000","last_clean":"2023-10-09T09:27:40.553681+0000","last_became_active":"2023-10-09T09:20:19.915607+0000","last_became_peered":"2023-10-09T09:20:19.915607+0000","last_unstale":"2023-10-09T09:27:40.553681+0000","last_undegraded":"2023-10-09T09:27:40.553681+0000","last_fullsized":"2023-10-09T09:27:40.553681+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:57:51.697949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":218,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.549830+0000","last_change":"2023-10-09T09:14:48.401041+0000","last_active":"2023-10-09T09:27:39.549830+0000","last_peered":"2023-10-09T09:27:39.549830+0000","last_clean":"2023-10-09T09:27:39.549830+0000","last_became_active":"2023-10-09T09:14:48.319357+0000","last_became_peered":"2023-10-09T09:14:48.319357+0000","last_unstale":"2023-10-09T09:27:39.549830+0000","last_undegraded":"2023-10-09T09:27:39.549830+0000","last_fullsized":"2023-10-09T09:27:39.549830+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":294,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547932+0000","last_change":"2023-10-09T09:14:48.363991+0000","last_active":"2023-10-09T09:27:39.547932+0000","last_peered":"2023-10-09T09:27:39.547932+0000","last_clean":"2023-10-09T09:27:39.547932+0000","last_became_active":"2023-10-09T09:14:48.363296+0000","last_became_peered":"2023-10-09T09:14:48.363296+0000","last_unstale":"2023-10-09T09:27:39.547932+0000","last_undegraded":"2023-10-09T09:27:39.547932+0000","last_fullsized":"2023-10-09T09:27:39.547932+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":205,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.550063+0000","last_change":"2023-10-09T09:08:24.659385+0000","last_active":"2023-10-09T09:27:39.550063+0000","last_peered":"2023-10-09T09:27:39.550063+0000","last_clean":"2023-10-09T09:27:39.550063+0000","last_became_active":"2023-10-09T09:08:24.655109+0000","last_became_peered":"2023-10-09T09:08:24.655109+0000","last_unstale":"2023-10-09T09:27:39.550063+0000","last_undegraded":"2023-10-09T09:27:39.550063+0000","last_fullsized":"2023-10-09T09:27:39.550063+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":276,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.547511+0000","last_change":"2023-10-09T09:11:52.434484+0000","last_active":"2023-10-09T09:27:39.547511+0000","last_peered":"2023-10-09T09:27:39.547511+0000","last_clean":"2023-10-09T09:27:39.547511+0000","last_became_active":"2023-10-09T09:11:52.433520+0000","last_became_peered":"2023-10-09T09:11:52.433520+0000","last_unstale":"2023-10-09T09:27:39.547511+0000","last_undegraded":"2023-10-09T09:27:39.547511+0000","last_fullsized":"2023-10-09T09:27:39.547511+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":249,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673691+0000","last_change":"2023-10-09T09:26:19.491307+0000","last_active":"2023-10-09T09:27:40.673691+0000","last_peered":"2023-10-09T09:27:40.673691+0000","last_clean":"2023-10-09T09:27:40.673691+0000","last_became_active":"2023-10-09T09:26:19.490921+0000","last_became_peered":"2023-10-09T09:26:19.490921+0000","last_unstale":"2023-10-09T09:27:40.673691+0000","last_undegraded":"2023-10-09T09:27:40.673691+0000","last_fullsized":"2023-10-09T09:27:40.673691+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:20.144589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":260,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T09:27:40.673278+0000","last_change":"2023-10-09T09:26:19.488134+0000","last_active":"2023-10-09T09:27:40.673278+0000","last_peered":"2023-10-09T09:27:40.673278+0000","last_clean":"2023-10-09T09:27:40.673278+0000","last_became_active":"2023-10-09T09:26:19.487724+0000","last_became_peered":"2023-10-09T09:26:19.487724+0000","last_unstale":"2023-10-09T09:27:40.673278+0000","last_undegraded":"2023-10-09T09:27:40.673278+0000","last_fullsized":"2023-10-09T09:27:40.673278+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:25.687144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":559,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.536731+0000","last_change":"2023-10-09T09:20:19.788084+0000","last_active":"2023-10-09T09:27:39.536731+0000","last_peered":"2023-10-09T09:27:39.536731+0000","last_clean":"2023-10-09T09:27:39.536731+0000","last_became_active":"2023-10-09T09:20:19.762112+0000","last_became_peered":"2023-10-09T09:20:19.762112+0000","last_unstale":"2023-10-09T09:27:39.536731+0000","last_undegraded":"2023-10-09T09:27:39.536731+0000","last_fullsized":"2023-10-09T09:27:39.536731+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:26.695862+0000","last_clean_scrub_stamp":"2023-10-09T08:34:26.695862+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:47.406781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":303,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-10-09T09:27:39.546810+0000","last_change":"2023-10-09T09:14:48.321121+0000","last_active":"2023-10-09T09:27:39.546810+0000","last_peered":"2023-10-09T09:27:39.546810+0000","last_clean":"2023-10-09T09:27:39.546810+0000","last_became_active":"2023-10-09T09:14:48.318972+0000","last_became_peered":"2023-10-09T09:14:48.318972+0000","last_unstale":"2023-10-09T09:27:39.546810+0000","last_undegraded":"2023-10-09T09:27:39.546810+0000","last_fullsized":"2023-10-09T09:27:39.546810+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:34:27.704437+0000","last_clean_scrub_stamp":"2023-10-09T08:34:27.704437+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":20760,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12224,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430905,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907472,"kb_used_data":4168,"kb_used_omap":0,"kb_used_meta":903296,"kb_avail":92837680,"statfs":{"total":95995035648,"available":95065784320,"internally_reserved":0,"allocated":4268032,"data_stored":3617173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924975104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97399999999999998,"15min":1.766},"min":{"1min":0.68799999999999994,"5min":0.39800000000000002,"15min":0.23499999999999999},"max":{"1min":1.319,"5min":1.464,"15min":51.859999999999999},"last":1.133},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.96399999999999997,"15min":1.784},"min":{"1min":0.63100000000000001,"5min":0.503,"15min":0.28899999999999998},"max":{"1min":1.2350000000000001,"5min":1.5369999999999999,"15min":51.957999999999998},"last":1.153}]},{"osd":1,"last update":"Mon Oct 9 09:28:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":2.7810000000000001,"15min":2.153},"min":{"1min":0.61699999999999999,"5min":0.55200000000000005,"15min":0.27400000000000002},"max":{"1min":1.2769999999999999,"5min":142.703,"15min":142.703},"last":1.054},{"interface":"front","average":{"1min":0.97299999999999998,"5min":2.762,"15min":2.2000000000000002},"min":{"1min":0.66200000000000003,"5min":0.628,"15min":0.316},"max":{"1min":1.3200000000000001,"5min":142.67500000000001,"15min":142.67500000000001},"last":1.2370000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:28:52 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.94199999999999995,"15min":1.486},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.32800000000000001},"max":{"1min":1.147,"5min":1.2529999999999999,"15min":51.914000000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.97599999999999998,"15min":1.3959999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.33200000000000002},"max":{"1min":1.3380000000000001,"5min":1.383,"15min":51.902999999999999},"last":1.2070000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:28:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.93300000000000005,"15min":1.379},"min":{"1min":0.68999999999999995,"5min":0.65000000000000002,"15min":0.374},"max":{"1min":1.159,"5min":1.506,"15min":18.178999999999998},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0629999999999999,"15min":1.3129999999999999},"min":{"1min":0.61899999999999999,"5min":0.54900000000000004,"15min":0.31},"max":{"1min":1.349,"5min":1.5149999999999999,"15min":17.931999999999999},"last":1.2869999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.96299999999999997,"15min":1.325},"min":{"1min":0.70199999999999996,"5min":0.52400000000000002,"15min":0.28699999999999998},"max":{"1min":1.335,"5min":1.3500000000000001,"15min":11.371},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.0449999999999999,"15min":1.3129999999999999},"min":{"1min":0.84799999999999998,"5min":0.52600000000000002,"15min":0.30599999999999999},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":8.9879999999999995},"last":0.79200000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":0.999,"15min":1.286},"min":{"1min":0.80200000000000005,"5min":0.48099999999999998,"15min":0.32200000000000001},"max":{"1min":1.242,"5min":1.395,"15min":15.616},"last":1.077},{"interface":"front","average":{"1min":1.119,"5min":1.081,"15min":1.5129999999999999},"min":{"1min":0.88200000000000001,"5min":0.57399999999999995,"15min":0.36599999999999999},"max":{"1min":1.3,"5min":1.494,"15min":18.501000000000001},"last":0.92900000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.1020000000000001,"15min":1.5209999999999999},"min":{"1min":0.82199999999999995,"5min":0.65400000000000003,"15min":0.34899999999999998},"max":{"1min":1.391,"5min":1.5669999999999999,"15min":21.145},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.2130000000000001,"15min":1.5189999999999999},"min":{"1min":0.92000000000000004,"5min":0.45800000000000002,"15min":0.34999999999999998},"max":{"1min":1.573,"5min":1.573,"15min":12.269},"last":0.98299999999999998}]}]},{"osd":6,"up_from":42,"seq":180388627138,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515464,"kb_used_data":8256,"kb_used_omap":0,"kb_used_meta":1507200,"kb_avail":92229688,"statfs":{"total":95995035648,"available":94443200512,"internally_reserved":0,"allocated":8454144,"data_stored":7811300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1543372800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.98199999999999998,"15min":2.7629999999999999},"min":{"1min":0.51700000000000002,"5min":0.48899999999999999,"15min":0.26100000000000001},"max":{"1min":1.3759999999999999,"5min":1.4690000000000001,"15min":120.812},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0580000000000001,"15min":2.6499999999999999},"min":{"1min":0.69699999999999995,"5min":0.68700000000000006,"15min":0.25600000000000001},"max":{"1min":1.4339999999999999,"5min":1.853,"15min":120.857},"last":1.244}]},{"osd":1,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.6180000000000001,"15min":2.4159999999999999},"min":{"1min":0.60799999999999998,"5min":0.48599999999999999,"15min":0.28100000000000003},"max":{"1min":1.9470000000000001,"5min":59.509,"15min":142.31100000000001},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.651,"15min":2.367},"min":{"1min":0.63300000000000001,"5min":0.53100000000000003,"15min":0.25700000000000001},"max":{"1min":1.9139999999999999,"5min":59.482999999999997,"15min":132.48099999999999},"last":1.077}]},{"osd":2,"last update":"Mon Oct 9 09:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.95299999999999996,"15min":1.6890000000000001},"min":{"1min":0.72499999999999998,"5min":0.49399999999999999,"15min":0.28999999999999998},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":82.170000000000002},"last":0.997},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.026,"15min":1.611},"min":{"1min":0.755,"5min":0.63,"15min":0.28699999999999998},"max":{"1min":1.581,"5min":1.581,"15min":82.352000000000004},"last":1.1739999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.94099999999999995,"15min":1.548},"min":{"1min":0.751,"5min":0.59099999999999997,"15min":0.33700000000000002},"max":{"1min":1.3460000000000001,"5min":1.8140000000000001,"15min":14.525},"last":1.048},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.96299999999999997,"15min":1.216},"min":{"1min":0.70399999999999996,"5min":0.629,"15min":0.32100000000000001},"max":{"1min":1.2849999999999999,"5min":1.5,"15min":9.0839999999999996},"last":1.0349999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:28:19 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.006,"15min":1.534},"min":{"1min":0.68500000000000005,"5min":0.45700000000000002,"15min":0.27600000000000002},"max":{"1min":1.444,"5min":2.1389999999999998,"15min":29.585999999999999},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.98399999999999999,"15min":1.4410000000000001},"min":{"1min":0.69899999999999995,"5min":0.38900000000000001,"15min":0.316},"max":{"1min":1.383,"5min":1.7769999999999999,"15min":20.318000000000001},"last":0.89400000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:28:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.052,"15min":1.4610000000000001},"min":{"1min":0.75700000000000001,"5min":0.53900000000000003,"15min":0.312},"max":{"1min":2.1549999999999998,"5min":2.1549999999999998,"15min":18.103000000000002},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.131,"5min":1.0580000000000001,"15min":1.401},"min":{"1min":0.77900000000000003,"5min":0.55600000000000005,"15min":0.33300000000000002},"max":{"1min":2.629,"5min":2.629,"15min":11.567},"last":1.1200000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.1559999999999999,"15min":1.5840000000000001},"min":{"1min":0.91500000000000004,"5min":0.55100000000000005,"15min":0.33700000000000002},"max":{"1min":1.994,"5min":1.994,"15min":29.530000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.2,"5min":1.111,"15min":1.4299999999999999},"min":{"1min":0.98499999999999999,"5min":0.497,"15min":0.32900000000000001},"max":{"1min":1.821,"5min":1.8779999999999999,"15min":11.477},"last":0.97599999999999998}]}]},{"osd":1,"up_from":83,"seq":356482286086,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":656324,"kb_used_data":8572,"kb_used_omap":9,"kb_used_meta":647734,"kb_avail":93088828,"statfs":{"total":95995035648,"available":95322959872,"internally_reserved":0,"allocated":8777728,"data_stored":7324979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":663280617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.70199999999999996,"15min":1.9790000000000001},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.215},"max":{"1min":0.83099999999999996,"5min":1.569,"15min":133.76499999999999},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.76100000000000001,"15min":1.7330000000000001},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.22900000000000001},"max":{"1min":1.0389999999999999,"5min":1.381,"15min":127.395},"last":0.93200000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.78000000000000003,"15min":1.7729999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.26500000000000001},"max":{"1min":1.1659999999999999,"5min":1.3240000000000001,"15min":236.29599999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.78700000000000003,"15min":1.73},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.27500000000000002},"max":{"1min":1.4510000000000001,"5min":1.4910000000000001,"15min":236.21000000000001},"last":1.2170000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.89100000000000001,"15min":1.744},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.308},"max":{"1min":1.165,"5min":1.486,"15min":152.637},"last":0.97699999999999998},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.85699999999999998,"15min":2.306},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.25800000000000001},"max":{"1min":1.0509999999999999,"5min":1.421,"15min":152.374},"last":1.163}]},{"osd":4,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.99399999999999999,"15min":1.462},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.38},"max":{"1min":1.2509999999999999,"5min":1.6830000000000001,"15min":14.56},"last":1.127},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.008,"15min":1.8779999999999999},"min":{"1min":0.63100000000000001,"5min":0.55800000000000005,"15min":0.33200000000000002},"max":{"1min":1.349,"5min":1.792,"15min":133.702},"last":1.3440000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0640000000000001,"15min":1.298},"min":{"1min":0.67400000000000004,"5min":0.61299999999999999,"15min":0.34699999999999998},"max":{"1min":1.347,"5min":1.768,"15min":9.4290000000000003},"last":1.3740000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.119,"15min":1.3129999999999999},"min":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.35199999999999998},"max":{"1min":1.4199999999999999,"5min":1.8819999999999999,"15min":14.464},"last":1.3160000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.149,"15min":1.984},"min":{"1min":0.70199999999999996,"5min":0.60399999999999998,"15min":0.38300000000000001},"max":{"1min":1.5669999999999999,"5min":1.9179999999999999,"15min":133.739},"last":1.534},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.155,"15min":1.4059999999999999},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.377},"max":{"1min":1.609,"5min":1.954,"15min":9.4619999999999997},"last":1.504}]},{"osd":7,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.1579999999999999,"15min":1.518},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.41499999999999998},"max":{"1min":1.3120000000000001,"5min":1.9870000000000001,"15min":19.861999999999998},"last":1.4039999999999999},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.1419999999999999,"15min":1.9610000000000001},"min":{"1min":0.71899999999999997,"5min":0.66200000000000003,"15min":0.40899999999999997},"max":{"1min":1.5189999999999999,"5min":1.784,"15min":133.66499999999999},"last":1.4730000000000001}]}]},{"osd":0,"up_from":78,"seq":335007449612,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":993368,"kb_used_data":5072,"kb_used_omap":28,"kb_used_meta":988259,"kb_avail":92751784,"statfs":{"total":95995035648,"available":94977826816,"internally_reserved":0,"allocated":5193728,"data_stored":3722137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1011978064},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:28:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.73799999999999999,"15min":1.347},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.19700000000000001},"max":{"1min":1.006,"5min":3.0139999999999998,"15min":83.671999999999997},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.76500000000000001,"5min":0.79000000000000004,"15min":1.3560000000000001},"min":{"1min":0.34799999999999998,"5min":0.29599999999999999,"15min":0.20300000000000001},"max":{"1min":1.1850000000000001,"5min":3.052,"15min":83.953000000000003},"last":0.90000000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.85099999999999998,"15min":1.472},"min":{"1min":0.56899999999999995,"5min":0.36899999999999999,"15min":0.26800000000000002},"max":{"1min":1.3959999999999999,"5min":2.9129999999999998,"15min":74.200999999999993},"last":0.87},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.96099999999999997,"15min":1.524},"min":{"1min":0.68600000000000005,"5min":0.46000000000000002,"15min":0.26800000000000002},"max":{"1min":1.427,"5min":1.7849999999999999,"15min":73.983999999999995},"last":0.79800000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.89500000000000002,"15min":1.29},"min":{"1min":0.45900000000000002,"5min":0.33100000000000002,"15min":0.253},"max":{"1min":1.198,"5min":2.9460000000000002,"15min":9.8239999999999998},"last":1.032},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.91800000000000004,"15min":1.171},"min":{"1min":0.438,"5min":0.40699999999999997,"15min":0.29399999999999998},"max":{"1min":1.2569999999999999,"5min":1.716,"15min":9.9269999999999996},"last":0.68799999999999994}]},{"osd":4,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.095,"15min":1.522},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.34000000000000002},"max":{"1min":1.597,"5min":1.597,"15min":14.163},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.96099999999999997,"15min":1.5640000000000001},"min":{"1min":0.66400000000000003,"5min":0.54300000000000004,"15min":0.314},"max":{"1min":1.2210000000000001,"5min":1.4139999999999999,"15min":11.346},"last":0.95799999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.121,"15min":1.4330000000000001},"min":{"1min":0.61599999999999999,"5min":0.57299999999999995,"15min":0.316},"max":{"1min":1.5289999999999999,"5min":2.988,"15min":9.9770000000000003},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.1850000000000001,"15min":1.3460000000000001},"min":{"1min":0.77200000000000002,"5min":0.66500000000000004,"15min":0.35499999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":9.5909999999999993},"last":0.99399999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.204,"15min":1.605},"min":{"1min":0.83699999999999997,"5min":0.53300000000000003,"15min":0.34799999999999998},"max":{"1min":1.6379999999999999,"5min":2.968,"15min":11.314},"last":1.165},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.254,"15min":1.6879999999999999},"min":{"1min":0.85799999999999998,"5min":0.70899999999999996,"15min":0.39400000000000002},"max":{"1min":1.8400000000000001,"5min":1.8400000000000001,"15min":16.911000000000001},"last":1.3089999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.28,"15min":1.4990000000000001},"min":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.41699999999999998},"max":{"1min":1.79,"5min":1.79,"15min":9.6780000000000008},"last":1.208},{"interface":"front","average":{"1min":1.113,"5min":1.0880000000000001,"15min":1.6719999999999999},"min":{"1min":0.82199999999999995,"5min":0.623,"15min":0.35899999999999999},"max":{"1min":1.26,"5min":1.4330000000000001,"15min":10.734},"last":1.2430000000000001}]}]},{"osd":2,"up_from":88,"seq":377957122560,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":971160,"kb_used_data":4496,"kb_used_omap":81,"kb_used_meta":966574,"kb_avail":92773992,"statfs":{"total":95995035648,"available":95000567808,"internally_reserved":0,"allocated":4603904,"data_stored":3130325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":989771994},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.71299999999999997,"15min":1.419},"min":{"1min":0.56100000000000005,"5min":0.35599999999999998,"15min":0.23899999999999999},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":83.421999999999997},"last":1.0660000000000001},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.79800000000000004,"15min":1.3400000000000001},"min":{"1min":0.38500000000000001,"5min":0.36699999999999999,"15min":0.23499999999999999},"max":{"1min":1.085,"5min":1.3999999999999999,"15min":83.373999999999995},"last":0.66400000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.82399999999999995,"15min":1.7410000000000001},"min":{"1min":0.49399999999999999,"5min":0.38400000000000001,"15min":0.245},"max":{"1min":1.0760000000000001,"5min":2.1840000000000002,"15min":122.367},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.878,"15min":1.7050000000000001},"min":{"1min":0.46000000000000002,"5min":0.35699999999999998,"15min":0.27800000000000002},"max":{"1min":1.262,"5min":3.5680000000000001,"15min":122.777},"last":0.79600000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.87,"15min":2.1549999999999998},"min":{"1min":0.51300000000000001,"5min":0.42699999999999999,"15min":0.26500000000000001},"max":{"1min":1.1439999999999999,"5min":1.456,"15min":152.29300000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.79700000000000004,"15min":2.5339999999999998},"min":{"1min":0.56200000000000006,"5min":0.41599999999999998,"15min":0.27100000000000002},"max":{"1min":1.073,"5min":1.9259999999999999,"15min":153.70400000000001},"last":1.409}]},{"osd":4,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0029999999999999,"15min":1.2889999999999999},"min":{"1min":0.78100000000000003,"5min":0.64400000000000002,"15min":0.32600000000000001},"max":{"1min":1.631,"5min":1.6850000000000001,"15min":14.343999999999999},"last":1.349},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.089,"15min":1.4259999999999999},"min":{"1min":0.749,"5min":0.60899999999999999,"15min":0.35799999999999998},"max":{"1min":1.4890000000000001,"5min":3.7090000000000001,"15min":12.351000000000001},"last":1.232}]},{"osd":5,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.1299999999999999,"15min":1.228},"min":{"1min":0.81200000000000006,"5min":0.624,"15min":0.35999999999999999},"max":{"1min":1.5469999999999999,"5min":3.9079999999999999,"15min":10.135},"last":1.2909999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.0629999999999999,"15min":1.369},"min":{"1min":0.81699999999999995,"5min":0.55900000000000005,"15min":0.371},"max":{"1min":1.8460000000000001,"5min":2.2189999999999999,"15min":13.303000000000001},"last":1.583}]},{"osd":6,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.151,"15min":1.6080000000000001},"min":{"1min":0.78500000000000003,"5min":0.64500000000000002,"15min":0.35899999999999999},"max":{"1min":1.7090000000000001,"5min":1.7090000000000001,"15min":14.714},"last":1.552},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.2310000000000001,"15min":1.629},"min":{"1min":0.83599999999999997,"5min":0.69299999999999995,"15min":0.432},"max":{"1min":1.6259999999999999,"5min":5.9130000000000003,"15min":14.571999999999999},"last":1.202}]},{"osd":7,"last update":"Mon Oct 9 09:28:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.238,"15min":1.5509999999999999},"min":{"1min":0.82199999999999995,"5min":0.67600000000000005,"15min":0.41599999999999998},"max":{"1min":1.6499999999999999,"5min":5.9500000000000002,"15min":29.513999999999999},"last":1.8759999999999999},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.206,"15min":1.4259999999999999},"min":{"1min":0.90300000000000002,"5min":0.65900000000000003,"15min":0.40200000000000002},"max":{"1min":1.738,"5min":1.883,"15min":30.216999999999999},"last":1.337}]}]},{"osd":3,"up_from":93,"seq":399431959034,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":973308,"kb_used_data":4468,"kb_used_omap":7,"kb_used_meta":968824,"kb_avail":92771844,"statfs":{"total":95995035648,"available":94998368256,"internally_reserved":0,"allocated":4575232,"data_stored":3129376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":992076392},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.72299999999999998,"15min":1.4750000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.20499999999999999},"max":{"1min":0.93500000000000005,"5min":1.228,"15min":46.433999999999997},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.78800000000000003,"15min":1.5800000000000001},"min":{"1min":0.40999999999999998,"5min":0.39300000000000002,"15min":0.20100000000000001},"max":{"1min":1.208,"5min":1.246,"15min":46.387},"last":1.131}]},{"osd":1,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.81200000000000006,"15min":1.2729999999999999},"min":{"1min":0.55200000000000005,"5min":0.376,"15min":0.23599999999999999},"max":{"1min":1.4259999999999999,"5min":1.429,"15min":32.520000000000003},"last":1.0920000000000001},{"interface":"front","average":{"1min":0.874,"5min":0.876,"15min":1.1919999999999999},"min":{"1min":0.44,"5min":0.313,"15min":0.23100000000000001},"max":{"1min":1.4490000000000001,"5min":1.4750000000000001,"15min":32.585999999999999},"last":1.006}]},{"osd":2,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.77800000000000002,"15min":1.1910000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.26900000000000002},"max":{"1min":1.1699999999999999,"5min":1.444,"15min":37.997},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.87,"5min":0.84699999999999998,"15min":1.236},"min":{"1min":0.44,"5min":0.39700000000000002,"15min":0.254},"max":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":37.499000000000002},"last":1.1719999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.022,"15min":1.3169999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.316},"max":{"1min":1.236,"5min":1.6120000000000001,"15min":10.445},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.0029999999999999,"15min":1.4259999999999999},"min":{"1min":0.61499999999999999,"5min":0.60599999999999998,"15min":0.33300000000000002},"max":{"1min":1.2390000000000001,"5min":1.8089999999999999,"15min":12.617000000000001},"last":0.94899999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.02,"15min":1.383},"min":{"1min":0.59099999999999997,"5min":0.53900000000000003,"15min":0.34100000000000003},"max":{"1min":1.298,"5min":1.601,"15min":8.5540000000000003},"last":1.2250000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0640000000000001,"15min":1.4019999999999999},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.36899999999999999},"max":{"1min":1.363,"5min":1.8300000000000001,"15min":9.5210000000000008},"last":1.2470000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.1060000000000001,"15min":1.6060000000000001},"min":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":0.37},"max":{"1min":1.393,"5min":1.8700000000000001,"15min":14.124000000000001},"last":1.2909999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.1619999999999999,"15min":1.595},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.38100000000000001},"max":{"1min":1.4770000000000001,"5min":2.0259999999999998,"15min":13.778},"last":1.353}]},{"osd":7,"last update":"Mon Oct 9 09:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.1499999999999999,"15min":1.4430000000000001},"min":{"1min":0.79000000000000004,"5min":0.59499999999999997,"15min":0.42699999999999999},"max":{"1min":1.4930000000000001,"5min":1.7150000000000001,"15min":9.673},"last":1.407},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.181,"15min":1.399},"min":{"1min":0.83299999999999996,"5min":0.748,"15min":0.377},"max":{"1min":1.3680000000000001,"5min":1.944,"15min":9.7850000000000001},"last":1.3129999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019602,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":893580,"kb_used_data":3588,"kb_used_omap":0,"kb_used_meta":889984,"kb_avail":92851572,"statfs":{"total":95995035648,"available":95080009728,"internally_reserved":0,"allocated":3674112,"data_stored":3027000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911343616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.89200000000000002,"15min":1.446},"min":{"1min":0.505,"5min":0.47499999999999998,"15min":0.247},"max":{"1min":1.1319999999999999,"5min":1.268,"15min":23.32},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.94199999999999995,"15min":1.323},"min":{"1min":0.57299999999999995,"5min":0.33800000000000002,"15min":0.24199999999999999},"max":{"1min":1.288,"5min":1.6180000000000001,"15min":23.581},"last":0.79800000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.88600000000000001,"15min":1.3839999999999999},"min":{"1min":0.61899999999999999,"5min":0.49299999999999999,"15min":0.27700000000000002},"max":{"1min":1.371,"5min":1.5129999999999999,"15min":37.481999999999999},"last":1.218},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98099999999999998,"15min":1.5880000000000001},"min":{"1min":0.626,"5min":0.55600000000000005,"15min":0.28100000000000003},"max":{"1min":1.3149999999999999,"5min":1.466,"15min":37.332999999999998},"last":0.748}]},{"osd":2,"last update":"Mon Oct 9 09:29:09 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.0409999999999999,"15min":1.373},"min":{"1min":0.79300000000000004,"5min":0.53500000000000003,"15min":0.32200000000000001},"max":{"1min":1.3560000000000001,"5min":1.4139999999999999,"15min":11.358000000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.002,"15min":1.2310000000000001},"min":{"1min":0.79800000000000004,"5min":0.50900000000000001,"15min":0.29499999999999998},"max":{"1min":1.6659999999999999,"5min":1.6659999999999999,"15min":11.1},"last":1.071}]},{"osd":3,"last update":"Mon Oct 9 09:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.95399999999999996,"15min":1.5489999999999999},"min":{"1min":0.73799999999999999,"5min":0.62,"15min":0.29999999999999999},"max":{"1min":1.208,"5min":1.45,"15min":55.299999999999997},"last":1.0209999999999999},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.042,"15min":1.659},"min":{"1min":0.79700000000000004,"5min":0.60699999999999998,"15min":0.30399999999999999},"max":{"1min":1.4119999999999999,"5min":1.544,"15min":55.622},"last":1.135}]},{"osd":5,"last update":"Mon Oct 9 09:28:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0149999999999999,"15min":1.52},"min":{"1min":0.74399999999999999,"5min":0.64300000000000002,"15min":0.314},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":13.907},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.103,"5min":1.042,"15min":1.2929999999999999},"min":{"1min":0.77400000000000002,"5min":0.61699999999999999,"15min":0.33300000000000002},"max":{"1min":1.837,"5min":1.837,"15min":11.042999999999999},"last":0.91200000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.012,"15min":1.5229999999999999},"min":{"1min":0.78300000000000003,"5min":0.45100000000000001,"15min":0.29999999999999999},"max":{"1min":1.3500000000000001,"5min":1.5149999999999999,"15min":18.103999999999999},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.141,"5min":1.1399999999999999,"15min":1.679},"min":{"1min":0.79200000000000004,"5min":0.42799999999999999,"15min":0.34000000000000002},"max":{"1min":1.46,"5min":1.637,"15min":13.444000000000001},"last":1.0449999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:29:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.149,"15min":1.5329999999999999},"min":{"1min":0.93100000000000005,"5min":0.61699999999999999,"15min":0.33000000000000002},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":15.484999999999999},"last":0.97799999999999998},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1240000000000001,"15min":1.409},"min":{"1min":0.85899999999999999,"5min":0.54600000000000004,"15min":0.33900000000000002},"max":{"1min":1.4430000000000001,"5min":1.577,"15min":12.003},"last":1.161}]}]},{"osd":5,"up_from":36,"seq":154618823372,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917184,"kb_used_data":7672,"kb_used_omap":0,"kb_used_meta":909504,"kb_avail":92827968,"statfs":{"total":95995035648,"available":95055839232,"internally_reserved":0,"allocated":7856128,"data_stored":7217067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931332096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.89700000000000002,"15min":2.0150000000000001},"min":{"1min":0.67900000000000005,"5min":0.44400000000000001,"15min":0.24299999999999999},"max":{"1min":1.2769999999999999,"5min":1.5629999999999999,"15min":139.81700000000001},"last":1.107},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.90600000000000003,"15min":2.137},"min":{"1min":0.63800000000000001,"5min":0.55100000000000005,"15min":0.251},"max":{"1min":1.2909999999999999,"5min":1.323,"15min":141.66900000000001},"last":1.234}]},{"osd":1,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.91200000000000003,"15min":1.6220000000000001},"min":{"1min":0.60099999999999998,"5min":0.49399999999999999,"15min":0.26800000000000002},"max":{"1min":1.246,"5min":1.486,"15min":36.609999999999999},"last":1.214},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.89600000000000002,"15min":1.5740000000000001},"min":{"1min":0.745,"5min":0.55200000000000005,"15min":0.29699999999999999},"max":{"1min":1.333,"5min":1.333,"15min":36.856999999999999},"last":0.94199999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.92500000000000004,"15min":1.458},"min":{"1min":0.69299999999999995,"5min":0.58399999999999996,"15min":0.29299999999999998},"max":{"1min":1.3700000000000001,"5min":1.488,"15min":46.058999999999997},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.006,"5min":1.0029999999999999,"15min":1.585},"min":{"1min":0.71099999999999997,"5min":0.45200000000000001,"15min":0.28599999999999998},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":46.030000000000001},"last":1.1459999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.97099999999999997,"15min":1.389},"min":{"1min":0.61599999999999999,"5min":0.57099999999999995,"15min":0.32000000000000001},"max":{"1min":1.181,"5min":1.3080000000000001,"15min":11.427},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.98699999999999999,"15min":1.3120000000000001},"min":{"1min":0.748,"5min":0.64700000000000002,"15min":0.33400000000000002},"max":{"1min":1.252,"5min":1.448,"15min":11.019},"last":1.034}]},{"osd":4,"last update":"Mon Oct 9 09:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.95799999999999996,"15min":1.4570000000000001},"min":{"1min":0.68600000000000005,"5min":0.48699999999999999,"15min":0.26600000000000001},"max":{"1min":1.4570000000000001,"5min":1.4570000000000001,"15min":13.904},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0249999999999999,"15min":1.5800000000000001},"min":{"1min":0.65900000000000003,"5min":0.504,"15min":0.36899999999999999},"max":{"1min":1.294,"5min":1.518,"15min":22.228000000000002},"last":1.004}]},{"osd":6,"last update":"Mon Oct 9 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0049999999999999,"15min":1.4370000000000001},"min":{"1min":0.622,"5min":0.56200000000000006,"15min":0.32000000000000001},"max":{"1min":1.4430000000000001,"5min":1.4430000000000001,"15min":53.840000000000003},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.1220000000000001,"15min":1.675},"min":{"1min":0.745,"5min":0.50800000000000001,"15min":0.29999999999999999},"max":{"1min":1.407,"5min":1.407,"15min":50.404000000000003},"last":1.0780000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:29:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.1259999999999999,"15min":1.419},"min":{"1min":0.82299999999999995,"5min":0.55800000000000005,"15min":0.36099999999999999},"max":{"1min":1.3999999999999999,"5min":1.458,"15min":12.159000000000001},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.151,"5min":1.1060000000000001,"15min":1.393},"min":{"1min":0.84499999999999997,"5min":0.53500000000000003,"15min":0.38600000000000001},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":11.331},"last":1.179}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:29:16.417 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T09:29:16.418 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T09:29:16.485 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=813ca7a19a004579be8710c5e5b9edb1 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T09:29:16.487 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T09:29:16.498 INFO:tasks.cephadm:Teardown begin 2023-10-09T09:29:16.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T09:29:16.534 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T09:29:16.637 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T09:29:16.637 DEBUG:teuthology.orchestra.run.smithi016:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T09:29:16.656 DEBUG:teuthology.orchestra.run.smithi094:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T09:29:16.663 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T09:29:16.663 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T09:29:16.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T09:29:16.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[75179]: audit 2023-10-09T09:29:15.401518+0000 mgr.y (mgr.25421) 1566 : audit 0 from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:16.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[77892]: audit 2023-10-09T09:29:15.401518+0000 mgr.y (mgr.25421) 1566 : audit 0 from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:16.990 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:16 smithi094 bash[59677]: audit 2023-10-09T09:29:15.401518+0000 mgr.y (mgr.25421) 1566 : audit 0 from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:17.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[71225]: [09/Oct/2023:09:29:16] ENGINE Bus STOPPING 2023-10-09T09:29:17.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[71225]: [09/Oct/2023:09:29:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:29:17.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[71225]: [09/Oct/2023:09:29:16] ENGINE Bus STOPPED 2023-10-09T09:29:17.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[71225]: [09/Oct/2023:09:29:16] ENGINE Bus STARTING 2023-10-09T09:29:17.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:16 smithi016 systemd[1]: Stopping Ceph mon.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:17.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[75179]: debug 2023-10-09T09:29:16.910+0000 7fa04c258700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:29:17.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:16 smithi016 bash[75179]: debug 2023-10-09T09:29:16.910+0000 7fa04c258700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T09:29:17.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:17 smithi016 bash[71225]: [09/Oct/2023:09:29:17] ENGINE Serving on http://:::9283 2023-10-09T09:29:17.500 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:17 smithi016 bash[71225]: [09/Oct/2023:09:29:17] ENGINE Bus STARTED 2023-10-09T09:29:18.177 INFO:journalctl@ceph.mon.a.smithi016.stdout:Oct 09 09:29:17 smithi016 bash[255959]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T09:29:18.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T09:29:18.372 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:18.372 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T09:29:18.372 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T09:29:18.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T09:29:18.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:18 smithi016 systemd[1]: Stopping Ceph mon.c for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:18.812 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:18 smithi016 bash[77892]: debug 2023-10-09T09:29:18.542+0000 7f2485bb5700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:29:18.812 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:18 smithi016 bash[77892]: debug 2023-10-09T09:29:18.542+0000 7f2485bb5700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T09:29:19.793 INFO:journalctl@ceph.mon.c.smithi016.stdout:Oct 09 09:29:19 smithi016 bash[256071]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T09:29:19.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T09:29:19.881 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:19.881 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T09:29:19.882 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T09:29:19.882 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T09:29:20.240 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:19 smithi094 systemd[1]: Stopping Ceph mon.b for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:20.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:20 smithi094 bash[59677]: debug 2023-10-09T09:29:20.081+0000 7f9d829c8700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:29:20.241 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:20 smithi094 bash[59677]: debug 2023-10-09T09:29:20.081+0000 7f9d829c8700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T09:29:21.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Oct 09 09:29:21 smithi094 bash[158643]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T09:29:21.353 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T09:29:21.395 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:21.395 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T09:29:21.396 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T09:29:21.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T09:29:21.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:21 smithi016 systemd[1]: Stopping Ceph mgr.y for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:21.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:21 smithi016 bash[71225]: debug 2023-10-09T09:29:21.550+0000 7f56bdf4e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:29:21.750 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:21 smithi016 bash[71225]: debug 2023-10-09T09:29:21.550+0000 7f56bdf4e700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-09T09:29:22.565 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Oct 09 09:29:22 smithi016 bash[256170]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T09:29:22.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T09:29:22.636 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:22.637 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T09:29:22.637 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T09:29:22.637 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T09:29:22.990 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 09:29:22 smithi094 systemd[1]: Stopping Ceph mgr.x for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:23.825 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Oct 09 09:29:23 smithi094 bash[158746]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T09:29:23.845 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T09:29:23.878 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:23.879 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T09:29:23.879 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T09:29:23.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T09:29:24.250 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 09:29:23 smithi016 systemd[1]: Stopping Ceph osd.0 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:24.250 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 09:29:24 smithi016 bash[83762]: debug 2023-10-09T09:29:24.074+0000 7f7aaa10a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:29:24.251 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 09:29:24 smithi016 bash[83762]: debug 2023-10-09T09:29:24.074+0000 7f7aaa10a700 -1 osd.0 414 *** Got signal Terminated *** 2023-10-09T09:29:24.251 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 09:29:24 smithi016 bash[83762]: debug 2023-10-09T09:29:24.074+0000 7f7aaa10a700 -1 osd.0 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:29:30.831 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 09:29:30 smithi016 bash[256273]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T09:29:31.250 INFO:journalctl@ceph.osd.0.smithi016.stdout:Oct 09 09:29:30 smithi016 bash[256342]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T09:29:33.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T09:29:33.858 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:33.858 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T09:29:33.858 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T09:29:33.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T09:29:34.099 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 09:29:33 smithi016 systemd[1]: Stopping Ceph osd.1 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:34.500 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 09:29:34 smithi016 bash[86637]: debug 2023-10-09T09:29:34.094+0000 7f3e5b2cd700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:29:34.500 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 09:29:34 smithi016 bash[86637]: debug 2023-10-09T09:29:34.094+0000 7f3e5b2cd700 -1 osd.1 414 *** Got signal Terminated *** 2023-10-09T09:29:34.500 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 09:29:34 smithi016 bash[86637]: debug 2023-10-09T09:29:34.094+0000 7f3e5b2cd700 -1 osd.1 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:29:40.852 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 09:29:40 smithi016 bash[256483]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T09:29:40.852 INFO:journalctl@ceph.osd.1.smithi016.stdout:Oct 09 09:29:40 smithi016 bash[256552]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T09:29:43.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T09:29:43.137 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:43.138 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T09:29:43.138 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T09:29:43.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T09:29:43.355 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 09:29:43 smithi016 systemd[1]: Stopping Ceph osd.2 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:43.750 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 09:29:43 smithi016 bash[89522]: debug 2023-10-09T09:29:43.350+0000 7f467af2d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:29:43.750 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 09:29:43 smithi016 bash[89522]: debug 2023-10-09T09:29:43.350+0000 7f467af2d700 -1 osd.2 414 *** Got signal Terminated *** 2023-10-09T09:29:43.751 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 09:29:43 smithi016 bash[89522]: debug 2023-10-09T09:29:43.350+0000 7f467af2d700 -1 osd.2 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:29:49.984 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 09:29:49 smithi016 bash[256693]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T09:29:49.985 INFO:journalctl@ceph.osd.2.smithi016.stdout:Oct 09 09:29:49 smithi016 bash[256763]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T09:29:51.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:51 smithi094 bash[39736]: debug 2023-10-09T09:29:51.156+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:51.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:51 smithi094 bash[43912]: debug 2023-10-09T09:29:51.100+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:51.500 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:51 smithi016 bash[92306]: debug 2023-10-09T09:29:51.130+0000 7fe389b6e700 -1 osd.3 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.682165+0000 front 2023-10-09T09:29:25.682341+0000 (oldest deadline 2023-10-09T09:29:50.981926+0000) 2023-10-09T09:29:52.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T09:29:52.112 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:29:52.113 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T09:29:52.113 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T09:29:52.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T09:29:52.352 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:52 smithi016 bash[92306]: debug 2023-10-09T09:29:52.170+0000 7fe389b6e700 -1 osd.3 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.682165+0000 front 2023-10-09T09:29:25.682341+0000 (oldest deadline 2023-10-09T09:29:50.981926+0000) 2023-10-09T09:29:52.352 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:52 smithi016 systemd[1]: Stopping Ceph osd.3 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:29:52.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:52 smithi094 bash[39736]: debug 2023-10-09T09:29:52.152+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:52.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:52 smithi094 bash[43912]: debug 2023-10-09T09:29:52.136+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:52.491 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:52 smithi094 bash[31426]: debug 2023-10-09T09:29:52.012+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:52.750 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:52 smithi016 bash[92306]: debug 2023-10-09T09:29:52.346+0000 7fe38d578700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:29:52.750 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:52 smithi016 bash[92306]: debug 2023-10-09T09:29:52.346+0000 7fe38d578700 -1 osd.3 414 *** Got signal Terminated *** 2023-10-09T09:29:52.750 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:52 smithi016 bash[92306]: debug 2023-10-09T09:29:52.346+0000 7fe38d578700 -1 osd.3 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:29:53.240 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:52 smithi094 bash[31426]: debug 2023-10-09T09:29:52.984+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:53.241 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:53 smithi094 bash[39736]: debug 2023-10-09T09:29:53.176+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:53.241 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:53 smithi094 bash[43912]: debug 2023-10-09T09:29:53.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:53.500 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:53 smithi016 bash[92306]: debug 2023-10-09T09:29:53.186+0000 7fe389b6e700 -1 osd.3 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.682165+0000 front 2023-10-09T09:29:25.682341+0000 (oldest deadline 2023-10-09T09:29:50.981926+0000) 2023-10-09T09:29:54.193 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:53 smithi094 bash[31426]: debug 2023-10-09T09:29:53.936+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:54.194 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:54 smithi094 bash[39736]: debug 2023-10-09T09:29:54.180+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:54.194 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:54 smithi094 bash[43912]: debug 2023-10-09T09:29:54.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:54.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:29:54 smithi094 bash[35505]: debug 2023-10-09T09:29:54.188+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:29:54.500 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:54 smithi016 bash[92306]: debug 2023-10-09T09:29:54.170+0000 7fe389b6e700 -1 osd.3 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.682165+0000 front 2023-10-09T09:29:25.682341+0000 (oldest deadline 2023-10-09T09:29:50.981926+0000) 2023-10-09T09:29:55.187 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:54 smithi094 bash[31426]: debug 2023-10-09T09:29:54.928+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:55.187 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:55 smithi094 bash[39736]: debug 2023-10-09T09:29:55.156+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:55.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:29:55 smithi094 bash[35505]: debug 2023-10-09T09:29:55.220+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:29:55.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:55 smithi094 bash[43912]: debug 2023-10-09T09:29:55.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:55.499 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:55 smithi016 bash[92306]: debug 2023-10-09T09:29:55.202+0000 7fe389b6e700 -1 osd.3 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.682165+0000 front 2023-10-09T09:29:25.682341+0000 (oldest deadline 2023-10-09T09:29:50.981926+0000) 2023-10-09T09:29:56.221 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:55 smithi094 bash[31426]: debug 2023-10-09T09:29:55.960+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:56.222 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:56 smithi094 bash[39736]: debug 2023-10-09T09:29:56.168+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:56.222 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:56 smithi094 bash[43912]: debug 2023-10-09T09:29:56.176+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:56.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:29:56 smithi094 bash[35505]: debug 2023-10-09T09:29:56.216+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:29:56.750 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:56 smithi016 bash[92306]: debug 2023-10-09T09:29:56.246+0000 7fe389b6e700 -1 osd.3 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.682165+0000 front 2023-10-09T09:29:25.682341+0000 (oldest deadline 2023-10-09T09:29:50.981926+0000) 2023-10-09T09:29:57.240 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:57 smithi094 bash[43912]: debug 2023-10-09T09:29:57.140+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:57.241 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:56 smithi094 bash[31426]: debug 2023-10-09T09:29:56.972+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:57.241 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:29:57 smithi094 bash[35505]: debug 2023-10-09T09:29:57.200+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:29:57.241 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:57 smithi094 bash[39736]: debug 2023-10-09T09:29:57.188+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:57.500 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:57 smithi016 bash[92306]: debug 2023-10-09T09:29:57.198+0000 7fe389b6e700 -1 osd.3 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.682165+0000 front 2023-10-09T09:29:25.682341+0000 (oldest deadline 2023-10-09T09:29:50.981926+0000) 2023-10-09T09:29:58.240 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:57 smithi094 bash[31426]: debug 2023-10-09T09:29:57.964+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:58.240 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:58 smithi094 bash[39736]: debug 2023-10-09T09:29:58.176+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:58.241 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:58 smithi094 bash[43912]: debug 2023-10-09T09:29:58.140+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:58.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:29:58 smithi094 bash[35505]: debug 2023-10-09T09:29:58.232+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:29:58.945 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:58 smithi016 bash[256904]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T09:29:58.945 INFO:journalctl@ceph.osd.3.smithi016.stdout:Oct 09 09:29:58 smithi016 bash[256971]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T09:29:59.240 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:58 smithi094 bash[31426]: debug 2023-10-09T09:29:58.928+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:29:59.241 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:29:59 smithi094 bash[39736]: debug 2023-10-09T09:29:59.148+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:29:59.241 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:29:59 smithi094 bash[43912]: debug 2023-10-09T09:29:59.108+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:29:59.740 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:29:59 smithi094 bash[35505]: debug 2023-10-09T09:29:59.236+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:00.240 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:00 smithi094 bash[43912]: debug 2023-10-09T09:30:00.112+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:00.241 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:29:59 smithi094 bash[31426]: debug 2023-10-09T09:29:59.920+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:30:00.241 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:00 smithi094 bash[39736]: debug 2023-10-09T09:30:00.112+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:00.740 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:00 smithi094 bash[35505]: debug 2023-10-09T09:30:00.240+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:01.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T09:30:01.070 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:30:01.071 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T09:30:01.071 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T09:30:01.071 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T09:30:01.218 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:00 smithi094 bash[31426]: debug 2023-10-09T09:30:00.960+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:30:01.218 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:00 smithi094 bash[31426]: debug 2023-10-09T09:30:00.960+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:34.724228+0000 front 2023-10-09T09:29:34.724477+0000 (oldest deadline 2023-10-09T09:30:00.024330+0000) 2023-10-09T09:30:01.218 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:01 smithi094 systemd[1]: Stopping Ceph osd.4 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:30:01.219 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[39736]: debug 2023-10-09T09:30:01.128+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:01.219 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[43912]: debug 2023-10-09T09:30:01.064+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:01.490 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[31426]: debug 2023-10-09T09:30:01.348+0000 7f7c04fbb700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:30:01.491 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[31426]: debug 2023-10-09T09:30:01.348+0000 7f7c04fbb700 -1 osd.4 414 *** Got signal Terminated *** 2023-10-09T09:30:01.491 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[31426]: debug 2023-10-09T09:30:01.348+0000 7f7c04fbb700 -1 osd.4 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:30:01.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[35505]: debug 2023-10-09T09:30:01.212+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:02.240 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[35505]: debug 2023-10-09T09:30:02.188+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:02.241 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[35505]: debug 2023-10-09T09:30:02.188+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:02.241 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[43912]: debug 2023-10-09T09:30:02.096+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:02.241 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[43912]: debug 2023-10-09T09:30:02.096+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:02.242 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[31426]: debug 2023-10-09T09:30:01.976+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:30:02.242 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:01 smithi094 bash[31426]: debug 2023-10-09T09:30:01.976+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:34.724228+0000 front 2023-10-09T09:29:34.724477+0000 (oldest deadline 2023-10-09T09:30:00.024330+0000) 2023-10-09T09:30:02.242 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[39736]: debug 2023-10-09T09:30:02.120+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:02.242 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[39736]: debug 2023-10-09T09:30:02.120+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:03.240 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:03 smithi094 bash[35505]: debug 2023-10-09T09:30:03.204+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:03.241 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:03 smithi094 bash[35505]: debug 2023-10-09T09:30:03.204+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:03.241 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:03 smithi094 bash[39736]: debug 2023-10-09T09:30:03.160+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:03.241 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:03 smithi094 bash[39736]: debug 2023-10-09T09:30:03.160+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:03.242 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[31426]: debug 2023-10-09T09:30:02.968+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:30:03.242 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:02 smithi094 bash[31426]: debug 2023-10-09T09:30:02.968+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:34.724228+0000 front 2023-10-09T09:29:34.724477+0000 (oldest deadline 2023-10-09T09:30:00.024330+0000) 2023-10-09T09:30:03.242 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:03 smithi094 bash[43912]: debug 2023-10-09T09:30:03.132+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:03.242 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:03 smithi094 bash[43912]: debug 2023-10-09T09:30:03.132+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:04.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[35505]: debug 2023-10-09T09:30:04.204+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:04.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[35505]: debug 2023-10-09T09:30:04.204+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:04.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[39736]: debug 2023-10-09T09:30:04.136+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:04.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[39736]: debug 2023-10-09T09:30:04.136+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:04.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[43912]: debug 2023-10-09T09:30:04.164+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:04.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[43912]: debug 2023-10-09T09:30:04.164+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:04.493 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[31426]: debug 2023-10-09T09:30:04.012+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:30:04.493 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:04 smithi094 bash[31426]: debug 2023-10-09T09:30:04.012+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:34.724228+0000 front 2023-10-09T09:29:34.724477+0000 (oldest deadline 2023-10-09T09:30:00.024330+0000) 2023-10-09T09:30:05.490 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[31426]: debug 2023-10-09T09:30:05.008+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:30:05.491 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[31426]: debug 2023-10-09T09:30:05.008+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:34.724228+0000 front 2023-10-09T09:29:34.724477+0000 (oldest deadline 2023-10-09T09:30:00.024330+0000) 2023-10-09T09:30:05.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[35505]: debug 2023-10-09T09:30:05.248+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:05.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[35505]: debug 2023-10-09T09:30:05.248+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:05.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[39736]: debug 2023-10-09T09:30:05.184+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:05.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[39736]: debug 2023-10-09T09:30:05.184+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:05.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[43912]: debug 2023-10-09T09:30:05.128+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:05.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:05 smithi094 bash[43912]: debug 2023-10-09T09:30:05.128+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:06.490 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[31426]: debug 2023-10-09T09:30:06.048+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.922671+0000 front 2023-10-09T09:29:25.922984+0000 (oldest deadline 2023-10-09T09:29:51.222383+0000) 2023-10-09T09:30:06.491 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[31426]: debug 2023-10-09T09:30:06.048+0000 7f7bff733700 -1 osd.4 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:34.724228+0000 front 2023-10-09T09:29:34.724477+0000 (oldest deadline 2023-10-09T09:30:00.024330+0000) 2023-10-09T09:30:06.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[35505]: debug 2023-10-09T09:30:06.296+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:06.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[35505]: debug 2023-10-09T09:30:06.296+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:06.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[39736]: debug 2023-10-09T09:30:06.176+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:06.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[39736]: debug 2023-10-09T09:30:06.176+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:06.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[43912]: debug 2023-10-09T09:30:06.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:06.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:06 smithi094 bash[43912]: debug 2023-10-09T09:30:06.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:07.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[35505]: debug 2023-10-09T09:30:07.308+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:07.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[35505]: debug 2023-10-09T09:30:07.308+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:07.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[39736]: debug 2023-10-09T09:30:07.208+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:07.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[39736]: debug 2023-10-09T09:30:07.208+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:07.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[43912]: debug 2023-10-09T09:30:07.172+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:07.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[43912]: debug 2023-10-09T09:30:07.172+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:07.798 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[158863]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T09:30:07.798 INFO:journalctl@ceph.osd.4.smithi094.stdout:Oct 09 09:30:07 smithi094 bash[158929]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T09:30:08.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:08 smithi094 bash[35505]: debug 2023-10-09T09:30:08.272+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:08.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:08 smithi094 bash[35505]: debug 2023-10-09T09:30:08.272+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:08.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:08 smithi094 bash[43912]: debug 2023-10-09T09:30:08.128+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:08.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:08 smithi094 bash[43912]: debug 2023-10-09T09:30:08.128+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:08.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:08 smithi094 bash[39736]: debug 2023-10-09T09:30:08.180+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:08.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:08 smithi094 bash[39736]: debug 2023-10-09T09:30:08.180+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:09.240 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:09 smithi094 bash[39736]: debug 2023-10-09T09:30:09.224+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:09.240 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:09 smithi094 bash[39736]: debug 2023-10-09T09:30:09.224+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:09.241 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:09 smithi094 bash[39736]: debug 2023-10-09T09:30:09.224+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:09.241 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:09 smithi094 bash[43912]: debug 2023-10-09T09:30:09.096+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:09.241 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:09 smithi094 bash[43912]: debug 2023-10-09T09:30:09.096+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:09.727 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:09 smithi094 bash[35505]: debug 2023-10-09T09:30:09.280+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:09.727 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:09 smithi094 bash[35505]: debug 2023-10-09T09:30:09.280+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:10.232 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[39736]: debug 2023-10-09T09:30:10.188+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:10.233 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[39736]: debug 2023-10-09T09:30:10.188+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:10.233 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[39736]: debug 2023-10-09T09:30:10.188+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:10.233 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[43912]: debug 2023-10-09T09:30:10.092+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:10.233 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[43912]: debug 2023-10-09T09:30:10.092+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:10.486 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[35505]: debug 2023-10-09T09:30:10.232+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:10.486 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[35505]: debug 2023-10-09T09:30:10.232+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:10.486 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[35505]: debug 2023-10-09T09:30:10.232+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.271386+0000 front 2023-10-09T09:29:48.271604+0000 (oldest deadline 2023-10-09T09:30:09.371113+0000) 2023-10-09T09:30:10.487 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[55195]: ts=2023-10-09T09:30:09.996Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-10-09T09:30:10.487 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[55195]: ts=2023-10-09T09:30:09.996Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-10-09T09:30:10.487 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[55195]: ts=2023-10-09T09:30:09.999Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-10-09T09:30:10.487 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[55195]: ts=2023-10-09T09:30:09.999Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-10-09T09:30:10.489 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T09:30:10.522 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:30:10.522 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T09:30:10.522 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T09:30:10.523 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T09:30:10.740 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:10 smithi094 systemd[1]: Stopping Ceph osd.5 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:30:11.074 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[35505]: debug 2023-10-09T09:30:10.740+0000 7f132aba2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:30:11.075 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[35505]: debug 2023-10-09T09:30:10.740+0000 7f132aba2700 -1 osd.5 414 *** Got signal Terminated *** 2023-10-09T09:30:11.075 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:10 smithi094 bash[35505]: debug 2023-10-09T09:30:10.740+0000 7f132aba2700 -1 osd.5 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:30:11.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[39736]: debug 2023-10-09T09:30:11.224+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:11.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[39736]: debug 2023-10-09T09:30:11.224+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:11.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[39736]: debug 2023-10-09T09:30:11.224+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:11.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[43912]: debug 2023-10-09T09:30:11.072+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:11.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[43912]: debug 2023-10-09T09:30:11.072+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:11.492 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[35505]: debug 2023-10-09T09:30:11.204+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:11.492 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[35505]: debug 2023-10-09T09:30:11.204+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:11.492 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:11 smithi094 bash[35505]: debug 2023-10-09T09:30:11.204+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.271386+0000 front 2023-10-09T09:29:48.271604+0000 (oldest deadline 2023-10-09T09:30:09.371113+0000) 2023-10-09T09:30:12.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[39736]: debug 2023-10-09T09:30:12.188+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:12.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[39736]: debug 2023-10-09T09:30:12.188+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:12.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[39736]: debug 2023-10-09T09:30:12.188+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:12.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[43912]: debug 2023-10-09T09:30:12.100+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:12.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[43912]: debug 2023-10-09T09:30:12.100+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:12.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[35505]: debug 2023-10-09T09:30:12.160+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:12.492 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[35505]: debug 2023-10-09T09:30:12.160+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:12.492 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:12 smithi094 bash[35505]: debug 2023-10-09T09:30:12.160+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.271386+0000 front 2023-10-09T09:29:48.271604+0000 (oldest deadline 2023-10-09T09:30:09.371113+0000) 2023-10-09T09:30:13.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[35505]: debug 2023-10-09T09:30:13.144+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:13.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[35505]: debug 2023-10-09T09:30:13.144+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:13.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[35505]: debug 2023-10-09T09:30:13.144+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.271386+0000 front 2023-10-09T09:29:48.271604+0000 (oldest deadline 2023-10-09T09:30:09.371113+0000) 2023-10-09T09:30:13.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[43912]: debug 2023-10-09T09:30:13.088+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:13.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[43912]: debug 2023-10-09T09:30:13.088+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:13.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[43912]: debug 2023-10-09T09:30:13.088+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:13.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[39736]: debug 2023-10-09T09:30:13.232+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:13.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[39736]: debug 2023-10-09T09:30:13.232+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:13.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:13 smithi094 bash[39736]: debug 2023-10-09T09:30:13.232+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:14.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[35505]: debug 2023-10-09T09:30:14.156+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:14.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[35505]: debug 2023-10-09T09:30:14.156+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:14.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[35505]: debug 2023-10-09T09:30:14.156+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.271386+0000 front 2023-10-09T09:29:48.271604+0000 (oldest deadline 2023-10-09T09:30:09.371113+0000) 2023-10-09T09:30:14.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[39736]: debug 2023-10-09T09:30:14.260+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:14.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[39736]: debug 2023-10-09T09:30:14.260+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:14.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[39736]: debug 2023-10-09T09:30:14.260+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:14.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[43912]: debug 2023-10-09T09:30:14.104+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:14.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[43912]: debug 2023-10-09T09:30:14.104+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:14.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:14 smithi094 bash[43912]: debug 2023-10-09T09:30:14.104+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:15.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[35505]: debug 2023-10-09T09:30:15.172+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:27.465980+0000 front 2023-10-09T09:29:27.466201+0000 (oldest deadline 2023-10-09T09:29:53.365990+0000) 2023-10-09T09:30:15.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[35505]: debug 2023-10-09T09:30:15.172+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:38.667393+0000 front 2023-10-09T09:29:38.667501+0000 (oldest deadline 2023-10-09T09:30:02.166984+0000) 2023-10-09T09:30:15.491 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[35505]: debug 2023-10-09T09:30:15.172+0000 7f1327b1f700 -1 osd.5 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.271386+0000 front 2023-10-09T09:29:48.271604+0000 (oldest deadline 2023-10-09T09:30:09.371113+0000) 2023-10-09T09:30:15.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[43912]: debug 2023-10-09T09:30:15.124+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:15.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[43912]: debug 2023-10-09T09:30:15.124+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:15.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[43912]: debug 2023-10-09T09:30:15.124+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:15.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[39736]: debug 2023-10-09T09:30:15.304+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:15.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[39736]: debug 2023-10-09T09:30:15.304+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:15.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:15 smithi094 bash[39736]: debug 2023-10-09T09:30:15.304+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:16.293 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:16 smithi094 bash[43912]: debug 2023-10-09T09:30:16.080+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:16.293 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:16 smithi094 bash[43912]: debug 2023-10-09T09:30:16.080+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:16.293 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:16 smithi094 bash[43912]: debug 2023-10-09T09:30:16.080+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:16.740 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:16 smithi094 bash[39736]: debug 2023-10-09T09:30:16.292+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:16.740 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:16 smithi094 bash[39736]: debug 2023-10-09T09:30:16.292+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:16.741 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:16 smithi094 bash[39736]: debug 2023-10-09T09:30:16.292+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:17.111 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:16 smithi094 bash[159067]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T09:30:17.490 INFO:journalctl@ceph.osd.5.smithi094.stdout:Oct 09 09:30:17 smithi094 bash[159130]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T09:30:17.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:17 smithi094 bash[43912]: debug 2023-10-09T09:30:17.108+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:17.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:17 smithi094 bash[43912]: debug 2023-10-09T09:30:17.108+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:17.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:17 smithi094 bash[43912]: debug 2023-10-09T09:30:17.108+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:17.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:17 smithi094 bash[39736]: debug 2023-10-09T09:30:17.284+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:17.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:17 smithi094 bash[39736]: debug 2023-10-09T09:30:17.284+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:17.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:17 smithi094 bash[39736]: debug 2023-10-09T09:30:17.284+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:18.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:18 smithi094 bash[39736]: debug 2023-10-09T09:30:18.280+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:18.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:18 smithi094 bash[39736]: debug 2023-10-09T09:30:18.280+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:18.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:18 smithi094 bash[39736]: debug 2023-10-09T09:30:18.280+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:18.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:18 smithi094 bash[43912]: debug 2023-10-09T09:30:18.140+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:18.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:18 smithi094 bash[43912]: debug 2023-10-09T09:30:18.140+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:18.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:18 smithi094 bash[43912]: debug 2023-10-09T09:30:18.140+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:19.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[39736]: debug 2023-10-09T09:30:19.272+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:19.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[39736]: debug 2023-10-09T09:30:19.272+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:19.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[39736]: debug 2023-10-09T09:30:19.272+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:19.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[43912]: debug 2023-10-09T09:30:19.160+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:19.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[43912]: debug 2023-10-09T09:30:19.160+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:19.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[43912]: debug 2023-10-09T09:30:19.160+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:19.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[43912]: debug 2023-10-09T09:30:19.160+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:19.611 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T09:30:19.642 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:30:19.643 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T09:30:19.643 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T09:30:19.643 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T09:30:19.870 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:19 smithi094 systemd[1]: Stopping Ceph osd.6 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:30:20.157 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[39736]: debug 2023-10-09T09:30:19.868+0000 7f6f3fe69700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:30:20.157 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[39736]: debug 2023-10-09T09:30:19.868+0000 7f6f3fe69700 -1 osd.6 414 *** Got signal Terminated *** 2023-10-09T09:30:20.157 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:19 smithi094 bash[39736]: debug 2023-10-09T09:30:19.868+0000 7f6f3fe69700 -1 osd.6 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:30:20.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:20 smithi094 bash[39736]: debug 2023-10-09T09:30:20.272+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:20.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:20 smithi094 bash[39736]: debug 2023-10-09T09:30:20.272+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:20.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:20 smithi094 bash[39736]: debug 2023-10-09T09:30:20.272+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:20.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:20 smithi094 bash[43912]: debug 2023-10-09T09:30:20.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:20.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:20 smithi094 bash[43912]: debug 2023-10-09T09:30:20.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:20.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:20 smithi094 bash[43912]: debug 2023-10-09T09:30:20.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:20.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:20 smithi094 bash[43912]: debug 2023-10-09T09:30:20.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:21.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:21 smithi094 bash[39736]: debug 2023-10-09T09:30:21.268+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:21.490 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:21 smithi094 bash[39736]: debug 2023-10-09T09:30:21.268+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:21.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:21 smithi094 bash[39736]: debug 2023-10-09T09:30:21.268+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:21.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:21 smithi094 bash[43912]: debug 2023-10-09T09:30:21.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:21.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:21 smithi094 bash[43912]: debug 2023-10-09T09:30:21.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:21.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:21 smithi094 bash[43912]: debug 2023-10-09T09:30:21.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:21.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:21 smithi094 bash[43912]: debug 2023-10-09T09:30:21.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:22.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[43912]: debug 2023-10-09T09:30:22.204+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:22.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[43912]: debug 2023-10-09T09:30:22.204+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:22.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[43912]: debug 2023-10-09T09:30:22.204+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:22.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[43912]: debug 2023-10-09T09:30:22.204+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:22.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[39736]: debug 2023-10-09T09:30:22.232+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:22.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[39736]: debug 2023-10-09T09:30:22.232+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:22.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[39736]: debug 2023-10-09T09:30:22.232+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:22.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:22 smithi094 bash[39736]: debug 2023-10-09T09:30:22.232+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:55.001615+0000 front 2023-10-09T09:29:55.001414+0000 (oldest deadline 2023-10-09T09:30:22.002028+0000) 2023-10-09T09:30:23.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[43912]: debug 2023-10-09T09:30:23.176+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:23.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[43912]: debug 2023-10-09T09:30:23.176+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:23.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[43912]: debug 2023-10-09T09:30:23.176+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:23.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[43912]: debug 2023-10-09T09:30:23.176+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:23.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[39736]: debug 2023-10-09T09:30:23.220+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:23.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[39736]: debug 2023-10-09T09:30:23.220+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:23.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[39736]: debug 2023-10-09T09:30:23.220+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:23.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:23 smithi094 bash[39736]: debug 2023-10-09T09:30:23.220+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:55.001615+0000 front 2023-10-09T09:29:55.001414+0000 (oldest deadline 2023-10-09T09:30:22.002028+0000) 2023-10-09T09:30:24.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[43912]: debug 2023-10-09T09:30:24.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:24.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[43912]: debug 2023-10-09T09:30:24.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:24.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[43912]: debug 2023-10-09T09:30:24.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:24.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[43912]: debug 2023-10-09T09:30:24.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:24.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[39736]: debug 2023-10-09T09:30:24.236+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:25.995437+0000 front 2023-10-09T09:29:25.995577+0000 (oldest deadline 2023-10-09T09:29:50.694969+0000) 2023-10-09T09:30:24.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[39736]: debug 2023-10-09T09:30:24.236+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.397963+0000 front 2023-10-09T09:29:36.398031+0000 (oldest deadline 2023-10-09T09:30:01.697871+0000) 2023-10-09T09:30:24.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[39736]: debug 2023-10-09T09:30:24.236+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:48.099758+0000 front 2023-10-09T09:29:48.099823+0000 (oldest deadline 2023-10-09T09:30:09.199499+0000) 2023-10-09T09:30:24.492 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:24 smithi094 bash[39736]: debug 2023-10-09T09:30:24.236+0000 7f6f3cde6700 -1 osd.6 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:55.001615+0000 front 2023-10-09T09:29:55.001414+0000 (oldest deadline 2023-10-09T09:30:22.002028+0000) 2023-10-09T09:30:25.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:25 smithi094 bash[43912]: debug 2023-10-09T09:30:25.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:25.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:25 smithi094 bash[43912]: debug 2023-10-09T09:30:25.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:25.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:25 smithi094 bash[43912]: debug 2023-10-09T09:30:25.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:25.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:25 smithi094 bash[43912]: debug 2023-10-09T09:30:25.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:26.047 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:25 smithi094 bash[159275]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T09:30:26.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:26 smithi094 bash[43912]: debug 2023-10-09T09:30:26.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:26.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:26 smithi094 bash[43912]: debug 2023-10-09T09:30:26.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:26.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:26 smithi094 bash[43912]: debug 2023-10-09T09:30:26.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:26.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:26 smithi094 bash[43912]: debug 2023-10-09T09:30:26.180+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:26.491 INFO:journalctl@ceph.osd.6.smithi094.stdout:Oct 09 09:30:26 smithi094 bash[159340]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T09:30:27.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:27 smithi094 bash[43912]: debug 2023-10-09T09:30:27.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:27.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:27 smithi094 bash[43912]: debug 2023-10-09T09:30:27.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:27.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:27 smithi094 bash[43912]: debug 2023-10-09T09:30:27.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:27.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:27 smithi094 bash[43912]: debug 2023-10-09T09:30:27.156+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:28.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.164+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:28.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.164+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:28.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.164+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:28.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.164+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:28.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.164+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.94:6804 osd.4 since back 2023-10-09T09:30:01.269015+0000 front 2023-10-09T09:30:01.268956+0000 (oldest deadline 2023-10-09T09:30:27.168958+0000) 2023-10-09T09:30:28.580 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T09:30:28.614 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:30:28.614 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T09:30:28.615 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T09:30:28.615 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T09:30:28.838 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 systemd[1]: Stopping Ceph osd.7 for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:30:29.149 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.836+0000 7fd6a33e2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:30:29.149 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.836+0000 7fd6a33e2700 -1 osd.7 414 *** Got signal Terminated *** 2023-10-09T09:30:29.149 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:28 smithi094 bash[43912]: debug 2023-10-09T09:30:28.836+0000 7fd6a33e2700 -1 osd.7 414 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:30:29.149 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:29 smithi094 bash[43912]: debug 2023-10-09T09:30:29.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:29.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:29 smithi094 bash[43912]: debug 2023-10-09T09:30:29.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:29.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:29 smithi094 bash[43912]: debug 2023-10-09T09:30:29.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:29.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:29 smithi094 bash[43912]: debug 2023-10-09T09:30:29.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:29.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:29 smithi094 bash[43912]: debug 2023-10-09T09:30:29.148+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.94:6804 osd.4 since back 2023-10-09T09:30:01.269015+0000 front 2023-10-09T09:30:01.268956+0000 (oldest deadline 2023-10-09T09:30:27.168958+0000) 2023-10-09T09:30:30.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:30 smithi094 bash[43912]: debug 2023-10-09T09:30:30.132+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:30.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:30 smithi094 bash[43912]: debug 2023-10-09T09:30:30.132+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:30.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:30 smithi094 bash[43912]: debug 2023-10-09T09:30:30.132+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:30.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:30 smithi094 bash[43912]: debug 2023-10-09T09:30:30.132+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:30.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:30 smithi094 bash[43912]: debug 2023-10-09T09:30:30.132+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.94:6804 osd.4 since back 2023-10-09T09:30:01.269015+0000 front 2023-10-09T09:30:01.268956+0000 (oldest deadline 2023-10-09T09:30:27.168958+0000) 2023-10-09T09:30:31.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:31 smithi094 bash[43912]: debug 2023-10-09T09:30:31.147+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:31.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:31 smithi094 bash[43912]: debug 2023-10-09T09:30:31.147+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:31.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:31 smithi094 bash[43912]: debug 2023-10-09T09:30:31.147+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:31.492 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:31 smithi094 bash[43912]: debug 2023-10-09T09:30:31.147+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:31.493 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:31 smithi094 bash[43912]: debug 2023-10-09T09:30:31.147+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.94:6804 osd.4 since back 2023-10-09T09:30:01.269015+0000 front 2023-10-09T09:30:01.268956+0000 (oldest deadline 2023-10-09T09:30:27.168958+0000) 2023-10-09T09:30:32.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:32 smithi094 bash[43912]: debug 2023-10-09T09:30:32.179+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:32.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:32 smithi094 bash[43912]: debug 2023-10-09T09:30:32.179+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:32.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:32 smithi094 bash[43912]: debug 2023-10-09T09:30:32.179+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:32.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:32 smithi094 bash[43912]: debug 2023-10-09T09:30:32.179+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:32.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:32 smithi094 bash[43912]: debug 2023-10-09T09:30:32.179+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.94:6804 osd.4 since back 2023-10-09T09:30:01.269015+0000 front 2023-10-09T09:30:01.268956+0000 (oldest deadline 2023-10-09T09:30:27.168958+0000) 2023-10-09T09:30:33.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:33 smithi094 bash[43912]: debug 2023-10-09T09:30:33.167+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-10-09T09:29:28.460194+0000 front 2023-10-09T09:29:28.460122+0000 (oldest deadline 2023-10-09T09:29:50.159763+0000) 2023-10-09T09:30:33.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:33 smithi094 bash[43912]: debug 2023-10-09T09:30:33.167+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-10-09T09:29:36.764527+0000 front 2023-10-09T09:29:36.764550+0000 (oldest deadline 2023-10-09T09:30:01.464214+0000) 2023-10-09T09:30:33.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:33 smithi094 bash[43912]: debug 2023-10-09T09:30:33.167+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-10-09T09:29:46.165868+0000 front 2023-10-09T09:29:46.165917+0000 (oldest deadline 2023-10-09T09:30:12.566530+0000) 2023-10-09T09:30:33.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:33 smithi094 bash[43912]: debug 2023-10-09T09:30:33.167+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-10-09T09:29:56.667934+0000 front 2023-10-09T09:29:56.667897+0000 (oldest deadline 2023-10-09T09:30:18.367722+0000) 2023-10-09T09:30:33.491 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:33 smithi094 bash[43912]: debug 2023-10-09T09:30:33.167+0000 7fd69cb58700 -1 osd.7 414 heartbeat_check: no reply from 172.21.15.94:6804 osd.4 since back 2023-10-09T09:30:01.269015+0000 front 2023-10-09T09:30:01.268956+0000 (oldest deadline 2023-10-09T09:30:27.168958+0000) 2023-10-09T09:30:35.019 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:34 smithi094 bash[159482]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T09:30:35.490 INFO:journalctl@ceph.osd.7.smithi094.stdout:Oct 09 09:30:35 smithi094 bash[159546]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T09:30:37.297 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T09:30:37.322 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:30:37.323 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T09:30:37.323 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T09:30:37.323 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T09:30:37.543 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 systemd[1]: Stopping Ceph prometheus.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:30:37.544 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:30:37.544 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T09:30:37.544 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T09:30:37.544 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T09:30:37.544 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T09:30:37.544 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T09:30:37.545 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T09:30:37.545 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T09:30:37.545 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.541Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T09:30:37.545 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.543Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T09:30:37.825 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.543Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T09:30:37.825 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Oct 09 09:30:37 smithi094 bash[55195]: ts=2023-10-09T09:30:37.543Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T09:30:38.466 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T09:30:38.496 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:30:38.496 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T09:30:38.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T09:30:42.000 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:41 smithi016 bash[70330]: level=warn ts=2023-10-09T09:30:41.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi016.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.16:8443: connect: connection refused" 2023-10-09T09:30:42.257 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:41 smithi016 bash[70330]: level=warn ts=2023-10-09T09:30:41.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi094.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.94:8443: connect: connection refused" 2023-10-09T09:30:45.510 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 09:30:45 smithi016 systemd[1]: Stopping Ceph node-exporter.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:30:46.493 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 09:30:46 smithi016 bash[257335]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T09:30:46.493 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 09:30:46 smithi016 bash[257397]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T09:30:46.493 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 09:30:46 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:30:46.493 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 09:30:46 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T09:30:46.493 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Oct 09 09:30:46 smithi016 systemd[1]: Stopped Ceph node-exporter.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T09:30:46.750 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:46 smithi016 systemd[1]: Stopping Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:30:46.750 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:46 smithi016 bash[70330]: level=info ts=2023-10-09T09:30:46.636Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:30:47.744 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:47 smithi016 bash[257458]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:30:47.745 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:47 smithi016 bash[257522]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:30:47.745 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:47 smithi016 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T09:30:47.745 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Oct 09 09:30:47 smithi016 systemd[1]: Stopped Ceph alertmanager.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T09:30:57.038 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T09:31:03.887 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 09:31:03 smithi094 systemd[1]: Stopping Ceph node-exporter.b for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:31:04.963 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 09:31:04 smithi094 bash[160009]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T09:31:04.963 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 09:31:04 smithi094 bash[160071]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T09:31:04.963 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 09:31:04 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:31:04.964 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 09:31:04 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T09:31:04.964 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Oct 09 09:31:04 smithi094 systemd[1]: Stopped Ceph node-exporter.b for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T09:31:05.240 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 09:31:04 smithi094 systemd[1]: Stopping Ceph grafana.a for f674ade6-667c-11ee-8db5-212e2dc638e7... 2023-10-09T09:31:05.241 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 09:31:05 smithi094 bash[53813]: t=2023-10-09T09:31:05+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T09:31:06.240 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 09:31:05 smithi094 bash[160133]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T09:31:06.241 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 09:31:06 smithi094 bash[160207]: Error response from daemon: No such container: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T09:31:06.241 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 09:31:06 smithi094 systemd[1]: ceph-f674ade6-667c-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T09:31:06.241 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Oct 09 09:31:06 smithi094 systemd[1]: Stopped Ceph grafana.a for f674ade6-667c-11ee-8db5-212e2dc638e7. 2023-10-09T09:31:09.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T09:31:09.908 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T09:31:09.923 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T09:31:09.926 DEBUG:teuthology.misc:Transferring archived files from smithi016:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587/remote/smithi016/crash 2023-10-09T09:31:09.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo tar c -f - -C /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T09:31:09.989 INFO:teuthology.orchestra.run.smithi016.stderr:tar: /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T09:31:09.990 INFO:teuthology.orchestra.run.smithi016.stderr:tar: Error is not recoverable: exiting now 2023-10-09T09:31:09.991 DEBUG:teuthology.misc:Transferring archived files from smithi094:/var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587/remote/smithi094/crash 2023-10-09T09:31:09.992 DEBUG:teuthology.orchestra.run.smithi094:> sudo tar c -f - -C /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T09:31:10.026 INFO:teuthology.orchestra.run.smithi094.stderr:tar: /var/lib/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T09:31:10.027 INFO:teuthology.orchestra.run.smithi094.stderr:tar: Error is not recoverable: exiting now 2023-10-09T09:31:10.028 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T09:31:10.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T09:31:10.053 INFO:teuthology.orchestra.run.smithi016.stderr:grep: /var/log/ceph/f674ade6-667c-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T09:31:10.055 INFO:tasks.cephadm:Compressing logs... 2023-10-09T09:31:10.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T09:31:10.098 DEBUG:teuthology.orchestra.run.smithi094:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T09:31:10.114 INFO:teuthology.orchestra.run.smithi094.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T09:31:10.126 INFO:teuthology.orchestra.run.smithi016.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T09:41:19.933 INFO:tasks.cephadm:Archiving logs... 2023-10-09T09:41:19.935 DEBUG:teuthology.misc:Transferring archived files from smithi016:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587/remote/smithi016/log 2023-10-09T09:41:19.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T09:42:35.170 DEBUG:teuthology.misc:Transferring archived files from smithi094:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587/remote/smithi094/log 2023-10-09T09:42:35.171 DEBUG:teuthology.orchestra.run.smithi094:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T09:43:22.076 INFO:tasks.cephadm:Removing cluster... 2023-10-09T09:43:22.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 --force 2023-10-09T09:43:24.183 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f674ade6-667c-11ee-8db5-212e2dc638e7 --force 2023-10-09T09:43:26.100 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T09:43:26.100 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T09:43:26.117 DEBUG:teuthology.orchestra.run.smithi094:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T09:43:26.124 INFO:tasks.cephadm:Teardown complete 2023-10-09T09:43:26.125 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T09:43:26.135 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T09:43:26.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T09:43:26.163 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T09:43:26.243 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T09:43:26.252 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T09:43:26.253 DEBUG:teuthology.orchestra.run.smithi016:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T09:43:26.255 DEBUG:teuthology.orchestra.run.smithi094:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T09:43:26.307 INFO:teuthology.orchestra.run.smithi016.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T09:43:26.308 INFO:teuthology.orchestra.run.smithi016.stdout:============================================================================== 2023-10-09T09:43:26.308 INFO:teuthology.orchestra.run.smithi016.stdout:*hv01.front.sepi 44.4.53.2 2 u 50 128 377 0.165 2.015 0.260 2023-10-09T09:43:26.309 INFO:teuthology.orchestra.run.smithi016.stdout:+hv02.front.sepi 63.231.80.2 3 u 46 128 377 0.147 1.039 0.222 2023-10-09T09:43:26.309 INFO:teuthology.orchestra.run.smithi016.stdout:+hv03.front.sepi 74.6.168.72 3 u 123 128 377 0.180 2.058 0.237 2023-10-09T09:43:26.309 INFO:teuthology.orchestra.run.smithi016.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T09:43:26.331 INFO:teuthology.orchestra.run.smithi094.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T09:43:26.331 INFO:teuthology.orchestra.run.smithi094.stdout:============================================================================== 2023-10-09T09:43:26.331 INFO:teuthology.orchestra.run.smithi094.stdout:*hv01.front.sepi 44.4.53.2 2 u 107 128 377 0.105 0.640 0.098 2023-10-09T09:43:26.331 INFO:teuthology.orchestra.run.smithi094.stdout:+hv02.front.sepi 63.231.80.2 3 u 50 128 377 0.081 -0.325 0.068 2023-10-09T09:43:26.331 INFO:teuthology.orchestra.run.smithi094.stdout:+hv03.front.sepi 74.6.168.72 3 u 49 128 377 0.085 0.655 0.115 2023-10-09T09:43:26.332 INFO:teuthology.orchestra.run.smithi094.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T09:43:26.333 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T09:43:26.342 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T09:43:26.343 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T09:43:26.378 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T09:43:26.428 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T09:43:26.478 INFO:teuthology.task.internal:Duration was 5339.444890 seconds 2023-10-09T09:43:26.478 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T09:43:26.528 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T09:43:26.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T09:43:26.531 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T09:43:26.645 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T09:43:26.645 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi016.front.sepia.ceph.com 2023-10-09T09:43:26.645 DEBUG:teuthology.orchestra.run.smithi016:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T09:43:26.659 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi094.front.sepia.ceph.com 2023-10-09T09:43:26.659 DEBUG:teuthology.orchestra.run.smithi094:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T09:43:26.679 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T09:43:26.680 DEBUG:teuthology.orchestra.run.smithi016:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T09:43:26.703 DEBUG:teuthology.orchestra.run.smithi094:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T09:43:26.891 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T09:43:26.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T09:43:26.893 DEBUG:teuthology.orchestra.run.smithi094:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T09:43:27.418 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T09:43:27.429 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T09:43:27.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T09:43:27.473 DEBUG:teuthology.orchestra.run.smithi094:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T09:43:27.502 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T09:43:27.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T09:43:27.518 DEBUG:teuthology.orchestra.run.smithi094:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T09:43:27.530 INFO:teuthology.orchestra.run.smithi016.stdout:kernel.core_pattern = core 2023-10-09T09:43:27.560 INFO:teuthology.orchestra.run.smithi094.stdout:kernel.core_pattern = core 2023-10-09T09:43:27.593 DEBUG:teuthology.orchestra.run.smithi016:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T09:43:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.606 DEBUG:teuthology.orchestra.run.smithi094:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T09:43:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.642 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T09:43:27.652 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T09:43:27.653 DEBUG:teuthology.misc:Transferring archived files from smithi016:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587/remote/smithi016 2023-10-09T09:43:27.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T09:43:27.706 DEBUG:teuthology.misc:Transferring archived files from smithi094:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587/remote/smithi094 2023-10-09T09:43:27.706 DEBUG:teuthology.orchestra.run.smithi094:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T09:43:27.745 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T09:43:27.746 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T09:43:27.748 DEBUG:teuthology.orchestra.run.smithi094:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T09:43:27.787 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T09:43:27.799 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T09:43:27.799 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T09:43:27.828 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T09:43:28.027 DEBUG:teuthology.orchestra.run.smithi016:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T09:43:28.031 DEBUG:teuthology.orchestra.run.smithi094:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T09:43:28.123 INFO:teuthology.orchestra.run.smithi016.stdout: 1048609 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 09:43 /home/ubuntu/cephtest 2023-10-09T09:43:28.147 INFO:teuthology.orchestra.run.smithi016.stdout: 1195926 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 09:24 /home/ubuntu/cephtest/mnt.0 2023-10-09T09:43:28.147 INFO:teuthology.orchestra.run.smithi016.stdout: 1195928 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 09:24 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:43:28.148 INFO:teuthology.orchestra.run.smithi016.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T09:43:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:28.263 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi016 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T09:43:28.270 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T09:43:28.281 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T09:43:28.342 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi016 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T09:43:28.342 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T09:43:28.348 INFO:teuthology.orchestra.run.smithi094.stdout: 1048617 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 09:43 /home/ubuntu/cephtest 2023-10-09T09:43:28.377 DEBUG:teuthology.nuke:shortname: smithi016 2023-10-09T09:43:28.378 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T09:43:28.410 DEBUG:teuthology.nuke:shortname: smithi094 2023-10-09T09:43:28.410 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T09:43:28.414 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi016.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:05:17.397456', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T09:43:28.439 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi094.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417587', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:05:17.395501', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T09:43:28.469 INFO:teuthology.orchestra.console.smithi016:Power off 2023-10-09T09:43:28.470 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T09:43:28.493 DEBUG:teuthology.orchestra.console.smithi016:power off output: Chassis Power Control: Down/Off 2023-10-09T09:43:28.493 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:28.509 INFO:teuthology.orchestra.console.smithi094:Power off 2023-10-09T09:43:28.510 DEBUG:teuthology.orchestra.console.smithi094:pexpect command: ipmitool -H smithi094.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T09:43:28.518 DEBUG:teuthology.orchestra.console.smithi016:check power output: Chassis Power is on 2023-10-09T09:43:28.534 DEBUG:teuthology.orchestra.console.smithi094:power off output: Chassis Power Control: Down/Off 2023-10-09T09:43:28.534 DEBUG:teuthology.orchestra.console.smithi094:pexpect command: ipmitool -H smithi094.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:28.557 DEBUG:teuthology.orchestra.console.smithi094:check power output: Chassis Power is on 2023-10-09T09:43:32.522 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:32.559 DEBUG:teuthology.orchestra.console.smithi094:pexpect command: ipmitool -H smithi094.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:32.633 DEBUG:teuthology.orchestra.console.smithi016:check power output: Chassis Power is on 2023-10-09T09:43:32.670 DEBUG:teuthology.orchestra.console.smithi094:check power output: Chassis Power is on 2023-10-09T09:43:36.638 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:36.671 DEBUG:teuthology.orchestra.console.smithi094:pexpect command: ipmitool -H smithi094.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:36.749 DEBUG:teuthology.orchestra.console.smithi016:check power output: Chassis Power is on 2023-10-09T09:43:36.782 DEBUG:teuthology.orchestra.console.smithi094:check power output: Chassis Power is on 2023-10-09T09:43:40.754 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:40.782 DEBUG:teuthology.orchestra.console.smithi094:pexpect command: ipmitool -H smithi094.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T09:43:40.864 DEBUG:teuthology.orchestra.console.smithi016:check power output: Chassis Power is off 2023-10-09T09:43:40.898 DEBUG:teuthology.orchestra.console.smithi094:check power output: Chassis Power is off 2023-10-09T09:43:40.966 INFO:teuthology.orchestra.console.smithi016:Power off completed 2023-10-09T09:43:40.999 INFO:teuthology.orchestra.console.smithi094:Power off completed 2023-10-09T09:43:41.104 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5339.444890260696 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=813ca7a19a004579be8710c5e5b9edb1 status: fail success: false 2023-10-09T09:43:41.104 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T09:43:41.187 INFO:teuthology.run:FAIL